Loading...
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33
34#include <drm/drm_atomic_helper.h>
35#include <drm/drm_probe_helper.h>
36#include <drm/amdgpu_drm.h>
37#include <linux/vgaarb.h>
38#include <linux/vga_switcheroo.h>
39#include <linux/efi.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_i2c.h"
43#include "atom.h"
44#include "amdgpu_atombios.h"
45#include "amdgpu_atomfirmware.h"
46#include "amd_pcie.h"
47#ifdef CONFIG_DRM_AMDGPU_SI
48#include "si.h"
49#endif
50#ifdef CONFIG_DRM_AMDGPU_CIK
51#include "cik.h"
52#endif
53#include "vi.h"
54#include "soc15.h"
55#include "nv.h"
56#include "bif/bif_4_1_d.h"
57#include <linux/pci.h>
58#include <linux/firmware.h>
59#include "amdgpu_vf_error.h"
60
61#include "amdgpu_amdkfd.h"
62#include "amdgpu_pm.h"
63
64#include "amdgpu_xgmi.h"
65#include "amdgpu_ras.h"
66#include "amdgpu_pmu.h"
67#include "amdgpu_fru_eeprom.h"
68
69#include <linux/suspend.h>
70#include <drm/task_barrier.h>
71#include <linux/pm_runtime.h>
72
73MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
74MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
75MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
76MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
77MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
78MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
79MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
80MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
81MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
82MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
83
84#define AMDGPU_RESUME_MS 2000
85
86const char *amdgpu_asic_name[] = {
87 "TAHITI",
88 "PITCAIRN",
89 "VERDE",
90 "OLAND",
91 "HAINAN",
92 "BONAIRE",
93 "KAVERI",
94 "KABINI",
95 "HAWAII",
96 "MULLINS",
97 "TOPAZ",
98 "TONGA",
99 "FIJI",
100 "CARRIZO",
101 "STONEY",
102 "POLARIS10",
103 "POLARIS11",
104 "POLARIS12",
105 "VEGAM",
106 "VEGA10",
107 "VEGA12",
108 "VEGA20",
109 "RAVEN",
110 "ARCTURUS",
111 "RENOIR",
112 "NAVI10",
113 "NAVI14",
114 "NAVI12",
115 "SIENNA_CICHLID",
116 "NAVY_FLOUNDER",
117 "LAST",
118};
119
120/**
121 * DOC: pcie_replay_count
122 *
123 * The amdgpu driver provides a sysfs API for reporting the total number
124 * of PCIe replays (NAKs)
125 * The file pcie_replay_count is used for this and returns the total
126 * number of replays as a sum of the NAKs generated and NAKs received
127 */
128
129static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
130 struct device_attribute *attr, char *buf)
131{
132 struct drm_device *ddev = dev_get_drvdata(dev);
133 struct amdgpu_device *adev = ddev->dev_private;
134 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
135
136 return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
137}
138
139static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
140 amdgpu_device_get_pcie_replay_count, NULL);
141
142static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
143
144/**
145 * DOC: product_name
146 *
147 * The amdgpu driver provides a sysfs API for reporting the product name
148 * for the device
149 * The file serial_number is used for this and returns the product name
150 * as returned from the FRU.
151 * NOTE: This is only available for certain server cards
152 */
153
154static ssize_t amdgpu_device_get_product_name(struct device *dev,
155 struct device_attribute *attr, char *buf)
156{
157 struct drm_device *ddev = dev_get_drvdata(dev);
158 struct amdgpu_device *adev = ddev->dev_private;
159
160 return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_name);
161}
162
163static DEVICE_ATTR(product_name, S_IRUGO,
164 amdgpu_device_get_product_name, NULL);
165
166/**
167 * DOC: product_number
168 *
169 * The amdgpu driver provides a sysfs API for reporting the part number
170 * for the device
171 * The file serial_number is used for this and returns the part number
172 * as returned from the FRU.
173 * NOTE: This is only available for certain server cards
174 */
175
176static ssize_t amdgpu_device_get_product_number(struct device *dev,
177 struct device_attribute *attr, char *buf)
178{
179 struct drm_device *ddev = dev_get_drvdata(dev);
180 struct amdgpu_device *adev = ddev->dev_private;
181
182 return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_number);
183}
184
185static DEVICE_ATTR(product_number, S_IRUGO,
186 amdgpu_device_get_product_number, NULL);
187
188/**
189 * DOC: serial_number
190 *
191 * The amdgpu driver provides a sysfs API for reporting the serial number
192 * for the device
193 * The file serial_number is used for this and returns the serial number
194 * as returned from the FRU.
195 * NOTE: This is only available for certain server cards
196 */
197
198static ssize_t amdgpu_device_get_serial_number(struct device *dev,
199 struct device_attribute *attr, char *buf)
200{
201 struct drm_device *ddev = dev_get_drvdata(dev);
202 struct amdgpu_device *adev = ddev->dev_private;
203
204 return snprintf(buf, PAGE_SIZE, "%s\n", adev->serial);
205}
206
207static DEVICE_ATTR(serial_number, S_IRUGO,
208 amdgpu_device_get_serial_number, NULL);
209
210/**
211 * amdgpu_device_supports_boco - Is the device a dGPU with HG/PX power control
212 *
213 * @dev: drm_device pointer
214 *
215 * Returns true if the device is a dGPU with HG/PX power control,
216 * otherwise return false.
217 */
218bool amdgpu_device_supports_boco(struct drm_device *dev)
219{
220 struct amdgpu_device *adev = dev->dev_private;
221
222 if (adev->flags & AMD_IS_PX)
223 return true;
224 return false;
225}
226
227/**
228 * amdgpu_device_supports_baco - Does the device support BACO
229 *
230 * @dev: drm_device pointer
231 *
232 * Returns true if the device supporte BACO,
233 * otherwise return false.
234 */
235bool amdgpu_device_supports_baco(struct drm_device *dev)
236{
237 struct amdgpu_device *adev = dev->dev_private;
238
239 return amdgpu_asic_supports_baco(adev);
240}
241
242/**
243 * VRAM access helper functions.
244 *
245 * amdgpu_device_vram_access - read/write a buffer in vram
246 *
247 * @adev: amdgpu_device pointer
248 * @pos: offset of the buffer in vram
249 * @buf: virtual address of the buffer in system memory
250 * @size: read/write size, sizeof(@buf) must > @size
251 * @write: true - write to vram, otherwise - read from vram
252 */
253void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
254 uint32_t *buf, size_t size, bool write)
255{
256 unsigned long flags;
257 uint32_t hi = ~0;
258 uint64_t last;
259
260
261#ifdef CONFIG_64BIT
262 last = min(pos + size, adev->gmc.visible_vram_size);
263 if (last > pos) {
264 void __iomem *addr = adev->mman.aper_base_kaddr + pos;
265 size_t count = last - pos;
266
267 if (write) {
268 memcpy_toio(addr, buf, count);
269 mb();
270 amdgpu_asic_flush_hdp(adev, NULL);
271 } else {
272 amdgpu_asic_invalidate_hdp(adev, NULL);
273 mb();
274 memcpy_fromio(buf, addr, count);
275 }
276
277 if (count == size)
278 return;
279
280 pos += count;
281 buf += count / 4;
282 size -= count;
283 }
284#endif
285
286 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
287 for (last = pos + size; pos < last; pos += 4) {
288 uint32_t tmp = pos >> 31;
289
290 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
291 if (tmp != hi) {
292 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
293 hi = tmp;
294 }
295 if (write)
296 WREG32_NO_KIQ(mmMM_DATA, *buf++);
297 else
298 *buf++ = RREG32_NO_KIQ(mmMM_DATA);
299 }
300 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
301}
302
303/*
304 * MMIO register access helper functions.
305 */
306/**
307 * amdgpu_mm_rreg - read a memory mapped IO register
308 *
309 * @adev: amdgpu_device pointer
310 * @reg: dword aligned register offset
311 * @acc_flags: access flags which require special behavior
312 *
313 * Returns the 32 bit value from the offset specified.
314 */
315uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
316 uint32_t acc_flags)
317{
318 uint32_t ret;
319
320 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
321 return amdgpu_kiq_rreg(adev, reg);
322
323 if ((reg * 4) < adev->rmmio_size)
324 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
325 else {
326 unsigned long flags;
327
328 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
329 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
330 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
331 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
332 }
333 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
334 return ret;
335}
336
337/*
338 * MMIO register read with bytes helper functions
339 * @offset:bytes offset from MMIO start
340 *
341*/
342
343/**
344 * amdgpu_mm_rreg8 - read a memory mapped IO register
345 *
346 * @adev: amdgpu_device pointer
347 * @offset: byte aligned register offset
348 *
349 * Returns the 8 bit value from the offset specified.
350 */
351uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
352 if (offset < adev->rmmio_size)
353 return (readb(adev->rmmio + offset));
354 BUG();
355}
356
357/*
358 * MMIO register write with bytes helper functions
359 * @offset:bytes offset from MMIO start
360 * @value: the value want to be written to the register
361 *
362*/
363/**
364 * amdgpu_mm_wreg8 - read a memory mapped IO register
365 *
366 * @adev: amdgpu_device pointer
367 * @offset: byte aligned register offset
368 * @value: 8 bit value to write
369 *
370 * Writes the value specified to the offset specified.
371 */
372void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
373 if (offset < adev->rmmio_size)
374 writeb(value, adev->rmmio + offset);
375 else
376 BUG();
377}
378
379void static inline amdgpu_mm_wreg_mmio(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t acc_flags)
380{
381 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
382
383 if ((reg * 4) < adev->rmmio_size)
384 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
385 else {
386 unsigned long flags;
387
388 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
389 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
390 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
391 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
392 }
393}
394
395/**
396 * amdgpu_mm_wreg - write to a memory mapped IO register
397 *
398 * @adev: amdgpu_device pointer
399 * @reg: dword aligned register offset
400 * @v: 32 bit value to write to the register
401 * @acc_flags: access flags which require special behavior
402 *
403 * Writes the value specified to the offset specified.
404 */
405void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
406 uint32_t acc_flags)
407{
408 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
409 return amdgpu_kiq_wreg(adev, reg, v);
410
411 amdgpu_mm_wreg_mmio(adev, reg, v, acc_flags);
412}
413
414/*
415 * amdgpu_mm_wreg_mmio_rlc - write register either with mmio or with RLC path if in range
416 *
417 * this function is invoked only the debugfs register access
418 * */
419void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
420 uint32_t acc_flags)
421{
422 if (amdgpu_sriov_fullaccess(adev) &&
423 adev->gfx.rlc.funcs &&
424 adev->gfx.rlc.funcs->is_rlcg_access_range) {
425
426 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
427 return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
428 }
429
430 amdgpu_mm_wreg_mmio(adev, reg, v, acc_flags);
431}
432
433/**
434 * amdgpu_io_rreg - read an IO register
435 *
436 * @adev: amdgpu_device pointer
437 * @reg: dword aligned register offset
438 *
439 * Returns the 32 bit value from the offset specified.
440 */
441u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
442{
443 if ((reg * 4) < adev->rio_mem_size)
444 return ioread32(adev->rio_mem + (reg * 4));
445 else {
446 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
447 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
448 }
449}
450
451/**
452 * amdgpu_io_wreg - write to an IO register
453 *
454 * @adev: amdgpu_device pointer
455 * @reg: dword aligned register offset
456 * @v: 32 bit value to write to the register
457 *
458 * Writes the value specified to the offset specified.
459 */
460void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
461{
462 if ((reg * 4) < adev->rio_mem_size)
463 iowrite32(v, adev->rio_mem + (reg * 4));
464 else {
465 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
466 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
467 }
468}
469
470/**
471 * amdgpu_mm_rdoorbell - read a doorbell dword
472 *
473 * @adev: amdgpu_device pointer
474 * @index: doorbell index
475 *
476 * Returns the value in the doorbell aperture at the
477 * requested doorbell index (CIK).
478 */
479u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
480{
481 if (index < adev->doorbell.num_doorbells) {
482 return readl(adev->doorbell.ptr + index);
483 } else {
484 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
485 return 0;
486 }
487}
488
489/**
490 * amdgpu_mm_wdoorbell - write a doorbell dword
491 *
492 * @adev: amdgpu_device pointer
493 * @index: doorbell index
494 * @v: value to write
495 *
496 * Writes @v to the doorbell aperture at the
497 * requested doorbell index (CIK).
498 */
499void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
500{
501 if (index < adev->doorbell.num_doorbells) {
502 writel(v, adev->doorbell.ptr + index);
503 } else {
504 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
505 }
506}
507
508/**
509 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
510 *
511 * @adev: amdgpu_device pointer
512 * @index: doorbell index
513 *
514 * Returns the value in the doorbell aperture at the
515 * requested doorbell index (VEGA10+).
516 */
517u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
518{
519 if (index < adev->doorbell.num_doorbells) {
520 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
521 } else {
522 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
523 return 0;
524 }
525}
526
527/**
528 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
529 *
530 * @adev: amdgpu_device pointer
531 * @index: doorbell index
532 * @v: value to write
533 *
534 * Writes @v to the doorbell aperture at the
535 * requested doorbell index (VEGA10+).
536 */
537void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
538{
539 if (index < adev->doorbell.num_doorbells) {
540 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
541 } else {
542 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
543 }
544}
545
546/**
547 * amdgpu_invalid_rreg - dummy reg read function
548 *
549 * @adev: amdgpu device pointer
550 * @reg: offset of register
551 *
552 * Dummy register read function. Used for register blocks
553 * that certain asics don't have (all asics).
554 * Returns the value in the register.
555 */
556static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
557{
558 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
559 BUG();
560 return 0;
561}
562
563/**
564 * amdgpu_invalid_wreg - dummy reg write function
565 *
566 * @adev: amdgpu device pointer
567 * @reg: offset of register
568 * @v: value to write to the register
569 *
570 * Dummy register read function. Used for register blocks
571 * that certain asics don't have (all asics).
572 */
573static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
574{
575 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
576 reg, v);
577 BUG();
578}
579
580/**
581 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
582 *
583 * @adev: amdgpu device pointer
584 * @reg: offset of register
585 *
586 * Dummy register read function. Used for register blocks
587 * that certain asics don't have (all asics).
588 * Returns the value in the register.
589 */
590static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
591{
592 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
593 BUG();
594 return 0;
595}
596
597/**
598 * amdgpu_invalid_wreg64 - dummy reg write function
599 *
600 * @adev: amdgpu device pointer
601 * @reg: offset of register
602 * @v: value to write to the register
603 *
604 * Dummy register read function. Used for register blocks
605 * that certain asics don't have (all asics).
606 */
607static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
608{
609 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
610 reg, v);
611 BUG();
612}
613
614/**
615 * amdgpu_block_invalid_rreg - dummy reg read function
616 *
617 * @adev: amdgpu device pointer
618 * @block: offset of instance
619 * @reg: offset of register
620 *
621 * Dummy register read function. Used for register blocks
622 * that certain asics don't have (all asics).
623 * Returns the value in the register.
624 */
625static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
626 uint32_t block, uint32_t reg)
627{
628 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
629 reg, block);
630 BUG();
631 return 0;
632}
633
634/**
635 * amdgpu_block_invalid_wreg - dummy reg write function
636 *
637 * @adev: amdgpu device pointer
638 * @block: offset of instance
639 * @reg: offset of register
640 * @v: value to write to the register
641 *
642 * Dummy register read function. Used for register blocks
643 * that certain asics don't have (all asics).
644 */
645static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
646 uint32_t block,
647 uint32_t reg, uint32_t v)
648{
649 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
650 reg, block, v);
651 BUG();
652}
653
654/**
655 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
656 *
657 * @adev: amdgpu device pointer
658 *
659 * Allocates a scratch page of VRAM for use by various things in the
660 * driver.
661 */
662static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
663{
664 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
665 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
666 &adev->vram_scratch.robj,
667 &adev->vram_scratch.gpu_addr,
668 (void **)&adev->vram_scratch.ptr);
669}
670
671/**
672 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
673 *
674 * @adev: amdgpu device pointer
675 *
676 * Frees the VRAM scratch page.
677 */
678static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
679{
680 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
681}
682
683/**
684 * amdgpu_device_program_register_sequence - program an array of registers.
685 *
686 * @adev: amdgpu_device pointer
687 * @registers: pointer to the register array
688 * @array_size: size of the register array
689 *
690 * Programs an array or registers with and and or masks.
691 * This is a helper for setting golden registers.
692 */
693void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
694 const u32 *registers,
695 const u32 array_size)
696{
697 u32 tmp, reg, and_mask, or_mask;
698 int i;
699
700 if (array_size % 3)
701 return;
702
703 for (i = 0; i < array_size; i +=3) {
704 reg = registers[i + 0];
705 and_mask = registers[i + 1];
706 or_mask = registers[i + 2];
707
708 if (and_mask == 0xffffffff) {
709 tmp = or_mask;
710 } else {
711 tmp = RREG32(reg);
712 tmp &= ~and_mask;
713 if (adev->family >= AMDGPU_FAMILY_AI)
714 tmp |= (or_mask & and_mask);
715 else
716 tmp |= or_mask;
717 }
718 WREG32(reg, tmp);
719 }
720}
721
722/**
723 * amdgpu_device_pci_config_reset - reset the GPU
724 *
725 * @adev: amdgpu_device pointer
726 *
727 * Resets the GPU using the pci config reset sequence.
728 * Only applicable to asics prior to vega10.
729 */
730void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
731{
732 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
733}
734
735/*
736 * GPU doorbell aperture helpers function.
737 */
738/**
739 * amdgpu_device_doorbell_init - Init doorbell driver information.
740 *
741 * @adev: amdgpu_device pointer
742 *
743 * Init doorbell driver information (CIK)
744 * Returns 0 on success, error on failure.
745 */
746static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
747{
748
749 /* No doorbell on SI hardware generation */
750 if (adev->asic_type < CHIP_BONAIRE) {
751 adev->doorbell.base = 0;
752 adev->doorbell.size = 0;
753 adev->doorbell.num_doorbells = 0;
754 adev->doorbell.ptr = NULL;
755 return 0;
756 }
757
758 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
759 return -EINVAL;
760
761 amdgpu_asic_init_doorbell_index(adev);
762
763 /* doorbell bar mapping */
764 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
765 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
766
767 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
768 adev->doorbell_index.max_assignment+1);
769 if (adev->doorbell.num_doorbells == 0)
770 return -EINVAL;
771
772 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
773 * paging queue doorbell use the second page. The
774 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
775 * doorbells are in the first page. So with paging queue enabled,
776 * the max num_doorbells should + 1 page (0x400 in dword)
777 */
778 if (adev->asic_type >= CHIP_VEGA10)
779 adev->doorbell.num_doorbells += 0x400;
780
781 adev->doorbell.ptr = ioremap(adev->doorbell.base,
782 adev->doorbell.num_doorbells *
783 sizeof(u32));
784 if (adev->doorbell.ptr == NULL)
785 return -ENOMEM;
786
787 return 0;
788}
789
790/**
791 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
792 *
793 * @adev: amdgpu_device pointer
794 *
795 * Tear down doorbell driver information (CIK)
796 */
797static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
798{
799 iounmap(adev->doorbell.ptr);
800 adev->doorbell.ptr = NULL;
801}
802
803
804
805/*
806 * amdgpu_device_wb_*()
807 * Writeback is the method by which the GPU updates special pages in memory
808 * with the status of certain GPU events (fences, ring pointers,etc.).
809 */
810
811/**
812 * amdgpu_device_wb_fini - Disable Writeback and free memory
813 *
814 * @adev: amdgpu_device pointer
815 *
816 * Disables Writeback and frees the Writeback memory (all asics).
817 * Used at driver shutdown.
818 */
819static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
820{
821 if (adev->wb.wb_obj) {
822 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
823 &adev->wb.gpu_addr,
824 (void **)&adev->wb.wb);
825 adev->wb.wb_obj = NULL;
826 }
827}
828
829/**
830 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
831 *
832 * @adev: amdgpu_device pointer
833 *
834 * Initializes writeback and allocates writeback memory (all asics).
835 * Used at driver startup.
836 * Returns 0 on success or an -error on failure.
837 */
838static int amdgpu_device_wb_init(struct amdgpu_device *adev)
839{
840 int r;
841
842 if (adev->wb.wb_obj == NULL) {
843 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
844 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
845 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
846 &adev->wb.wb_obj, &adev->wb.gpu_addr,
847 (void **)&adev->wb.wb);
848 if (r) {
849 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
850 return r;
851 }
852
853 adev->wb.num_wb = AMDGPU_MAX_WB;
854 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
855
856 /* clear wb memory */
857 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
858 }
859
860 return 0;
861}
862
863/**
864 * amdgpu_device_wb_get - Allocate a wb entry
865 *
866 * @adev: amdgpu_device pointer
867 * @wb: wb index
868 *
869 * Allocate a wb slot for use by the driver (all asics).
870 * Returns 0 on success or -EINVAL on failure.
871 */
872int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
873{
874 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
875
876 if (offset < adev->wb.num_wb) {
877 __set_bit(offset, adev->wb.used);
878 *wb = offset << 3; /* convert to dw offset */
879 return 0;
880 } else {
881 return -EINVAL;
882 }
883}
884
885/**
886 * amdgpu_device_wb_free - Free a wb entry
887 *
888 * @adev: amdgpu_device pointer
889 * @wb: wb index
890 *
891 * Free a wb slot allocated for use by the driver (all asics)
892 */
893void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
894{
895 wb >>= 3;
896 if (wb < adev->wb.num_wb)
897 __clear_bit(wb, adev->wb.used);
898}
899
900/**
901 * amdgpu_device_resize_fb_bar - try to resize FB BAR
902 *
903 * @adev: amdgpu_device pointer
904 *
905 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
906 * to fail, but if any of the BARs is not accessible after the size we abort
907 * driver loading by returning -ENODEV.
908 */
909int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
910{
911 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
912 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
913 struct pci_bus *root;
914 struct resource *res;
915 unsigned i;
916 u16 cmd;
917 int r;
918
919 /* Bypass for VF */
920 if (amdgpu_sriov_vf(adev))
921 return 0;
922
923 /* skip if the bios has already enabled large BAR */
924 if (adev->gmc.real_vram_size &&
925 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
926 return 0;
927
928 /* Check if the root BUS has 64bit memory resources */
929 root = adev->pdev->bus;
930 while (root->parent)
931 root = root->parent;
932
933 pci_bus_for_each_resource(root, res, i) {
934 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
935 res->start > 0x100000000ull)
936 break;
937 }
938
939 /* Trying to resize is pointless without a root hub window above 4GB */
940 if (!res)
941 return 0;
942
943 /* Disable memory decoding while we change the BAR addresses and size */
944 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
945 pci_write_config_word(adev->pdev, PCI_COMMAND,
946 cmd & ~PCI_COMMAND_MEMORY);
947
948 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
949 amdgpu_device_doorbell_fini(adev);
950 if (adev->asic_type >= CHIP_BONAIRE)
951 pci_release_resource(adev->pdev, 2);
952
953 pci_release_resource(adev->pdev, 0);
954
955 r = pci_resize_resource(adev->pdev, 0, rbar_size);
956 if (r == -ENOSPC)
957 DRM_INFO("Not enough PCI address space for a large BAR.");
958 else if (r && r != -ENOTSUPP)
959 DRM_ERROR("Problem resizing BAR0 (%d).", r);
960
961 pci_assign_unassigned_bus_resources(adev->pdev->bus);
962
963 /* When the doorbell or fb BAR isn't available we have no chance of
964 * using the device.
965 */
966 r = amdgpu_device_doorbell_init(adev);
967 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
968 return -ENODEV;
969
970 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
971
972 return 0;
973}
974
975/*
976 * GPU helpers function.
977 */
978/**
979 * amdgpu_device_need_post - check if the hw need post or not
980 *
981 * @adev: amdgpu_device pointer
982 *
983 * Check if the asic has been initialized (all asics) at driver startup
984 * or post is needed if hw reset is performed.
985 * Returns true if need or false if not.
986 */
987bool amdgpu_device_need_post(struct amdgpu_device *adev)
988{
989 uint32_t reg;
990
991 if (amdgpu_sriov_vf(adev))
992 return false;
993
994 if (amdgpu_passthrough(adev)) {
995 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
996 * some old smc fw still need driver do vPost otherwise gpu hang, while
997 * those smc fw version above 22.15 doesn't have this flaw, so we force
998 * vpost executed for smc version below 22.15
999 */
1000 if (adev->asic_type == CHIP_FIJI) {
1001 int err;
1002 uint32_t fw_ver;
1003 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1004 /* force vPost if error occured */
1005 if (err)
1006 return true;
1007
1008 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1009 if (fw_ver < 0x00160e00)
1010 return true;
1011 }
1012 }
1013
1014 if (adev->has_hw_reset) {
1015 adev->has_hw_reset = false;
1016 return true;
1017 }
1018
1019 /* bios scratch used on CIK+ */
1020 if (adev->asic_type >= CHIP_BONAIRE)
1021 return amdgpu_atombios_scratch_need_asic_init(adev);
1022
1023 /* check MEM_SIZE for older asics */
1024 reg = amdgpu_asic_get_config_memsize(adev);
1025
1026 if ((reg != 0) && (reg != 0xffffffff))
1027 return false;
1028
1029 return true;
1030}
1031
1032/* if we get transitioned to only one device, take VGA back */
1033/**
1034 * amdgpu_device_vga_set_decode - enable/disable vga decode
1035 *
1036 * @cookie: amdgpu_device pointer
1037 * @state: enable/disable vga decode
1038 *
1039 * Enable/disable vga decode (all asics).
1040 * Returns VGA resource flags.
1041 */
1042static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
1043{
1044 struct amdgpu_device *adev = cookie;
1045 amdgpu_asic_set_vga_state(adev, state);
1046 if (state)
1047 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1048 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1049 else
1050 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1051}
1052
1053/**
1054 * amdgpu_device_check_block_size - validate the vm block size
1055 *
1056 * @adev: amdgpu_device pointer
1057 *
1058 * Validates the vm block size specified via module parameter.
1059 * The vm block size defines number of bits in page table versus page directory,
1060 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1061 * page table and the remaining bits are in the page directory.
1062 */
1063static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1064{
1065 /* defines number of bits in page table versus page directory,
1066 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1067 * page table and the remaining bits are in the page directory */
1068 if (amdgpu_vm_block_size == -1)
1069 return;
1070
1071 if (amdgpu_vm_block_size < 9) {
1072 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1073 amdgpu_vm_block_size);
1074 amdgpu_vm_block_size = -1;
1075 }
1076}
1077
1078/**
1079 * amdgpu_device_check_vm_size - validate the vm size
1080 *
1081 * @adev: amdgpu_device pointer
1082 *
1083 * Validates the vm size in GB specified via module parameter.
1084 * The VM size is the size of the GPU virtual memory space in GB.
1085 */
1086static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1087{
1088 /* no need to check the default value */
1089 if (amdgpu_vm_size == -1)
1090 return;
1091
1092 if (amdgpu_vm_size < 1) {
1093 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1094 amdgpu_vm_size);
1095 amdgpu_vm_size = -1;
1096 }
1097}
1098
1099static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1100{
1101 struct sysinfo si;
1102 bool is_os_64 = (sizeof(void *) == 8);
1103 uint64_t total_memory;
1104 uint64_t dram_size_seven_GB = 0x1B8000000;
1105 uint64_t dram_size_three_GB = 0xB8000000;
1106
1107 if (amdgpu_smu_memory_pool_size == 0)
1108 return;
1109
1110 if (!is_os_64) {
1111 DRM_WARN("Not 64-bit OS, feature not supported\n");
1112 goto def_value;
1113 }
1114 si_meminfo(&si);
1115 total_memory = (uint64_t)si.totalram * si.mem_unit;
1116
1117 if ((amdgpu_smu_memory_pool_size == 1) ||
1118 (amdgpu_smu_memory_pool_size == 2)) {
1119 if (total_memory < dram_size_three_GB)
1120 goto def_value1;
1121 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1122 (amdgpu_smu_memory_pool_size == 8)) {
1123 if (total_memory < dram_size_seven_GB)
1124 goto def_value1;
1125 } else {
1126 DRM_WARN("Smu memory pool size not supported\n");
1127 goto def_value;
1128 }
1129 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1130
1131 return;
1132
1133def_value1:
1134 DRM_WARN("No enough system memory\n");
1135def_value:
1136 adev->pm.smu_prv_buffer_size = 0;
1137}
1138
1139/**
1140 * amdgpu_device_check_arguments - validate module params
1141 *
1142 * @adev: amdgpu_device pointer
1143 *
1144 * Validates certain module parameters and updates
1145 * the associated values used by the driver (all asics).
1146 */
1147static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1148{
1149 if (amdgpu_sched_jobs < 4) {
1150 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1151 amdgpu_sched_jobs);
1152 amdgpu_sched_jobs = 4;
1153 } else if (!is_power_of_2(amdgpu_sched_jobs)){
1154 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1155 amdgpu_sched_jobs);
1156 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1157 }
1158
1159 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1160 /* gart size must be greater or equal to 32M */
1161 dev_warn(adev->dev, "gart size (%d) too small\n",
1162 amdgpu_gart_size);
1163 amdgpu_gart_size = -1;
1164 }
1165
1166 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1167 /* gtt size must be greater or equal to 32M */
1168 dev_warn(adev->dev, "gtt size (%d) too small\n",
1169 amdgpu_gtt_size);
1170 amdgpu_gtt_size = -1;
1171 }
1172
1173 /* valid range is between 4 and 9 inclusive */
1174 if (amdgpu_vm_fragment_size != -1 &&
1175 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1176 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1177 amdgpu_vm_fragment_size = -1;
1178 }
1179
1180 if (amdgpu_sched_hw_submission < 2) {
1181 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1182 amdgpu_sched_hw_submission);
1183 amdgpu_sched_hw_submission = 2;
1184 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1185 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1186 amdgpu_sched_hw_submission);
1187 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1188 }
1189
1190 amdgpu_device_check_smu_prv_buffer_size(adev);
1191
1192 amdgpu_device_check_vm_size(adev);
1193
1194 amdgpu_device_check_block_size(adev);
1195
1196 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1197
1198 amdgpu_gmc_tmz_set(adev);
1199
1200 return 0;
1201}
1202
1203/**
1204 * amdgpu_switcheroo_set_state - set switcheroo state
1205 *
1206 * @pdev: pci dev pointer
1207 * @state: vga_switcheroo state
1208 *
1209 * Callback for the switcheroo driver. Suspends or resumes the
1210 * the asics before or after it is powered up using ACPI methods.
1211 */
1212static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1213{
1214 struct drm_device *dev = pci_get_drvdata(pdev);
1215 int r;
1216
1217 if (amdgpu_device_supports_boco(dev) && state == VGA_SWITCHEROO_OFF)
1218 return;
1219
1220 if (state == VGA_SWITCHEROO_ON) {
1221 pr_info("switched on\n");
1222 /* don't suspend or resume card normally */
1223 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1224
1225 pci_set_power_state(dev->pdev, PCI_D0);
1226 pci_restore_state(dev->pdev);
1227 r = pci_enable_device(dev->pdev);
1228 if (r)
1229 DRM_WARN("pci_enable_device failed (%d)\n", r);
1230 amdgpu_device_resume(dev, true);
1231
1232 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1233 drm_kms_helper_poll_enable(dev);
1234 } else {
1235 pr_info("switched off\n");
1236 drm_kms_helper_poll_disable(dev);
1237 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1238 amdgpu_device_suspend(dev, true);
1239 pci_save_state(dev->pdev);
1240 /* Shut down the device */
1241 pci_disable_device(dev->pdev);
1242 pci_set_power_state(dev->pdev, PCI_D3cold);
1243 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1244 }
1245}
1246
1247/**
1248 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1249 *
1250 * @pdev: pci dev pointer
1251 *
1252 * Callback for the switcheroo driver. Check of the switcheroo
1253 * state can be changed.
1254 * Returns true if the state can be changed, false if not.
1255 */
1256static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1257{
1258 struct drm_device *dev = pci_get_drvdata(pdev);
1259
1260 /*
1261 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1262 * locking inversion with the driver load path. And the access here is
1263 * completely racy anyway. So don't bother with locking for now.
1264 */
1265 return atomic_read(&dev->open_count) == 0;
1266}
1267
1268static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1269 .set_gpu_state = amdgpu_switcheroo_set_state,
1270 .reprobe = NULL,
1271 .can_switch = amdgpu_switcheroo_can_switch,
1272};
1273
1274/**
1275 * amdgpu_device_ip_set_clockgating_state - set the CG state
1276 *
1277 * @dev: amdgpu_device pointer
1278 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1279 * @state: clockgating state (gate or ungate)
1280 *
1281 * Sets the requested clockgating state for all instances of
1282 * the hardware IP specified.
1283 * Returns the error code from the last instance.
1284 */
1285int amdgpu_device_ip_set_clockgating_state(void *dev,
1286 enum amd_ip_block_type block_type,
1287 enum amd_clockgating_state state)
1288{
1289 struct amdgpu_device *adev = dev;
1290 int i, r = 0;
1291
1292 for (i = 0; i < adev->num_ip_blocks; i++) {
1293 if (!adev->ip_blocks[i].status.valid)
1294 continue;
1295 if (adev->ip_blocks[i].version->type != block_type)
1296 continue;
1297 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1298 continue;
1299 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1300 (void *)adev, state);
1301 if (r)
1302 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1303 adev->ip_blocks[i].version->funcs->name, r);
1304 }
1305 return r;
1306}
1307
1308/**
1309 * amdgpu_device_ip_set_powergating_state - set the PG state
1310 *
1311 * @dev: amdgpu_device pointer
1312 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1313 * @state: powergating state (gate or ungate)
1314 *
1315 * Sets the requested powergating state for all instances of
1316 * the hardware IP specified.
1317 * Returns the error code from the last instance.
1318 */
1319int amdgpu_device_ip_set_powergating_state(void *dev,
1320 enum amd_ip_block_type block_type,
1321 enum amd_powergating_state state)
1322{
1323 struct amdgpu_device *adev = dev;
1324 int i, r = 0;
1325
1326 for (i = 0; i < adev->num_ip_blocks; i++) {
1327 if (!adev->ip_blocks[i].status.valid)
1328 continue;
1329 if (adev->ip_blocks[i].version->type != block_type)
1330 continue;
1331 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1332 continue;
1333 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1334 (void *)adev, state);
1335 if (r)
1336 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1337 adev->ip_blocks[i].version->funcs->name, r);
1338 }
1339 return r;
1340}
1341
1342/**
1343 * amdgpu_device_ip_get_clockgating_state - get the CG state
1344 *
1345 * @adev: amdgpu_device pointer
1346 * @flags: clockgating feature flags
1347 *
1348 * Walks the list of IPs on the device and updates the clockgating
1349 * flags for each IP.
1350 * Updates @flags with the feature flags for each hardware IP where
1351 * clockgating is enabled.
1352 */
1353void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1354 u32 *flags)
1355{
1356 int i;
1357
1358 for (i = 0; i < adev->num_ip_blocks; i++) {
1359 if (!adev->ip_blocks[i].status.valid)
1360 continue;
1361 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1362 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1363 }
1364}
1365
1366/**
1367 * amdgpu_device_ip_wait_for_idle - wait for idle
1368 *
1369 * @adev: amdgpu_device pointer
1370 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1371 *
1372 * Waits for the request hardware IP to be idle.
1373 * Returns 0 for success or a negative error code on failure.
1374 */
1375int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1376 enum amd_ip_block_type block_type)
1377{
1378 int i, r;
1379
1380 for (i = 0; i < adev->num_ip_blocks; i++) {
1381 if (!adev->ip_blocks[i].status.valid)
1382 continue;
1383 if (adev->ip_blocks[i].version->type == block_type) {
1384 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1385 if (r)
1386 return r;
1387 break;
1388 }
1389 }
1390 return 0;
1391
1392}
1393
1394/**
1395 * amdgpu_device_ip_is_idle - is the hardware IP idle
1396 *
1397 * @adev: amdgpu_device pointer
1398 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1399 *
1400 * Check if the hardware IP is idle or not.
1401 * Returns true if it the IP is idle, false if not.
1402 */
1403bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1404 enum amd_ip_block_type block_type)
1405{
1406 int i;
1407
1408 for (i = 0; i < adev->num_ip_blocks; i++) {
1409 if (!adev->ip_blocks[i].status.valid)
1410 continue;
1411 if (adev->ip_blocks[i].version->type == block_type)
1412 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1413 }
1414 return true;
1415
1416}
1417
1418/**
1419 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1420 *
1421 * @adev: amdgpu_device pointer
1422 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1423 *
1424 * Returns a pointer to the hardware IP block structure
1425 * if it exists for the asic, otherwise NULL.
1426 */
1427struct amdgpu_ip_block *
1428amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1429 enum amd_ip_block_type type)
1430{
1431 int i;
1432
1433 for (i = 0; i < adev->num_ip_blocks; i++)
1434 if (adev->ip_blocks[i].version->type == type)
1435 return &adev->ip_blocks[i];
1436
1437 return NULL;
1438}
1439
1440/**
1441 * amdgpu_device_ip_block_version_cmp
1442 *
1443 * @adev: amdgpu_device pointer
1444 * @type: enum amd_ip_block_type
1445 * @major: major version
1446 * @minor: minor version
1447 *
1448 * return 0 if equal or greater
1449 * return 1 if smaller or the ip_block doesn't exist
1450 */
1451int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1452 enum amd_ip_block_type type,
1453 u32 major, u32 minor)
1454{
1455 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1456
1457 if (ip_block && ((ip_block->version->major > major) ||
1458 ((ip_block->version->major == major) &&
1459 (ip_block->version->minor >= minor))))
1460 return 0;
1461
1462 return 1;
1463}
1464
1465/**
1466 * amdgpu_device_ip_block_add
1467 *
1468 * @adev: amdgpu_device pointer
1469 * @ip_block_version: pointer to the IP to add
1470 *
1471 * Adds the IP block driver information to the collection of IPs
1472 * on the asic.
1473 */
1474int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1475 const struct amdgpu_ip_block_version *ip_block_version)
1476{
1477 if (!ip_block_version)
1478 return -EINVAL;
1479
1480 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1481 ip_block_version->funcs->name);
1482
1483 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1484
1485 return 0;
1486}
1487
1488/**
1489 * amdgpu_device_enable_virtual_display - enable virtual display feature
1490 *
1491 * @adev: amdgpu_device pointer
1492 *
1493 * Enabled the virtual display feature if the user has enabled it via
1494 * the module parameter virtual_display. This feature provides a virtual
1495 * display hardware on headless boards or in virtualized environments.
1496 * This function parses and validates the configuration string specified by
1497 * the user and configues the virtual display configuration (number of
1498 * virtual connectors, crtcs, etc.) specified.
1499 */
1500static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1501{
1502 adev->enable_virtual_display = false;
1503
1504 if (amdgpu_virtual_display) {
1505 struct drm_device *ddev = adev->ddev;
1506 const char *pci_address_name = pci_name(ddev->pdev);
1507 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1508
1509 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1510 pciaddstr_tmp = pciaddstr;
1511 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1512 pciaddname = strsep(&pciaddname_tmp, ",");
1513 if (!strcmp("all", pciaddname)
1514 || !strcmp(pci_address_name, pciaddname)) {
1515 long num_crtc;
1516 int res = -1;
1517
1518 adev->enable_virtual_display = true;
1519
1520 if (pciaddname_tmp)
1521 res = kstrtol(pciaddname_tmp, 10,
1522 &num_crtc);
1523
1524 if (!res) {
1525 if (num_crtc < 1)
1526 num_crtc = 1;
1527 if (num_crtc > 6)
1528 num_crtc = 6;
1529 adev->mode_info.num_crtc = num_crtc;
1530 } else {
1531 adev->mode_info.num_crtc = 1;
1532 }
1533 break;
1534 }
1535 }
1536
1537 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1538 amdgpu_virtual_display, pci_address_name,
1539 adev->enable_virtual_display, adev->mode_info.num_crtc);
1540
1541 kfree(pciaddstr);
1542 }
1543}
1544
1545/**
1546 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1547 *
1548 * @adev: amdgpu_device pointer
1549 *
1550 * Parses the asic configuration parameters specified in the gpu info
1551 * firmware and makes them availale to the driver for use in configuring
1552 * the asic.
1553 * Returns 0 on success, -EINVAL on failure.
1554 */
1555static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1556{
1557 const char *chip_name;
1558 char fw_name[40];
1559 int err;
1560 const struct gpu_info_firmware_header_v1_0 *hdr;
1561
1562 adev->firmware.gpu_info_fw = NULL;
1563
1564 if (adev->discovery_bin) {
1565 amdgpu_discovery_get_gfx_info(adev);
1566
1567 /*
1568 * FIXME: The bounding box is still needed by Navi12, so
1569 * temporarily read it from gpu_info firmware. Should be droped
1570 * when DAL no longer needs it.
1571 */
1572 if (adev->asic_type != CHIP_NAVI12)
1573 return 0;
1574 }
1575
1576 switch (adev->asic_type) {
1577#ifdef CONFIG_DRM_AMDGPU_SI
1578 case CHIP_VERDE:
1579 case CHIP_TAHITI:
1580 case CHIP_PITCAIRN:
1581 case CHIP_OLAND:
1582 case CHIP_HAINAN:
1583#endif
1584#ifdef CONFIG_DRM_AMDGPU_CIK
1585 case CHIP_BONAIRE:
1586 case CHIP_HAWAII:
1587 case CHIP_KAVERI:
1588 case CHIP_KABINI:
1589 case CHIP_MULLINS:
1590#endif
1591 case CHIP_TOPAZ:
1592 case CHIP_TONGA:
1593 case CHIP_FIJI:
1594 case CHIP_POLARIS10:
1595 case CHIP_POLARIS11:
1596 case CHIP_POLARIS12:
1597 case CHIP_VEGAM:
1598 case CHIP_CARRIZO:
1599 case CHIP_STONEY:
1600 case CHIP_VEGA20:
1601 case CHIP_SIENNA_CICHLID:
1602 case CHIP_NAVY_FLOUNDER:
1603 default:
1604 return 0;
1605 case CHIP_VEGA10:
1606 chip_name = "vega10";
1607 break;
1608 case CHIP_VEGA12:
1609 chip_name = "vega12";
1610 break;
1611 case CHIP_RAVEN:
1612 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1613 chip_name = "raven2";
1614 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1615 chip_name = "picasso";
1616 else
1617 chip_name = "raven";
1618 break;
1619 case CHIP_ARCTURUS:
1620 chip_name = "arcturus";
1621 break;
1622 case CHIP_RENOIR:
1623 chip_name = "renoir";
1624 break;
1625 case CHIP_NAVI10:
1626 chip_name = "navi10";
1627 break;
1628 case CHIP_NAVI14:
1629 chip_name = "navi14";
1630 break;
1631 case CHIP_NAVI12:
1632 chip_name = "navi12";
1633 break;
1634 }
1635
1636 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1637 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1638 if (err) {
1639 dev_err(adev->dev,
1640 "Failed to load gpu_info firmware \"%s\"\n",
1641 fw_name);
1642 goto out;
1643 }
1644 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1645 if (err) {
1646 dev_err(adev->dev,
1647 "Failed to validate gpu_info firmware \"%s\"\n",
1648 fw_name);
1649 goto out;
1650 }
1651
1652 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1653 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1654
1655 switch (hdr->version_major) {
1656 case 1:
1657 {
1658 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1659 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1660 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1661
1662 /*
1663 * Should be droped when DAL no longer needs it.
1664 */
1665 if (adev->asic_type == CHIP_NAVI12)
1666 goto parse_soc_bounding_box;
1667
1668 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1669 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1670 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1671 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1672 adev->gfx.config.max_texture_channel_caches =
1673 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1674 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1675 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1676 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1677 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1678 adev->gfx.config.double_offchip_lds_buf =
1679 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1680 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1681 adev->gfx.cu_info.max_waves_per_simd =
1682 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1683 adev->gfx.cu_info.max_scratch_slots_per_cu =
1684 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1685 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1686 if (hdr->version_minor >= 1) {
1687 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1688 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1689 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1690 adev->gfx.config.num_sc_per_sh =
1691 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1692 adev->gfx.config.num_packer_per_sc =
1693 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1694 }
1695
1696parse_soc_bounding_box:
1697 /*
1698 * soc bounding box info is not integrated in disocovery table,
1699 * we always need to parse it from gpu info firmware if needed.
1700 */
1701 if (hdr->version_minor == 2) {
1702 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1703 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1704 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1705 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1706 }
1707 break;
1708 }
1709 default:
1710 dev_err(adev->dev,
1711 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1712 err = -EINVAL;
1713 goto out;
1714 }
1715out:
1716 return err;
1717}
1718
1719/**
1720 * amdgpu_device_ip_early_init - run early init for hardware IPs
1721 *
1722 * @adev: amdgpu_device pointer
1723 *
1724 * Early initialization pass for hardware IPs. The hardware IPs that make
1725 * up each asic are discovered each IP's early_init callback is run. This
1726 * is the first stage in initializing the asic.
1727 * Returns 0 on success, negative error code on failure.
1728 */
1729static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1730{
1731 int i, r;
1732
1733 amdgpu_device_enable_virtual_display(adev);
1734
1735 if (amdgpu_sriov_vf(adev)) {
1736 r = amdgpu_virt_request_full_gpu(adev, true);
1737 if (r)
1738 return r;
1739 }
1740
1741 switch (adev->asic_type) {
1742#ifdef CONFIG_DRM_AMDGPU_SI
1743 case CHIP_VERDE:
1744 case CHIP_TAHITI:
1745 case CHIP_PITCAIRN:
1746 case CHIP_OLAND:
1747 case CHIP_HAINAN:
1748 adev->family = AMDGPU_FAMILY_SI;
1749 r = si_set_ip_blocks(adev);
1750 if (r)
1751 return r;
1752 break;
1753#endif
1754#ifdef CONFIG_DRM_AMDGPU_CIK
1755 case CHIP_BONAIRE:
1756 case CHIP_HAWAII:
1757 case CHIP_KAVERI:
1758 case CHIP_KABINI:
1759 case CHIP_MULLINS:
1760 if (adev->flags & AMD_IS_APU)
1761 adev->family = AMDGPU_FAMILY_KV;
1762 else
1763 adev->family = AMDGPU_FAMILY_CI;
1764
1765 r = cik_set_ip_blocks(adev);
1766 if (r)
1767 return r;
1768 break;
1769#endif
1770 case CHIP_TOPAZ:
1771 case CHIP_TONGA:
1772 case CHIP_FIJI:
1773 case CHIP_POLARIS10:
1774 case CHIP_POLARIS11:
1775 case CHIP_POLARIS12:
1776 case CHIP_VEGAM:
1777 case CHIP_CARRIZO:
1778 case CHIP_STONEY:
1779 if (adev->flags & AMD_IS_APU)
1780 adev->family = AMDGPU_FAMILY_CZ;
1781 else
1782 adev->family = AMDGPU_FAMILY_VI;
1783
1784 r = vi_set_ip_blocks(adev);
1785 if (r)
1786 return r;
1787 break;
1788 case CHIP_VEGA10:
1789 case CHIP_VEGA12:
1790 case CHIP_VEGA20:
1791 case CHIP_RAVEN:
1792 case CHIP_ARCTURUS:
1793 case CHIP_RENOIR:
1794 if (adev->flags & AMD_IS_APU)
1795 adev->family = AMDGPU_FAMILY_RV;
1796 else
1797 adev->family = AMDGPU_FAMILY_AI;
1798
1799 r = soc15_set_ip_blocks(adev);
1800 if (r)
1801 return r;
1802 break;
1803 case CHIP_NAVI10:
1804 case CHIP_NAVI14:
1805 case CHIP_NAVI12:
1806 case CHIP_SIENNA_CICHLID:
1807 case CHIP_NAVY_FLOUNDER:
1808 adev->family = AMDGPU_FAMILY_NV;
1809
1810 r = nv_set_ip_blocks(adev);
1811 if (r)
1812 return r;
1813 break;
1814 default:
1815 /* FIXME: not supported yet */
1816 return -EINVAL;
1817 }
1818
1819 amdgpu_amdkfd_device_probe(adev);
1820
1821 adev->pm.pp_feature = amdgpu_pp_feature_mask;
1822 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
1823 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1824
1825 for (i = 0; i < adev->num_ip_blocks; i++) {
1826 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1827 DRM_ERROR("disabled ip block: %d <%s>\n",
1828 i, adev->ip_blocks[i].version->funcs->name);
1829 adev->ip_blocks[i].status.valid = false;
1830 } else {
1831 if (adev->ip_blocks[i].version->funcs->early_init) {
1832 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1833 if (r == -ENOENT) {
1834 adev->ip_blocks[i].status.valid = false;
1835 } else if (r) {
1836 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1837 adev->ip_blocks[i].version->funcs->name, r);
1838 return r;
1839 } else {
1840 adev->ip_blocks[i].status.valid = true;
1841 }
1842 } else {
1843 adev->ip_blocks[i].status.valid = true;
1844 }
1845 }
1846 /* get the vbios after the asic_funcs are set up */
1847 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
1848 r = amdgpu_device_parse_gpu_info_fw(adev);
1849 if (r)
1850 return r;
1851
1852 /* Read BIOS */
1853 if (!amdgpu_get_bios(adev))
1854 return -EINVAL;
1855
1856 r = amdgpu_atombios_init(adev);
1857 if (r) {
1858 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1859 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
1860 return r;
1861 }
1862 }
1863 }
1864
1865 adev->cg_flags &= amdgpu_cg_mask;
1866 adev->pg_flags &= amdgpu_pg_mask;
1867
1868 return 0;
1869}
1870
1871static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
1872{
1873 int i, r;
1874
1875 for (i = 0; i < adev->num_ip_blocks; i++) {
1876 if (!adev->ip_blocks[i].status.sw)
1877 continue;
1878 if (adev->ip_blocks[i].status.hw)
1879 continue;
1880 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1881 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
1882 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
1883 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1884 if (r) {
1885 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1886 adev->ip_blocks[i].version->funcs->name, r);
1887 return r;
1888 }
1889 adev->ip_blocks[i].status.hw = true;
1890 }
1891 }
1892
1893 return 0;
1894}
1895
1896static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
1897{
1898 int i, r;
1899
1900 for (i = 0; i < adev->num_ip_blocks; i++) {
1901 if (!adev->ip_blocks[i].status.sw)
1902 continue;
1903 if (adev->ip_blocks[i].status.hw)
1904 continue;
1905 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1906 if (r) {
1907 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1908 adev->ip_blocks[i].version->funcs->name, r);
1909 return r;
1910 }
1911 adev->ip_blocks[i].status.hw = true;
1912 }
1913
1914 return 0;
1915}
1916
1917static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
1918{
1919 int r = 0;
1920 int i;
1921 uint32_t smu_version;
1922
1923 if (adev->asic_type >= CHIP_VEGA10) {
1924 for (i = 0; i < adev->num_ip_blocks; i++) {
1925 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
1926 continue;
1927
1928 /* no need to do the fw loading again if already done*/
1929 if (adev->ip_blocks[i].status.hw == true)
1930 break;
1931
1932 if (adev->in_gpu_reset || adev->in_suspend) {
1933 r = adev->ip_blocks[i].version->funcs->resume(adev);
1934 if (r) {
1935 DRM_ERROR("resume of IP block <%s> failed %d\n",
1936 adev->ip_blocks[i].version->funcs->name, r);
1937 return r;
1938 }
1939 } else {
1940 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1941 if (r) {
1942 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1943 adev->ip_blocks[i].version->funcs->name, r);
1944 return r;
1945 }
1946 }
1947
1948 adev->ip_blocks[i].status.hw = true;
1949 break;
1950 }
1951 }
1952
1953 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
1954 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
1955
1956 return r;
1957}
1958
1959/**
1960 * amdgpu_device_ip_init - run init for hardware IPs
1961 *
1962 * @adev: amdgpu_device pointer
1963 *
1964 * Main initialization pass for hardware IPs. The list of all the hardware
1965 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1966 * are run. sw_init initializes the software state associated with each IP
1967 * and hw_init initializes the hardware associated with each IP.
1968 * Returns 0 on success, negative error code on failure.
1969 */
1970static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1971{
1972 int i, r;
1973
1974 r = amdgpu_ras_init(adev);
1975 if (r)
1976 return r;
1977
1978 for (i = 0; i < adev->num_ip_blocks; i++) {
1979 if (!adev->ip_blocks[i].status.valid)
1980 continue;
1981 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1982 if (r) {
1983 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1984 adev->ip_blocks[i].version->funcs->name, r);
1985 goto init_failed;
1986 }
1987 adev->ip_blocks[i].status.sw = true;
1988
1989 /* need to do gmc hw init early so we can allocate gpu mem */
1990 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1991 r = amdgpu_device_vram_scratch_init(adev);
1992 if (r) {
1993 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1994 goto init_failed;
1995 }
1996 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1997 if (r) {
1998 DRM_ERROR("hw_init %d failed %d\n", i, r);
1999 goto init_failed;
2000 }
2001 r = amdgpu_device_wb_init(adev);
2002 if (r) {
2003 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2004 goto init_failed;
2005 }
2006 adev->ip_blocks[i].status.hw = true;
2007
2008 /* right after GMC hw init, we create CSA */
2009 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2010 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2011 AMDGPU_GEM_DOMAIN_VRAM,
2012 AMDGPU_CSA_SIZE);
2013 if (r) {
2014 DRM_ERROR("allocate CSA failed %d\n", r);
2015 goto init_failed;
2016 }
2017 }
2018 }
2019 }
2020
2021 if (amdgpu_sriov_vf(adev))
2022 amdgpu_virt_init_data_exchange(adev);
2023
2024 r = amdgpu_ib_pool_init(adev);
2025 if (r) {
2026 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2027 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2028 goto init_failed;
2029 }
2030
2031 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2032 if (r)
2033 goto init_failed;
2034
2035 r = amdgpu_device_ip_hw_init_phase1(adev);
2036 if (r)
2037 goto init_failed;
2038
2039 r = amdgpu_device_fw_loading(adev);
2040 if (r)
2041 goto init_failed;
2042
2043 r = amdgpu_device_ip_hw_init_phase2(adev);
2044 if (r)
2045 goto init_failed;
2046
2047 /*
2048 * retired pages will be loaded from eeprom and reserved here,
2049 * it should be called after amdgpu_device_ip_hw_init_phase2 since
2050 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2051 * for I2C communication which only true at this point.
2052 * recovery_init may fail, but it can free all resources allocated by
2053 * itself and its failure should not stop amdgpu init process.
2054 *
2055 * Note: theoretically, this should be called before all vram allocations
2056 * to protect retired page from abusing
2057 */
2058 amdgpu_ras_recovery_init(adev);
2059
2060 if (adev->gmc.xgmi.num_physical_nodes > 1)
2061 amdgpu_xgmi_add_device(adev);
2062 amdgpu_amdkfd_device_init(adev);
2063
2064 amdgpu_fru_get_product_info(adev);
2065
2066init_failed:
2067 if (amdgpu_sriov_vf(adev))
2068 amdgpu_virt_release_full_gpu(adev, true);
2069
2070 return r;
2071}
2072
2073/**
2074 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2075 *
2076 * @adev: amdgpu_device pointer
2077 *
2078 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
2079 * this function before a GPU reset. If the value is retained after a
2080 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2081 */
2082static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2083{
2084 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2085}
2086
2087/**
2088 * amdgpu_device_check_vram_lost - check if vram is valid
2089 *
2090 * @adev: amdgpu_device pointer
2091 *
2092 * Checks the reset magic value written to the gart pointer in VRAM.
2093 * The driver calls this after a GPU reset to see if the contents of
2094 * VRAM is lost or now.
2095 * returns true if vram is lost, false if not.
2096 */
2097static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2098{
2099 if (memcmp(adev->gart.ptr, adev->reset_magic,
2100 AMDGPU_RESET_MAGIC_NUM))
2101 return true;
2102
2103 if (!adev->in_gpu_reset)
2104 return false;
2105
2106 /*
2107 * For all ASICs with baco/mode1 reset, the VRAM is
2108 * always assumed to be lost.
2109 */
2110 switch (amdgpu_asic_reset_method(adev)) {
2111 case AMD_RESET_METHOD_BACO:
2112 case AMD_RESET_METHOD_MODE1:
2113 return true;
2114 default:
2115 return false;
2116 }
2117}
2118
2119/**
2120 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2121 *
2122 * @adev: amdgpu_device pointer
2123 * @state: clockgating state (gate or ungate)
2124 *
2125 * The list of all the hardware IPs that make up the asic is walked and the
2126 * set_clockgating_state callbacks are run.
2127 * Late initialization pass enabling clockgating for hardware IPs.
2128 * Fini or suspend, pass disabling clockgating for hardware IPs.
2129 * Returns 0 on success, negative error code on failure.
2130 */
2131
2132static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2133 enum amd_clockgating_state state)
2134{
2135 int i, j, r;
2136
2137 if (amdgpu_emu_mode == 1)
2138 return 0;
2139
2140 for (j = 0; j < adev->num_ip_blocks; j++) {
2141 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2142 if (!adev->ip_blocks[i].status.late_initialized)
2143 continue;
2144 /* skip CG for VCE/UVD, it's handled specially */
2145 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2146 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2147 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2148 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2149 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2150 /* enable clockgating to save power */
2151 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2152 state);
2153 if (r) {
2154 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2155 adev->ip_blocks[i].version->funcs->name, r);
2156 return r;
2157 }
2158 }
2159 }
2160
2161 return 0;
2162}
2163
2164static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
2165{
2166 int i, j, r;
2167
2168 if (amdgpu_emu_mode == 1)
2169 return 0;
2170
2171 for (j = 0; j < adev->num_ip_blocks; j++) {
2172 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2173 if (!adev->ip_blocks[i].status.late_initialized)
2174 continue;
2175 /* skip CG for VCE/UVD, it's handled specially */
2176 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2177 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2178 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2179 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2180 adev->ip_blocks[i].version->funcs->set_powergating_state) {
2181 /* enable powergating to save power */
2182 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2183 state);
2184 if (r) {
2185 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2186 adev->ip_blocks[i].version->funcs->name, r);
2187 return r;
2188 }
2189 }
2190 }
2191 return 0;
2192}
2193
2194static int amdgpu_device_enable_mgpu_fan_boost(void)
2195{
2196 struct amdgpu_gpu_instance *gpu_ins;
2197 struct amdgpu_device *adev;
2198 int i, ret = 0;
2199
2200 mutex_lock(&mgpu_info.mutex);
2201
2202 /*
2203 * MGPU fan boost feature should be enabled
2204 * only when there are two or more dGPUs in
2205 * the system
2206 */
2207 if (mgpu_info.num_dgpu < 2)
2208 goto out;
2209
2210 for (i = 0; i < mgpu_info.num_dgpu; i++) {
2211 gpu_ins = &(mgpu_info.gpu_ins[i]);
2212 adev = gpu_ins->adev;
2213 if (!(adev->flags & AMD_IS_APU) &&
2214 !gpu_ins->mgpu_fan_enabled &&
2215 adev->powerplay.pp_funcs &&
2216 adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
2217 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2218 if (ret)
2219 break;
2220
2221 gpu_ins->mgpu_fan_enabled = 1;
2222 }
2223 }
2224
2225out:
2226 mutex_unlock(&mgpu_info.mutex);
2227
2228 return ret;
2229}
2230
2231/**
2232 * amdgpu_device_ip_late_init - run late init for hardware IPs
2233 *
2234 * @adev: amdgpu_device pointer
2235 *
2236 * Late initialization pass for hardware IPs. The list of all the hardware
2237 * IPs that make up the asic is walked and the late_init callbacks are run.
2238 * late_init covers any special initialization that an IP requires
2239 * after all of the have been initialized or something that needs to happen
2240 * late in the init process.
2241 * Returns 0 on success, negative error code on failure.
2242 */
2243static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2244{
2245 struct amdgpu_gpu_instance *gpu_instance;
2246 int i = 0, r;
2247
2248 for (i = 0; i < adev->num_ip_blocks; i++) {
2249 if (!adev->ip_blocks[i].status.hw)
2250 continue;
2251 if (adev->ip_blocks[i].version->funcs->late_init) {
2252 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2253 if (r) {
2254 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2255 adev->ip_blocks[i].version->funcs->name, r);
2256 return r;
2257 }
2258 }
2259 adev->ip_blocks[i].status.late_initialized = true;
2260 }
2261
2262 amdgpu_ras_set_error_query_ready(adev, true);
2263
2264 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2265 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2266
2267 amdgpu_device_fill_reset_magic(adev);
2268
2269 r = amdgpu_device_enable_mgpu_fan_boost();
2270 if (r)
2271 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2272
2273
2274 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2275 mutex_lock(&mgpu_info.mutex);
2276
2277 /*
2278 * Reset device p-state to low as this was booted with high.
2279 *
2280 * This should be performed only after all devices from the same
2281 * hive get initialized.
2282 *
2283 * However, it's unknown how many device in the hive in advance.
2284 * As this is counted one by one during devices initializations.
2285 *
2286 * So, we wait for all XGMI interlinked devices initialized.
2287 * This may bring some delays as those devices may come from
2288 * different hives. But that should be OK.
2289 */
2290 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2291 for (i = 0; i < mgpu_info.num_gpu; i++) {
2292 gpu_instance = &(mgpu_info.gpu_ins[i]);
2293 if (gpu_instance->adev->flags & AMD_IS_APU)
2294 continue;
2295
2296 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2297 AMDGPU_XGMI_PSTATE_MIN);
2298 if (r) {
2299 DRM_ERROR("pstate setting failed (%d).\n", r);
2300 break;
2301 }
2302 }
2303 }
2304
2305 mutex_unlock(&mgpu_info.mutex);
2306 }
2307
2308 return 0;
2309}
2310
2311/**
2312 * amdgpu_device_ip_fini - run fini for hardware IPs
2313 *
2314 * @adev: amdgpu_device pointer
2315 *
2316 * Main teardown pass for hardware IPs. The list of all the hardware
2317 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2318 * are run. hw_fini tears down the hardware associated with each IP
2319 * and sw_fini tears down any software state associated with each IP.
2320 * Returns 0 on success, negative error code on failure.
2321 */
2322static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2323{
2324 int i, r;
2325
2326 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2327 amdgpu_virt_release_ras_err_handler_data(adev);
2328
2329 amdgpu_ras_pre_fini(adev);
2330
2331 if (adev->gmc.xgmi.num_physical_nodes > 1)
2332 amdgpu_xgmi_remove_device(adev);
2333
2334 amdgpu_amdkfd_device_fini(adev);
2335
2336 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2337 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2338
2339 /* need to disable SMC first */
2340 for (i = 0; i < adev->num_ip_blocks; i++) {
2341 if (!adev->ip_blocks[i].status.hw)
2342 continue;
2343 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2344 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2345 /* XXX handle errors */
2346 if (r) {
2347 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2348 adev->ip_blocks[i].version->funcs->name, r);
2349 }
2350 adev->ip_blocks[i].status.hw = false;
2351 break;
2352 }
2353 }
2354
2355 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2356 if (!adev->ip_blocks[i].status.hw)
2357 continue;
2358
2359 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2360 /* XXX handle errors */
2361 if (r) {
2362 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2363 adev->ip_blocks[i].version->funcs->name, r);
2364 }
2365
2366 adev->ip_blocks[i].status.hw = false;
2367 }
2368
2369
2370 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2371 if (!adev->ip_blocks[i].status.sw)
2372 continue;
2373
2374 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2375 amdgpu_ucode_free_bo(adev);
2376 amdgpu_free_static_csa(&adev->virt.csa_obj);
2377 amdgpu_device_wb_fini(adev);
2378 amdgpu_device_vram_scratch_fini(adev);
2379 amdgpu_ib_pool_fini(adev);
2380 }
2381
2382 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2383 /* XXX handle errors */
2384 if (r) {
2385 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2386 adev->ip_blocks[i].version->funcs->name, r);
2387 }
2388 adev->ip_blocks[i].status.sw = false;
2389 adev->ip_blocks[i].status.valid = false;
2390 }
2391
2392 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2393 if (!adev->ip_blocks[i].status.late_initialized)
2394 continue;
2395 if (adev->ip_blocks[i].version->funcs->late_fini)
2396 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2397 adev->ip_blocks[i].status.late_initialized = false;
2398 }
2399
2400 amdgpu_ras_fini(adev);
2401
2402 if (amdgpu_sriov_vf(adev))
2403 if (amdgpu_virt_release_full_gpu(adev, false))
2404 DRM_ERROR("failed to release exclusive mode on fini\n");
2405
2406 return 0;
2407}
2408
2409/**
2410 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2411 *
2412 * @work: work_struct.
2413 */
2414static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2415{
2416 struct amdgpu_device *adev =
2417 container_of(work, struct amdgpu_device, delayed_init_work.work);
2418 int r;
2419
2420 r = amdgpu_ib_ring_tests(adev);
2421 if (r)
2422 DRM_ERROR("ib ring test failed (%d).\n", r);
2423}
2424
2425static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2426{
2427 struct amdgpu_device *adev =
2428 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2429
2430 mutex_lock(&adev->gfx.gfx_off_mutex);
2431 if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2432 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2433 adev->gfx.gfx_off_state = true;
2434 }
2435 mutex_unlock(&adev->gfx.gfx_off_mutex);
2436}
2437
2438/**
2439 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2440 *
2441 * @adev: amdgpu_device pointer
2442 *
2443 * Main suspend function for hardware IPs. The list of all the hardware
2444 * IPs that make up the asic is walked, clockgating is disabled and the
2445 * suspend callbacks are run. suspend puts the hardware and software state
2446 * in each IP into a state suitable for suspend.
2447 * Returns 0 on success, negative error code on failure.
2448 */
2449static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2450{
2451 int i, r;
2452
2453 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2454 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2455
2456 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2457 if (!adev->ip_blocks[i].status.valid)
2458 continue;
2459
2460 /* displays are handled separately */
2461 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2462 continue;
2463
2464 /* XXX handle errors */
2465 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2466 /* XXX handle errors */
2467 if (r) {
2468 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2469 adev->ip_blocks[i].version->funcs->name, r);
2470 return r;
2471 }
2472
2473 adev->ip_blocks[i].status.hw = false;
2474 }
2475
2476 return 0;
2477}
2478
2479/**
2480 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2481 *
2482 * @adev: amdgpu_device pointer
2483 *
2484 * Main suspend function for hardware IPs. The list of all the hardware
2485 * IPs that make up the asic is walked, clockgating is disabled and the
2486 * suspend callbacks are run. suspend puts the hardware and software state
2487 * in each IP into a state suitable for suspend.
2488 * Returns 0 on success, negative error code on failure.
2489 */
2490static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2491{
2492 int i, r;
2493
2494 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2495 if (!adev->ip_blocks[i].status.valid)
2496 continue;
2497 /* displays are handled in phase1 */
2498 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2499 continue;
2500 /* PSP lost connection when err_event_athub occurs */
2501 if (amdgpu_ras_intr_triggered() &&
2502 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2503 adev->ip_blocks[i].status.hw = false;
2504 continue;
2505 }
2506 /* XXX handle errors */
2507 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2508 /* XXX handle errors */
2509 if (r) {
2510 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2511 adev->ip_blocks[i].version->funcs->name, r);
2512 }
2513 adev->ip_blocks[i].status.hw = false;
2514 /* handle putting the SMC in the appropriate state */
2515 if(!amdgpu_sriov_vf(adev)){
2516 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2517 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2518 if (r) {
2519 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2520 adev->mp1_state, r);
2521 return r;
2522 }
2523 }
2524 }
2525 adev->ip_blocks[i].status.hw = false;
2526 }
2527
2528 return 0;
2529}
2530
2531/**
2532 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2533 *
2534 * @adev: amdgpu_device pointer
2535 *
2536 * Main suspend function for hardware IPs. The list of all the hardware
2537 * IPs that make up the asic is walked, clockgating is disabled and the
2538 * suspend callbacks are run. suspend puts the hardware and software state
2539 * in each IP into a state suitable for suspend.
2540 * Returns 0 on success, negative error code on failure.
2541 */
2542int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2543{
2544 int r;
2545
2546 if (amdgpu_sriov_vf(adev))
2547 amdgpu_virt_request_full_gpu(adev, false);
2548
2549 r = amdgpu_device_ip_suspend_phase1(adev);
2550 if (r)
2551 return r;
2552 r = amdgpu_device_ip_suspend_phase2(adev);
2553
2554 if (amdgpu_sriov_vf(adev))
2555 amdgpu_virt_release_full_gpu(adev, false);
2556
2557 return r;
2558}
2559
2560static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2561{
2562 int i, r;
2563
2564 static enum amd_ip_block_type ip_order[] = {
2565 AMD_IP_BLOCK_TYPE_GMC,
2566 AMD_IP_BLOCK_TYPE_COMMON,
2567 AMD_IP_BLOCK_TYPE_PSP,
2568 AMD_IP_BLOCK_TYPE_IH,
2569 };
2570
2571 for (i = 0; i < adev->num_ip_blocks; i++)
2572 adev->ip_blocks[i].status.hw = false;
2573
2574 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2575 int j;
2576 struct amdgpu_ip_block *block;
2577
2578 for (j = 0; j < adev->num_ip_blocks; j++) {
2579 block = &adev->ip_blocks[j];
2580
2581 if (block->version->type != ip_order[i] ||
2582 !block->status.valid)
2583 continue;
2584
2585 r = block->version->funcs->hw_init(adev);
2586 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2587 if (r)
2588 return r;
2589 block->status.hw = true;
2590 }
2591 }
2592
2593 return 0;
2594}
2595
2596static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2597{
2598 int i, r;
2599
2600 static enum amd_ip_block_type ip_order[] = {
2601 AMD_IP_BLOCK_TYPE_SMC,
2602 AMD_IP_BLOCK_TYPE_DCE,
2603 AMD_IP_BLOCK_TYPE_GFX,
2604 AMD_IP_BLOCK_TYPE_SDMA,
2605 AMD_IP_BLOCK_TYPE_UVD,
2606 AMD_IP_BLOCK_TYPE_VCE,
2607 AMD_IP_BLOCK_TYPE_VCN
2608 };
2609
2610 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2611 int j;
2612 struct amdgpu_ip_block *block;
2613
2614 for (j = 0; j < adev->num_ip_blocks; j++) {
2615 block = &adev->ip_blocks[j];
2616
2617 if (block->version->type != ip_order[i] ||
2618 !block->status.valid ||
2619 block->status.hw)
2620 continue;
2621
2622 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
2623 r = block->version->funcs->resume(adev);
2624 else
2625 r = block->version->funcs->hw_init(adev);
2626
2627 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2628 if (r)
2629 return r;
2630 block->status.hw = true;
2631 }
2632 }
2633
2634 return 0;
2635}
2636
2637/**
2638 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2639 *
2640 * @adev: amdgpu_device pointer
2641 *
2642 * First resume function for hardware IPs. The list of all the hardware
2643 * IPs that make up the asic is walked and the resume callbacks are run for
2644 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2645 * after a suspend and updates the software state as necessary. This
2646 * function is also used for restoring the GPU after a GPU reset.
2647 * Returns 0 on success, negative error code on failure.
2648 */
2649static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2650{
2651 int i, r;
2652
2653 for (i = 0; i < adev->num_ip_blocks; i++) {
2654 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2655 continue;
2656 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2657 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2658 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2659
2660 r = adev->ip_blocks[i].version->funcs->resume(adev);
2661 if (r) {
2662 DRM_ERROR("resume of IP block <%s> failed %d\n",
2663 adev->ip_blocks[i].version->funcs->name, r);
2664 return r;
2665 }
2666 adev->ip_blocks[i].status.hw = true;
2667 }
2668 }
2669
2670 return 0;
2671}
2672
2673/**
2674 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2675 *
2676 * @adev: amdgpu_device pointer
2677 *
2678 * First resume function for hardware IPs. The list of all the hardware
2679 * IPs that make up the asic is walked and the resume callbacks are run for
2680 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2681 * functional state after a suspend and updates the software state as
2682 * necessary. This function is also used for restoring the GPU after a GPU
2683 * reset.
2684 * Returns 0 on success, negative error code on failure.
2685 */
2686static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2687{
2688 int i, r;
2689
2690 for (i = 0; i < adev->num_ip_blocks; i++) {
2691 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2692 continue;
2693 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2694 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2695 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2696 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2697 continue;
2698 r = adev->ip_blocks[i].version->funcs->resume(adev);
2699 if (r) {
2700 DRM_ERROR("resume of IP block <%s> failed %d\n",
2701 adev->ip_blocks[i].version->funcs->name, r);
2702 return r;
2703 }
2704 adev->ip_blocks[i].status.hw = true;
2705 }
2706
2707 return 0;
2708}
2709
2710/**
2711 * amdgpu_device_ip_resume - run resume for hardware IPs
2712 *
2713 * @adev: amdgpu_device pointer
2714 *
2715 * Main resume function for hardware IPs. The hardware IPs
2716 * are split into two resume functions because they are
2717 * are also used in in recovering from a GPU reset and some additional
2718 * steps need to be take between them. In this case (S3/S4) they are
2719 * run sequentially.
2720 * Returns 0 on success, negative error code on failure.
2721 */
2722static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2723{
2724 int r;
2725
2726 r = amdgpu_device_ip_resume_phase1(adev);
2727 if (r)
2728 return r;
2729
2730 r = amdgpu_device_fw_loading(adev);
2731 if (r)
2732 return r;
2733
2734 r = amdgpu_device_ip_resume_phase2(adev);
2735
2736 return r;
2737}
2738
2739/**
2740 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2741 *
2742 * @adev: amdgpu_device pointer
2743 *
2744 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2745 */
2746static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2747{
2748 if (amdgpu_sriov_vf(adev)) {
2749 if (adev->is_atom_fw) {
2750 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2751 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2752 } else {
2753 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2754 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2755 }
2756
2757 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2758 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2759 }
2760}
2761
2762/**
2763 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2764 *
2765 * @asic_type: AMD asic type
2766 *
2767 * Check if there is DC (new modesetting infrastructre) support for an asic.
2768 * returns true if DC has support, false if not.
2769 */
2770bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2771{
2772 switch (asic_type) {
2773#if defined(CONFIG_DRM_AMD_DC)
2774 case CHIP_BONAIRE:
2775 case CHIP_KAVERI:
2776 case CHIP_KABINI:
2777 case CHIP_MULLINS:
2778 /*
2779 * We have systems in the wild with these ASICs that require
2780 * LVDS and VGA support which is not supported with DC.
2781 *
2782 * Fallback to the non-DC driver here by default so as not to
2783 * cause regressions.
2784 */
2785 return amdgpu_dc > 0;
2786 case CHIP_HAWAII:
2787 case CHIP_CARRIZO:
2788 case CHIP_STONEY:
2789 case CHIP_POLARIS10:
2790 case CHIP_POLARIS11:
2791 case CHIP_POLARIS12:
2792 case CHIP_VEGAM:
2793 case CHIP_TONGA:
2794 case CHIP_FIJI:
2795 case CHIP_VEGA10:
2796 case CHIP_VEGA12:
2797 case CHIP_VEGA20:
2798#if defined(CONFIG_DRM_AMD_DC_DCN)
2799 case CHIP_RAVEN:
2800 case CHIP_NAVI10:
2801 case CHIP_NAVI14:
2802 case CHIP_NAVI12:
2803 case CHIP_RENOIR:
2804#endif
2805#if defined(CONFIG_DRM_AMD_DC_DCN3_0)
2806 case CHIP_SIENNA_CICHLID:
2807 case CHIP_NAVY_FLOUNDER:
2808#endif
2809 return amdgpu_dc != 0;
2810#endif
2811 default:
2812 if (amdgpu_dc > 0)
2813 DRM_INFO("Display Core has been requested via kernel parameter "
2814 "but isn't supported by ASIC, ignoring\n");
2815 return false;
2816 }
2817}
2818
2819/**
2820 * amdgpu_device_has_dc_support - check if dc is supported
2821 *
2822 * @adev: amdgpu_device_pointer
2823 *
2824 * Returns true for supported, false for not supported
2825 */
2826bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2827{
2828 if (amdgpu_sriov_vf(adev))
2829 return false;
2830
2831 return amdgpu_device_asic_has_dc_support(adev->asic_type);
2832}
2833
2834
2835static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
2836{
2837 struct amdgpu_device *adev =
2838 container_of(__work, struct amdgpu_device, xgmi_reset_work);
2839 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
2840
2841 /* It's a bug to not have a hive within this function */
2842 if (WARN_ON(!hive))
2843 return;
2844
2845 /*
2846 * Use task barrier to synchronize all xgmi reset works across the
2847 * hive. task_barrier_enter and task_barrier_exit will block
2848 * until all the threads running the xgmi reset works reach
2849 * those points. task_barrier_full will do both blocks.
2850 */
2851 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
2852
2853 task_barrier_enter(&hive->tb);
2854 adev->asic_reset_res = amdgpu_device_baco_enter(adev->ddev);
2855
2856 if (adev->asic_reset_res)
2857 goto fail;
2858
2859 task_barrier_exit(&hive->tb);
2860 adev->asic_reset_res = amdgpu_device_baco_exit(adev->ddev);
2861
2862 if (adev->asic_reset_res)
2863 goto fail;
2864
2865 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
2866 adev->mmhub.funcs->reset_ras_error_count(adev);
2867 } else {
2868
2869 task_barrier_full(&hive->tb);
2870 adev->asic_reset_res = amdgpu_asic_reset(adev);
2871 }
2872
2873fail:
2874 if (adev->asic_reset_res)
2875 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
2876 adev->asic_reset_res, adev->ddev->unique);
2877}
2878
2879static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
2880{
2881 char *input = amdgpu_lockup_timeout;
2882 char *timeout_setting = NULL;
2883 int index = 0;
2884 long timeout;
2885 int ret = 0;
2886
2887 /*
2888 * By default timeout for non compute jobs is 10000.
2889 * And there is no timeout enforced on compute jobs.
2890 * In SR-IOV or passthrough mode, timeout for compute
2891 * jobs are 60000 by default.
2892 */
2893 adev->gfx_timeout = msecs_to_jiffies(10000);
2894 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
2895 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
2896 adev->compute_timeout = msecs_to_jiffies(60000);
2897 else
2898 adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;
2899
2900 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
2901 while ((timeout_setting = strsep(&input, ",")) &&
2902 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
2903 ret = kstrtol(timeout_setting, 0, &timeout);
2904 if (ret)
2905 return ret;
2906
2907 if (timeout == 0) {
2908 index++;
2909 continue;
2910 } else if (timeout < 0) {
2911 timeout = MAX_SCHEDULE_TIMEOUT;
2912 } else {
2913 timeout = msecs_to_jiffies(timeout);
2914 }
2915
2916 switch (index++) {
2917 case 0:
2918 adev->gfx_timeout = timeout;
2919 break;
2920 case 1:
2921 adev->compute_timeout = timeout;
2922 break;
2923 case 2:
2924 adev->sdma_timeout = timeout;
2925 break;
2926 case 3:
2927 adev->video_timeout = timeout;
2928 break;
2929 default:
2930 break;
2931 }
2932 }
2933 /*
2934 * There is only one value specified and
2935 * it should apply to all non-compute jobs.
2936 */
2937 if (index == 1) {
2938 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
2939 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
2940 adev->compute_timeout = adev->gfx_timeout;
2941 }
2942 }
2943
2944 return ret;
2945}
2946
2947static const struct attribute *amdgpu_dev_attributes[] = {
2948 &dev_attr_product_name.attr,
2949 &dev_attr_product_number.attr,
2950 &dev_attr_serial_number.attr,
2951 &dev_attr_pcie_replay_count.attr,
2952 NULL
2953};
2954
2955/**
2956 * amdgpu_device_init - initialize the driver
2957 *
2958 * @adev: amdgpu_device pointer
2959 * @ddev: drm dev pointer
2960 * @pdev: pci dev pointer
2961 * @flags: driver flags
2962 *
2963 * Initializes the driver info and hw (all asics).
2964 * Returns 0 for success or an error on failure.
2965 * Called at driver startup.
2966 */
2967int amdgpu_device_init(struct amdgpu_device *adev,
2968 struct drm_device *ddev,
2969 struct pci_dev *pdev,
2970 uint32_t flags)
2971{
2972 int r, i;
2973 bool boco = false;
2974 u32 max_MBps;
2975
2976 adev->shutdown = false;
2977 adev->dev = &pdev->dev;
2978 adev->ddev = ddev;
2979 adev->pdev = pdev;
2980 adev->flags = flags;
2981
2982 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
2983 adev->asic_type = amdgpu_force_asic_type;
2984 else
2985 adev->asic_type = flags & AMD_ASIC_MASK;
2986
2987 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2988 if (amdgpu_emu_mode == 1)
2989 adev->usec_timeout *= 10;
2990 adev->gmc.gart_size = 512 * 1024 * 1024;
2991 adev->accel_working = false;
2992 adev->num_rings = 0;
2993 adev->mman.buffer_funcs = NULL;
2994 adev->mman.buffer_funcs_ring = NULL;
2995 adev->vm_manager.vm_pte_funcs = NULL;
2996 adev->vm_manager.vm_pte_num_scheds = 0;
2997 adev->gmc.gmc_funcs = NULL;
2998 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2999 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3000
3001 adev->smc_rreg = &amdgpu_invalid_rreg;
3002 adev->smc_wreg = &amdgpu_invalid_wreg;
3003 adev->pcie_rreg = &amdgpu_invalid_rreg;
3004 adev->pcie_wreg = &amdgpu_invalid_wreg;
3005 adev->pciep_rreg = &amdgpu_invalid_rreg;
3006 adev->pciep_wreg = &amdgpu_invalid_wreg;
3007 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3008 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3009 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3010 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3011 adev->didt_rreg = &amdgpu_invalid_rreg;
3012 adev->didt_wreg = &amdgpu_invalid_wreg;
3013 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3014 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3015 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3016 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3017
3018 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3019 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3020 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3021
3022 /* mutex initialization are all done here so we
3023 * can recall function without having locking issues */
3024 atomic_set(&adev->irq.ih.lock, 0);
3025 mutex_init(&adev->firmware.mutex);
3026 mutex_init(&adev->pm.mutex);
3027 mutex_init(&adev->gfx.gpu_clock_mutex);
3028 mutex_init(&adev->srbm_mutex);
3029 mutex_init(&adev->gfx.pipe_reserve_mutex);
3030 mutex_init(&adev->gfx.gfx_off_mutex);
3031 mutex_init(&adev->grbm_idx_mutex);
3032 mutex_init(&adev->mn_lock);
3033 mutex_init(&adev->virt.vf_errors.lock);
3034 hash_init(adev->mn_hash);
3035 mutex_init(&adev->lock_reset);
3036 mutex_init(&adev->psp.mutex);
3037 mutex_init(&adev->notifier_lock);
3038
3039 r = amdgpu_device_check_arguments(adev);
3040 if (r)
3041 return r;
3042
3043 spin_lock_init(&adev->mmio_idx_lock);
3044 spin_lock_init(&adev->smc_idx_lock);
3045 spin_lock_init(&adev->pcie_idx_lock);
3046 spin_lock_init(&adev->uvd_ctx_idx_lock);
3047 spin_lock_init(&adev->didt_idx_lock);
3048 spin_lock_init(&adev->gc_cac_idx_lock);
3049 spin_lock_init(&adev->se_cac_idx_lock);
3050 spin_lock_init(&adev->audio_endpt_idx_lock);
3051 spin_lock_init(&adev->mm_stats.lock);
3052
3053 INIT_LIST_HEAD(&adev->shadow_list);
3054 mutex_init(&adev->shadow_list_lock);
3055
3056 INIT_DELAYED_WORK(&adev->delayed_init_work,
3057 amdgpu_device_delayed_init_work_handler);
3058 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3059 amdgpu_device_delay_enable_gfx_off);
3060
3061 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3062
3063 adev->gfx.gfx_off_req_count = 1;
3064 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3065
3066 atomic_set(&adev->throttling_logging_enabled, 1);
3067 /*
3068 * If throttling continues, logging will be performed every minute
3069 * to avoid log flooding. "-1" is subtracted since the thermal
3070 * throttling interrupt comes every second. Thus, the total logging
3071 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3072 * for throttling interrupt) = 60 seconds.
3073 */
3074 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3075 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3076
3077 /* Registers mapping */
3078 /* TODO: block userspace mapping of io register */
3079 if (adev->asic_type >= CHIP_BONAIRE) {
3080 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3081 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3082 } else {
3083 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3084 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3085 }
3086
3087 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3088 if (adev->rmmio == NULL) {
3089 return -ENOMEM;
3090 }
3091 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3092 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3093
3094 /* io port mapping */
3095 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
3096 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
3097 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
3098 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
3099 break;
3100 }
3101 }
3102 if (adev->rio_mem == NULL)
3103 DRM_INFO("PCI I/O BAR is not found.\n");
3104
3105 /* enable PCIE atomic ops */
3106 r = pci_enable_atomic_ops_to_root(adev->pdev,
3107 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3108 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3109 if (r) {
3110 adev->have_atomics_support = false;
3111 DRM_INFO("PCIE atomic ops is not supported\n");
3112 } else {
3113 adev->have_atomics_support = true;
3114 }
3115
3116 amdgpu_device_get_pcie_info(adev);
3117
3118 if (amdgpu_mcbp)
3119 DRM_INFO("MCBP is enabled\n");
3120
3121 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3122 adev->enable_mes = true;
3123
3124 /* detect hw virtualization here */
3125 amdgpu_detect_virtualization(adev);
3126
3127 r = amdgpu_device_get_job_timeout_settings(adev);
3128 if (r) {
3129 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3130 return r;
3131 }
3132
3133 /* early init functions */
3134 r = amdgpu_device_ip_early_init(adev);
3135 if (r)
3136 return r;
3137
3138 /* doorbell bar mapping and doorbell index init*/
3139 amdgpu_device_doorbell_init(adev);
3140
3141 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3142 /* this will fail for cards that aren't VGA class devices, just
3143 * ignore it */
3144 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
3145
3146 if (amdgpu_device_supports_boco(ddev))
3147 boco = true;
3148 if (amdgpu_has_atpx() &&
3149 (amdgpu_is_atpx_hybrid() ||
3150 amdgpu_has_atpx_dgpu_power_cntl()) &&
3151 !pci_is_thunderbolt_attached(adev->pdev))
3152 vga_switcheroo_register_client(adev->pdev,
3153 &amdgpu_switcheroo_ops, boco);
3154 if (boco)
3155 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3156
3157 if (amdgpu_emu_mode == 1) {
3158 /* post the asic on emulation mode */
3159 emu_soc_asic_init(adev);
3160 goto fence_driver_init;
3161 }
3162
3163 /* detect if we are with an SRIOV vbios */
3164 amdgpu_device_detect_sriov_bios(adev);
3165
3166 /* check if we need to reset the asic
3167 * E.g., driver was not cleanly unloaded previously, etc.
3168 */
3169 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3170 r = amdgpu_asic_reset(adev);
3171 if (r) {
3172 dev_err(adev->dev, "asic reset on init failed\n");
3173 goto failed;
3174 }
3175 }
3176
3177 /* Post card if necessary */
3178 if (amdgpu_device_need_post(adev)) {
3179 if (!adev->bios) {
3180 dev_err(adev->dev, "no vBIOS found\n");
3181 r = -EINVAL;
3182 goto failed;
3183 }
3184 DRM_INFO("GPU posting now...\n");
3185 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
3186 if (r) {
3187 dev_err(adev->dev, "gpu post error!\n");
3188 goto failed;
3189 }
3190 }
3191
3192 if (adev->is_atom_fw) {
3193 /* Initialize clocks */
3194 r = amdgpu_atomfirmware_get_clock_info(adev);
3195 if (r) {
3196 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3197 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3198 goto failed;
3199 }
3200 } else {
3201 /* Initialize clocks */
3202 r = amdgpu_atombios_get_clock_info(adev);
3203 if (r) {
3204 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3205 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3206 goto failed;
3207 }
3208 /* init i2c buses */
3209 if (!amdgpu_device_has_dc_support(adev))
3210 amdgpu_atombios_i2c_init(adev);
3211 }
3212
3213fence_driver_init:
3214 /* Fence driver */
3215 r = amdgpu_fence_driver_init(adev);
3216 if (r) {
3217 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
3218 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3219 goto failed;
3220 }
3221
3222 /* init the mode config */
3223 drm_mode_config_init(adev->ddev);
3224
3225 r = amdgpu_device_ip_init(adev);
3226 if (r) {
3227 /* failed in exclusive mode due to timeout */
3228 if (amdgpu_sriov_vf(adev) &&
3229 !amdgpu_sriov_runtime(adev) &&
3230 amdgpu_virt_mmio_blocked(adev) &&
3231 !amdgpu_virt_wait_reset(adev)) {
3232 dev_err(adev->dev, "VF exclusive mode timeout\n");
3233 /* Don't send request since VF is inactive. */
3234 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3235 adev->virt.ops = NULL;
3236 r = -EAGAIN;
3237 goto failed;
3238 }
3239 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3240 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3241 goto failed;
3242 }
3243
3244 dev_info(adev->dev,
3245 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3246 adev->gfx.config.max_shader_engines,
3247 adev->gfx.config.max_sh_per_se,
3248 adev->gfx.config.max_cu_per_sh,
3249 adev->gfx.cu_info.number);
3250
3251 adev->accel_working = true;
3252
3253 amdgpu_vm_check_compute_bug(adev);
3254
3255 /* Initialize the buffer migration limit. */
3256 if (amdgpu_moverate >= 0)
3257 max_MBps = amdgpu_moverate;
3258 else
3259 max_MBps = 8; /* Allow 8 MB/s. */
3260 /* Get a log2 for easy divisions. */
3261 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3262
3263 amdgpu_fbdev_init(adev);
3264
3265 r = amdgpu_pm_sysfs_init(adev);
3266 if (r) {
3267 adev->pm_sysfs_en = false;
3268 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3269 } else
3270 adev->pm_sysfs_en = true;
3271
3272 r = amdgpu_ucode_sysfs_init(adev);
3273 if (r) {
3274 adev->ucode_sysfs_en = false;
3275 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3276 } else
3277 adev->ucode_sysfs_en = true;
3278
3279 if ((amdgpu_testing & 1)) {
3280 if (adev->accel_working)
3281 amdgpu_test_moves(adev);
3282 else
3283 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3284 }
3285 if (amdgpu_benchmarking) {
3286 if (adev->accel_working)
3287 amdgpu_benchmark(adev, amdgpu_benchmarking);
3288 else
3289 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3290 }
3291
3292 /*
3293 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3294 * Otherwise the mgpu fan boost feature will be skipped due to the
3295 * gpu instance is counted less.
3296 */
3297 amdgpu_register_gpu_instance(adev);
3298
3299 /* enable clockgating, etc. after ib tests, etc. since some blocks require
3300 * explicit gating rather than handling it automatically.
3301 */
3302 r = amdgpu_device_ip_late_init(adev);
3303 if (r) {
3304 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3305 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3306 goto failed;
3307 }
3308
3309 /* must succeed. */
3310 amdgpu_ras_resume(adev);
3311
3312 queue_delayed_work(system_wq, &adev->delayed_init_work,
3313 msecs_to_jiffies(AMDGPU_RESUME_MS));
3314
3315 if (amdgpu_sriov_vf(adev))
3316 flush_delayed_work(&adev->delayed_init_work);
3317
3318 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3319 if (r) {
3320 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3321 return r;
3322 }
3323
3324 if (IS_ENABLED(CONFIG_PERF_EVENTS))
3325 r = amdgpu_pmu_init(adev);
3326 if (r)
3327 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3328
3329 return 0;
3330
3331failed:
3332 amdgpu_vf_error_trans_all(adev);
3333 if (boco)
3334 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3335
3336 return r;
3337}
3338
3339/**
3340 * amdgpu_device_fini - tear down the driver
3341 *
3342 * @adev: amdgpu_device pointer
3343 *
3344 * Tear down the driver info (all asics).
3345 * Called at driver shutdown.
3346 */
3347void amdgpu_device_fini(struct amdgpu_device *adev)
3348{
3349 int r;
3350
3351 DRM_INFO("amdgpu: finishing device.\n");
3352 flush_delayed_work(&adev->delayed_init_work);
3353 adev->shutdown = true;
3354
3355 /* make sure IB test finished before entering exclusive mode
3356 * to avoid preemption on IB test
3357 * */
3358 if (amdgpu_sriov_vf(adev))
3359 amdgpu_virt_request_full_gpu(adev, false);
3360
3361 /* disable all interrupts */
3362 amdgpu_irq_disable_all(adev);
3363 if (adev->mode_info.mode_config_initialized){
3364 if (!amdgpu_device_has_dc_support(adev))
3365 drm_helper_force_disable_all(adev->ddev);
3366 else
3367 drm_atomic_helper_shutdown(adev->ddev);
3368 }
3369 amdgpu_fence_driver_fini(adev);
3370 if (adev->pm_sysfs_en)
3371 amdgpu_pm_sysfs_fini(adev);
3372 amdgpu_fbdev_fini(adev);
3373 r = amdgpu_device_ip_fini(adev);
3374 release_firmware(adev->firmware.gpu_info_fw);
3375 adev->firmware.gpu_info_fw = NULL;
3376 adev->accel_working = false;
3377 /* free i2c buses */
3378 if (!amdgpu_device_has_dc_support(adev))
3379 amdgpu_i2c_fini(adev);
3380
3381 if (amdgpu_emu_mode != 1)
3382 amdgpu_atombios_fini(adev);
3383
3384 kfree(adev->bios);
3385 adev->bios = NULL;
3386 if (amdgpu_has_atpx() &&
3387 (amdgpu_is_atpx_hybrid() ||
3388 amdgpu_has_atpx_dgpu_power_cntl()) &&
3389 !pci_is_thunderbolt_attached(adev->pdev))
3390 vga_switcheroo_unregister_client(adev->pdev);
3391 if (amdgpu_device_supports_boco(adev->ddev))
3392 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3393 vga_client_register(adev->pdev, NULL, NULL, NULL);
3394 if (adev->rio_mem)
3395 pci_iounmap(adev->pdev, adev->rio_mem);
3396 adev->rio_mem = NULL;
3397 iounmap(adev->rmmio);
3398 adev->rmmio = NULL;
3399 amdgpu_device_doorbell_fini(adev);
3400
3401 if (adev->ucode_sysfs_en)
3402 amdgpu_ucode_sysfs_fini(adev);
3403
3404 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3405 if (IS_ENABLED(CONFIG_PERF_EVENTS))
3406 amdgpu_pmu_fini(adev);
3407 if (adev->discovery_bin)
3408 amdgpu_discovery_fini(adev);
3409}
3410
3411
3412/*
3413 * Suspend & resume.
3414 */
3415/**
3416 * amdgpu_device_suspend - initiate device suspend
3417 *
3418 * @dev: drm dev pointer
3419 * @fbcon : notify the fbdev of suspend
3420 *
3421 * Puts the hw in the suspend state (all asics).
3422 * Returns 0 for success or an error on failure.
3423 * Called at driver suspend.
3424 */
3425int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3426{
3427 struct amdgpu_device *adev;
3428 struct drm_crtc *crtc;
3429 struct drm_connector *connector;
3430 struct drm_connector_list_iter iter;
3431 int r;
3432
3433 if (dev == NULL || dev->dev_private == NULL) {
3434 return -ENODEV;
3435 }
3436
3437 adev = dev->dev_private;
3438
3439 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3440 return 0;
3441
3442 adev->in_suspend = true;
3443 drm_kms_helper_poll_disable(dev);
3444
3445 if (fbcon)
3446 amdgpu_fbdev_set_suspend(adev, 1);
3447
3448 cancel_delayed_work_sync(&adev->delayed_init_work);
3449
3450 if (!amdgpu_device_has_dc_support(adev)) {
3451 /* turn off display hw */
3452 drm_modeset_lock_all(dev);
3453 drm_connector_list_iter_begin(dev, &iter);
3454 drm_for_each_connector_iter(connector, &iter)
3455 drm_helper_connector_dpms(connector,
3456 DRM_MODE_DPMS_OFF);
3457 drm_connector_list_iter_end(&iter);
3458 drm_modeset_unlock_all(dev);
3459 /* unpin the front buffers and cursors */
3460 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3461 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3462 struct drm_framebuffer *fb = crtc->primary->fb;
3463 struct amdgpu_bo *robj;
3464
3465 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3466 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3467 r = amdgpu_bo_reserve(aobj, true);
3468 if (r == 0) {
3469 amdgpu_bo_unpin(aobj);
3470 amdgpu_bo_unreserve(aobj);
3471 }
3472 }
3473
3474 if (fb == NULL || fb->obj[0] == NULL) {
3475 continue;
3476 }
3477 robj = gem_to_amdgpu_bo(fb->obj[0]);
3478 /* don't unpin kernel fb objects */
3479 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
3480 r = amdgpu_bo_reserve(robj, true);
3481 if (r == 0) {
3482 amdgpu_bo_unpin(robj);
3483 amdgpu_bo_unreserve(robj);
3484 }
3485 }
3486 }
3487 }
3488
3489 amdgpu_ras_suspend(adev);
3490
3491 r = amdgpu_device_ip_suspend_phase1(adev);
3492
3493 amdgpu_amdkfd_suspend(adev, !fbcon);
3494
3495 /* evict vram memory */
3496 amdgpu_bo_evict_vram(adev);
3497
3498 amdgpu_fence_driver_suspend(adev);
3499
3500 r = amdgpu_device_ip_suspend_phase2(adev);
3501
3502 /* evict remaining vram memory
3503 * This second call to evict vram is to evict the gart page table
3504 * using the CPU.
3505 */
3506 amdgpu_bo_evict_vram(adev);
3507
3508 return 0;
3509}
3510
3511/**
3512 * amdgpu_device_resume - initiate device resume
3513 *
3514 * @dev: drm dev pointer
3515 * @fbcon : notify the fbdev of resume
3516 *
3517 * Bring the hw back to operating state (all asics).
3518 * Returns 0 for success or an error on failure.
3519 * Called at driver resume.
3520 */
3521int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
3522{
3523 struct drm_connector *connector;
3524 struct drm_connector_list_iter iter;
3525 struct amdgpu_device *adev = dev->dev_private;
3526 struct drm_crtc *crtc;
3527 int r = 0;
3528
3529 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3530 return 0;
3531
3532 /* post card */
3533 if (amdgpu_device_need_post(adev)) {
3534 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
3535 if (r)
3536 DRM_ERROR("amdgpu asic init failed\n");
3537 }
3538
3539 r = amdgpu_device_ip_resume(adev);
3540 if (r) {
3541 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
3542 return r;
3543 }
3544 amdgpu_fence_driver_resume(adev);
3545
3546
3547 r = amdgpu_device_ip_late_init(adev);
3548 if (r)
3549 return r;
3550
3551 queue_delayed_work(system_wq, &adev->delayed_init_work,
3552 msecs_to_jiffies(AMDGPU_RESUME_MS));
3553
3554 if (!amdgpu_device_has_dc_support(adev)) {
3555 /* pin cursors */
3556 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3557 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3558
3559 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3560 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3561 r = amdgpu_bo_reserve(aobj, true);
3562 if (r == 0) {
3563 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
3564 if (r != 0)
3565 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
3566 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
3567 amdgpu_bo_unreserve(aobj);
3568 }
3569 }
3570 }
3571 }
3572 r = amdgpu_amdkfd_resume(adev, !fbcon);
3573 if (r)
3574 return r;
3575
3576 /* Make sure IB tests flushed */
3577 flush_delayed_work(&adev->delayed_init_work);
3578
3579 /* blat the mode back in */
3580 if (fbcon) {
3581 if (!amdgpu_device_has_dc_support(adev)) {
3582 /* pre DCE11 */
3583 drm_helper_resume_force_mode(dev);
3584
3585 /* turn on display hw */
3586 drm_modeset_lock_all(dev);
3587
3588 drm_connector_list_iter_begin(dev, &iter);
3589 drm_for_each_connector_iter(connector, &iter)
3590 drm_helper_connector_dpms(connector,
3591 DRM_MODE_DPMS_ON);
3592 drm_connector_list_iter_end(&iter);
3593
3594 drm_modeset_unlock_all(dev);
3595 }
3596 amdgpu_fbdev_set_suspend(adev, 0);
3597 }
3598
3599 drm_kms_helper_poll_enable(dev);
3600
3601 amdgpu_ras_resume(adev);
3602
3603 /*
3604 * Most of the connector probing functions try to acquire runtime pm
3605 * refs to ensure that the GPU is powered on when connector polling is
3606 * performed. Since we're calling this from a runtime PM callback,
3607 * trying to acquire rpm refs will cause us to deadlock.
3608 *
3609 * Since we're guaranteed to be holding the rpm lock, it's safe to
3610 * temporarily disable the rpm helpers so this doesn't deadlock us.
3611 */
3612#ifdef CONFIG_PM
3613 dev->dev->power.disable_depth++;
3614#endif
3615 if (!amdgpu_device_has_dc_support(adev))
3616 drm_helper_hpd_irq_event(dev);
3617 else
3618 drm_kms_helper_hotplug_event(dev);
3619#ifdef CONFIG_PM
3620 dev->dev->power.disable_depth--;
3621#endif
3622 adev->in_suspend = false;
3623
3624 return 0;
3625}
3626
3627/**
3628 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3629 *
3630 * @adev: amdgpu_device pointer
3631 *
3632 * The list of all the hardware IPs that make up the asic is walked and
3633 * the check_soft_reset callbacks are run. check_soft_reset determines
3634 * if the asic is still hung or not.
3635 * Returns true if any of the IPs are still in a hung state, false if not.
3636 */
3637static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3638{
3639 int i;
3640 bool asic_hang = false;
3641
3642 if (amdgpu_sriov_vf(adev))
3643 return true;
3644
3645 if (amdgpu_asic_need_full_reset(adev))
3646 return true;
3647
3648 for (i = 0; i < adev->num_ip_blocks; i++) {
3649 if (!adev->ip_blocks[i].status.valid)
3650 continue;
3651 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3652 adev->ip_blocks[i].status.hang =
3653 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3654 if (adev->ip_blocks[i].status.hang) {
3655 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3656 asic_hang = true;
3657 }
3658 }
3659 return asic_hang;
3660}
3661
3662/**
3663 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3664 *
3665 * @adev: amdgpu_device pointer
3666 *
3667 * The list of all the hardware IPs that make up the asic is walked and the
3668 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
3669 * handles any IP specific hardware or software state changes that are
3670 * necessary for a soft reset to succeed.
3671 * Returns 0 on success, negative error code on failure.
3672 */
3673static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3674{
3675 int i, r = 0;
3676
3677 for (i = 0; i < adev->num_ip_blocks; i++) {
3678 if (!adev->ip_blocks[i].status.valid)
3679 continue;
3680 if (adev->ip_blocks[i].status.hang &&
3681 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3682 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3683 if (r)
3684 return r;
3685 }
3686 }
3687
3688 return 0;
3689}
3690
3691/**
3692 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3693 *
3694 * @adev: amdgpu_device pointer
3695 *
3696 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
3697 * reset is necessary to recover.
3698 * Returns true if a full asic reset is required, false if not.
3699 */
3700static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3701{
3702 int i;
3703
3704 if (amdgpu_asic_need_full_reset(adev))
3705 return true;
3706
3707 for (i = 0; i < adev->num_ip_blocks; i++) {
3708 if (!adev->ip_blocks[i].status.valid)
3709 continue;
3710 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3711 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3712 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3713 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3714 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3715 if (adev->ip_blocks[i].status.hang) {
3716 DRM_INFO("Some block need full reset!\n");
3717 return true;
3718 }
3719 }
3720 }
3721 return false;
3722}
3723
3724/**
3725 * amdgpu_device_ip_soft_reset - do a soft reset
3726 *
3727 * @adev: amdgpu_device pointer
3728 *
3729 * The list of all the hardware IPs that make up the asic is walked and the
3730 * soft_reset callbacks are run if the block is hung. soft_reset handles any
3731 * IP specific hardware or software state changes that are necessary to soft
3732 * reset the IP.
3733 * Returns 0 on success, negative error code on failure.
3734 */
3735static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3736{
3737 int i, r = 0;
3738
3739 for (i = 0; i < adev->num_ip_blocks; i++) {
3740 if (!adev->ip_blocks[i].status.valid)
3741 continue;
3742 if (adev->ip_blocks[i].status.hang &&
3743 adev->ip_blocks[i].version->funcs->soft_reset) {
3744 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
3745 if (r)
3746 return r;
3747 }
3748 }
3749
3750 return 0;
3751}
3752
3753/**
3754 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
3755 *
3756 * @adev: amdgpu_device pointer
3757 *
3758 * The list of all the hardware IPs that make up the asic is walked and the
3759 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
3760 * handles any IP specific hardware or software state changes that are
3761 * necessary after the IP has been soft reset.
3762 * Returns 0 on success, negative error code on failure.
3763 */
3764static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
3765{
3766 int i, r = 0;
3767
3768 for (i = 0; i < adev->num_ip_blocks; i++) {
3769 if (!adev->ip_blocks[i].status.valid)
3770 continue;
3771 if (adev->ip_blocks[i].status.hang &&
3772 adev->ip_blocks[i].version->funcs->post_soft_reset)
3773 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
3774 if (r)
3775 return r;
3776 }
3777
3778 return 0;
3779}
3780
3781/**
3782 * amdgpu_device_recover_vram - Recover some VRAM contents
3783 *
3784 * @adev: amdgpu_device pointer
3785 *
3786 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
3787 * restore things like GPUVM page tables after a GPU reset where
3788 * the contents of VRAM might be lost.
3789 *
3790 * Returns:
3791 * 0 on success, negative error code on failure.
3792 */
3793static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
3794{
3795 struct dma_fence *fence = NULL, *next = NULL;
3796 struct amdgpu_bo *shadow;
3797 long r = 1, tmo;
3798
3799 if (amdgpu_sriov_runtime(adev))
3800 tmo = msecs_to_jiffies(8000);
3801 else
3802 tmo = msecs_to_jiffies(100);
3803
3804 DRM_INFO("recover vram bo from shadow start\n");
3805 mutex_lock(&adev->shadow_list_lock);
3806 list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {
3807
3808 /* No need to recover an evicted BO */
3809 if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
3810 shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
3811 shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
3812 continue;
3813
3814 r = amdgpu_bo_restore_shadow(shadow, &next);
3815 if (r)
3816 break;
3817
3818 if (fence) {
3819 tmo = dma_fence_wait_timeout(fence, false, tmo);
3820 dma_fence_put(fence);
3821 fence = next;
3822 if (tmo == 0) {
3823 r = -ETIMEDOUT;
3824 break;
3825 } else if (tmo < 0) {
3826 r = tmo;
3827 break;
3828 }
3829 } else {
3830 fence = next;
3831 }
3832 }
3833 mutex_unlock(&adev->shadow_list_lock);
3834
3835 if (fence)
3836 tmo = dma_fence_wait_timeout(fence, false, tmo);
3837 dma_fence_put(fence);
3838
3839 if (r < 0 || tmo <= 0) {
3840 DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
3841 return -EIO;
3842 }
3843
3844 DRM_INFO("recover vram bo from shadow done\n");
3845 return 0;
3846}
3847
3848
3849/**
3850 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3851 *
3852 * @adev: amdgpu device pointer
3853 * @from_hypervisor: request from hypervisor
3854 *
3855 * do VF FLR and reinitialize Asic
3856 * return 0 means succeeded otherwise failed
3857 */
3858static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3859 bool from_hypervisor)
3860{
3861 int r;
3862
3863 if (from_hypervisor)
3864 r = amdgpu_virt_request_full_gpu(adev, true);
3865 else
3866 r = amdgpu_virt_reset_gpu(adev);
3867 if (r)
3868 return r;
3869
3870 amdgpu_amdkfd_pre_reset(adev);
3871
3872 /* Resume IP prior to SMC */
3873 r = amdgpu_device_ip_reinit_early_sriov(adev);
3874 if (r)
3875 goto error;
3876
3877 amdgpu_virt_init_data_exchange(adev);
3878 /* we need recover gart prior to run SMC/CP/SDMA resume */
3879 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3880
3881 r = amdgpu_device_fw_loading(adev);
3882 if (r)
3883 return r;
3884
3885 /* now we are okay to resume SMC/CP/SDMA */
3886 r = amdgpu_device_ip_reinit_late_sriov(adev);
3887 if (r)
3888 goto error;
3889
3890 amdgpu_irq_gpu_reset_resume_helper(adev);
3891 r = amdgpu_ib_ring_tests(adev);
3892 amdgpu_amdkfd_post_reset(adev);
3893
3894error:
3895 amdgpu_virt_release_full_gpu(adev, true);
3896 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3897 amdgpu_inc_vram_lost(adev);
3898 r = amdgpu_device_recover_vram(adev);
3899 }
3900
3901 return r;
3902}
3903
3904/**
3905 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
3906 *
3907 * @adev: amdgpu device pointer
3908 *
3909 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
3910 * a hung GPU.
3911 */
3912bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
3913{
3914 if (!amdgpu_device_ip_check_soft_reset(adev)) {
3915 DRM_INFO("Timeout, but no hardware hang detected.\n");
3916 return false;
3917 }
3918
3919 if (amdgpu_gpu_recovery == 0)
3920 goto disabled;
3921
3922 if (amdgpu_sriov_vf(adev))
3923 return true;
3924
3925 if (amdgpu_gpu_recovery == -1) {
3926 switch (adev->asic_type) {
3927 case CHIP_BONAIRE:
3928 case CHIP_HAWAII:
3929 case CHIP_TOPAZ:
3930 case CHIP_TONGA:
3931 case CHIP_FIJI:
3932 case CHIP_POLARIS10:
3933 case CHIP_POLARIS11:
3934 case CHIP_POLARIS12:
3935 case CHIP_VEGAM:
3936 case CHIP_VEGA20:
3937 case CHIP_VEGA10:
3938 case CHIP_VEGA12:
3939 case CHIP_RAVEN:
3940 case CHIP_ARCTURUS:
3941 case CHIP_RENOIR:
3942 case CHIP_NAVI10:
3943 case CHIP_NAVI14:
3944 case CHIP_NAVI12:
3945 case CHIP_SIENNA_CICHLID:
3946 break;
3947 default:
3948 goto disabled;
3949 }
3950 }
3951
3952 return true;
3953
3954disabled:
3955 DRM_INFO("GPU recovery disabled.\n");
3956 return false;
3957}
3958
3959
3960static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
3961 struct amdgpu_job *job,
3962 bool *need_full_reset_arg)
3963{
3964 int i, r = 0;
3965 bool need_full_reset = *need_full_reset_arg;
3966
3967 amdgpu_debugfs_wait_dump(adev);
3968
3969 /* block all schedulers and reset given job's ring */
3970 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3971 struct amdgpu_ring *ring = adev->rings[i];
3972
3973 if (!ring || !ring->sched.thread)
3974 continue;
3975
3976 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3977 amdgpu_fence_driver_force_completion(ring);
3978 }
3979
3980 if(job)
3981 drm_sched_increase_karma(&job->base);
3982
3983 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
3984 if (!amdgpu_sriov_vf(adev)) {
3985
3986 if (!need_full_reset)
3987 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
3988
3989 if (!need_full_reset) {
3990 amdgpu_device_ip_pre_soft_reset(adev);
3991 r = amdgpu_device_ip_soft_reset(adev);
3992 amdgpu_device_ip_post_soft_reset(adev);
3993 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
3994 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3995 need_full_reset = true;
3996 }
3997 }
3998
3999 if (need_full_reset)
4000 r = amdgpu_device_ip_suspend(adev);
4001
4002 *need_full_reset_arg = need_full_reset;
4003 }
4004
4005 return r;
4006}
4007
4008static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
4009 struct list_head *device_list_handle,
4010 bool *need_full_reset_arg)
4011{
4012 struct amdgpu_device *tmp_adev = NULL;
4013 bool need_full_reset = *need_full_reset_arg, vram_lost = false;
4014 int r = 0;
4015
4016 /*
4017 * ASIC reset has to be done on all HGMI hive nodes ASAP
4018 * to allow proper links negotiation in FW (within 1 sec)
4019 */
4020 if (need_full_reset) {
4021 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4022 /* For XGMI run all resets in parallel to speed up the process */
4023 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4024 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4025 r = -EALREADY;
4026 } else
4027 r = amdgpu_asic_reset(tmp_adev);
4028
4029 if (r) {
4030 DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s",
4031 r, tmp_adev->ddev->unique);
4032 break;
4033 }
4034 }
4035
4036 /* For XGMI wait for all resets to complete before proceed */
4037 if (!r) {
4038 list_for_each_entry(tmp_adev, device_list_handle,
4039 gmc.xgmi.head) {
4040 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4041 flush_work(&tmp_adev->xgmi_reset_work);
4042 r = tmp_adev->asic_reset_res;
4043 if (r)
4044 break;
4045 }
4046 }
4047 }
4048 }
4049
4050 if (!r && amdgpu_ras_intr_triggered()) {
4051 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4052 if (tmp_adev->mmhub.funcs &&
4053 tmp_adev->mmhub.funcs->reset_ras_error_count)
4054 tmp_adev->mmhub.funcs->reset_ras_error_count(tmp_adev);
4055 }
4056
4057 amdgpu_ras_intr_cleared();
4058 }
4059
4060 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4061 if (need_full_reset) {
4062 /* post card */
4063 if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context))
4064 DRM_WARN("asic atom init failed!");
4065
4066 if (!r) {
4067 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4068 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4069 if (r)
4070 goto out;
4071
4072 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4073 if (vram_lost) {
4074 DRM_INFO("VRAM is lost due to GPU reset!\n");
4075 amdgpu_inc_vram_lost(tmp_adev);
4076 }
4077
4078 r = amdgpu_gtt_mgr_recover(
4079 &tmp_adev->mman.bdev.man[TTM_PL_TT]);
4080 if (r)
4081 goto out;
4082
4083 r = amdgpu_device_fw_loading(tmp_adev);
4084 if (r)
4085 return r;
4086
4087 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4088 if (r)
4089 goto out;
4090
4091 if (vram_lost)
4092 amdgpu_device_fill_reset_magic(tmp_adev);
4093
4094 /*
4095 * Add this ASIC as tracked as reset was already
4096 * complete successfully.
4097 */
4098 amdgpu_register_gpu_instance(tmp_adev);
4099
4100 r = amdgpu_device_ip_late_init(tmp_adev);
4101 if (r)
4102 goto out;
4103
4104 amdgpu_fbdev_set_suspend(tmp_adev, 0);
4105
4106 /* must succeed. */
4107 amdgpu_ras_resume(tmp_adev);
4108
4109 /* Update PSP FW topology after reset */
4110 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4111 r = amdgpu_xgmi_update_topology(hive, tmp_adev);
4112 }
4113 }
4114
4115
4116out:
4117 if (!r) {
4118 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4119 r = amdgpu_ib_ring_tests(tmp_adev);
4120 if (r) {
4121 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4122 r = amdgpu_device_ip_suspend(tmp_adev);
4123 need_full_reset = true;
4124 r = -EAGAIN;
4125 goto end;
4126 }
4127 }
4128
4129 if (!r)
4130 r = amdgpu_device_recover_vram(tmp_adev);
4131 else
4132 tmp_adev->asic_reset_res = r;
4133 }
4134
4135end:
4136 *need_full_reset_arg = need_full_reset;
4137 return r;
4138}
4139
4140static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock)
4141{
4142 if (trylock) {
4143 if (!mutex_trylock(&adev->lock_reset))
4144 return false;
4145 } else
4146 mutex_lock(&adev->lock_reset);
4147
4148 atomic_inc(&adev->gpu_reset_counter);
4149 adev->in_gpu_reset = true;
4150 switch (amdgpu_asic_reset_method(adev)) {
4151 case AMD_RESET_METHOD_MODE1:
4152 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4153 break;
4154 case AMD_RESET_METHOD_MODE2:
4155 adev->mp1_state = PP_MP1_STATE_RESET;
4156 break;
4157 default:
4158 adev->mp1_state = PP_MP1_STATE_NONE;
4159 break;
4160 }
4161
4162 return true;
4163}
4164
4165static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4166{
4167 amdgpu_vf_error_trans_all(adev);
4168 adev->mp1_state = PP_MP1_STATE_NONE;
4169 adev->in_gpu_reset = false;
4170 mutex_unlock(&adev->lock_reset);
4171}
4172
4173static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4174{
4175 struct pci_dev *p = NULL;
4176
4177 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4178 adev->pdev->bus->number, 1);
4179 if (p) {
4180 pm_runtime_enable(&(p->dev));
4181 pm_runtime_resume(&(p->dev));
4182 }
4183}
4184
4185static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4186{
4187 enum amd_reset_method reset_method;
4188 struct pci_dev *p = NULL;
4189 u64 expires;
4190
4191 /*
4192 * For now, only BACO and mode1 reset are confirmed
4193 * to suffer the audio issue without proper suspended.
4194 */
4195 reset_method = amdgpu_asic_reset_method(adev);
4196 if ((reset_method != AMD_RESET_METHOD_BACO) &&
4197 (reset_method != AMD_RESET_METHOD_MODE1))
4198 return -EINVAL;
4199
4200 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4201 adev->pdev->bus->number, 1);
4202 if (!p)
4203 return -ENODEV;
4204
4205 expires = pm_runtime_autosuspend_expiration(&(p->dev));
4206 if (!expires)
4207 /*
4208 * If we cannot get the audio device autosuspend delay,
4209 * a fixed 4S interval will be used. Considering 3S is
4210 * the audio controller default autosuspend delay setting.
4211 * 4S used here is guaranteed to cover that.
4212 */
4213 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4214
4215 while (!pm_runtime_status_suspended(&(p->dev))) {
4216 if (!pm_runtime_suspend(&(p->dev)))
4217 break;
4218
4219 if (expires < ktime_get_mono_fast_ns()) {
4220 dev_warn(adev->dev, "failed to suspend display audio\n");
4221 /* TODO: abort the succeeding gpu reset? */
4222 return -ETIMEDOUT;
4223 }
4224 }
4225
4226 pm_runtime_disable(&(p->dev));
4227
4228 return 0;
4229}
4230
4231/**
4232 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4233 *
4234 * @adev: amdgpu device pointer
4235 * @job: which job trigger hang
4236 *
4237 * Attempt to reset the GPU if it has hung (all asics).
4238 * Attempt to do soft-reset or full-reset and reinitialize Asic
4239 * Returns 0 for success or an error on failure.
4240 */
4241
4242int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4243 struct amdgpu_job *job)
4244{
4245 struct list_head device_list, *device_list_handle = NULL;
4246 bool need_full_reset = false;
4247 bool job_signaled = false;
4248 struct amdgpu_hive_info *hive = NULL;
4249 struct amdgpu_device *tmp_adev = NULL;
4250 int i, r = 0;
4251 bool need_emergency_restart = false;
4252 bool audio_suspended = false;
4253
4254 /**
4255 * Special case: RAS triggered and full reset isn't supported
4256 */
4257 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4258
4259 /*
4260 * Flush RAM to disk so that after reboot
4261 * the user can read log and see why the system rebooted.
4262 */
4263 if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
4264 DRM_WARN("Emergency reboot.");
4265
4266 ksys_sync_helper();
4267 emergency_restart();
4268 }
4269
4270 dev_info(adev->dev, "GPU %s begin!\n",
4271 need_emergency_restart ? "jobs stop":"reset");
4272
4273 /*
4274 * Here we trylock to avoid chain of resets executing from
4275 * either trigger by jobs on different adevs in XGMI hive or jobs on
4276 * different schedulers for same device while this TO handler is running.
4277 * We always reset all schedulers for device and all devices for XGMI
4278 * hive so that should take care of them too.
4279 */
4280 hive = amdgpu_get_xgmi_hive(adev, true);
4281 if (hive && !mutex_trylock(&hive->reset_lock)) {
4282 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
4283 job ? job->base.id : -1, hive->hive_id);
4284 mutex_unlock(&hive->hive_lock);
4285 return 0;
4286 }
4287
4288 /*
4289 * Build list of devices to reset.
4290 * In case we are in XGMI hive mode, resort the device list
4291 * to put adev in the 1st position.
4292 */
4293 INIT_LIST_HEAD(&device_list);
4294 if (adev->gmc.xgmi.num_physical_nodes > 1) {
4295 if (!hive)
4296 return -ENODEV;
4297 if (!list_is_first(&adev->gmc.xgmi.head, &hive->device_list))
4298 list_rotate_to_front(&adev->gmc.xgmi.head, &hive->device_list);
4299 device_list_handle = &hive->device_list;
4300 } else {
4301 list_add_tail(&adev->gmc.xgmi.head, &device_list);
4302 device_list_handle = &device_list;
4303 }
4304
4305 /* block all schedulers and reset given job's ring */
4306 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4307 if (!amdgpu_device_lock_adev(tmp_adev, !hive)) {
4308 DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress",
4309 job ? job->base.id : -1);
4310 mutex_unlock(&hive->hive_lock);
4311 return 0;
4312 }
4313
4314 /*
4315 * Try to put the audio codec into suspend state
4316 * before gpu reset started.
4317 *
4318 * Due to the power domain of the graphics device
4319 * is shared with AZ power domain. Without this,
4320 * we may change the audio hardware from behind
4321 * the audio driver's back. That will trigger
4322 * some audio codec errors.
4323 */
4324 if (!amdgpu_device_suspend_display_audio(tmp_adev))
4325 audio_suspended = true;
4326
4327 amdgpu_ras_set_error_query_ready(tmp_adev, false);
4328
4329 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
4330
4331 if (!amdgpu_sriov_vf(tmp_adev))
4332 amdgpu_amdkfd_pre_reset(tmp_adev);
4333
4334 /*
4335 * Mark these ASICs to be reseted as untracked first
4336 * And add them back after reset completed
4337 */
4338 amdgpu_unregister_gpu_instance(tmp_adev);
4339
4340 amdgpu_fbdev_set_suspend(tmp_adev, 1);
4341
4342 /* disable ras on ALL IPs */
4343 if (!need_emergency_restart &&
4344 amdgpu_device_ip_need_full_reset(tmp_adev))
4345 amdgpu_ras_suspend(tmp_adev);
4346
4347 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4348 struct amdgpu_ring *ring = tmp_adev->rings[i];
4349
4350 if (!ring || !ring->sched.thread)
4351 continue;
4352
4353 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
4354
4355 if (need_emergency_restart)
4356 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
4357 }
4358 }
4359
4360 if (need_emergency_restart)
4361 goto skip_sched_resume;
4362
4363 /*
4364 * Must check guilty signal here since after this point all old
4365 * HW fences are force signaled.
4366 *
4367 * job->base holds a reference to parent fence
4368 */
4369 if (job && job->base.s_fence->parent &&
4370 dma_fence_is_signaled(job->base.s_fence->parent)) {
4371 job_signaled = true;
4372 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
4373 goto skip_hw_reset;
4374 }
4375
4376retry: /* Rest of adevs pre asic reset from XGMI hive. */
4377 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4378 r = amdgpu_device_pre_asic_reset(tmp_adev,
4379 NULL,
4380 &need_full_reset);
4381 /*TODO Should we stop ?*/
4382 if (r) {
4383 DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
4384 r, tmp_adev->ddev->unique);
4385 tmp_adev->asic_reset_res = r;
4386 }
4387 }
4388
4389 /* Actual ASIC resets if needed.*/
4390 /* TODO Implement XGMI hive reset logic for SRIOV */
4391 if (amdgpu_sriov_vf(adev)) {
4392 r = amdgpu_device_reset_sriov(adev, job ? false : true);
4393 if (r)
4394 adev->asic_reset_res = r;
4395 } else {
4396 r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
4397 if (r && r == -EAGAIN)
4398 goto retry;
4399 }
4400
4401skip_hw_reset:
4402
4403 /* Post ASIC reset for all devs .*/
4404 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4405
4406 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4407 struct amdgpu_ring *ring = tmp_adev->rings[i];
4408
4409 if (!ring || !ring->sched.thread)
4410 continue;
4411
4412 /* No point to resubmit jobs if we didn't HW reset*/
4413 if (!tmp_adev->asic_reset_res && !job_signaled)
4414 drm_sched_resubmit_jobs(&ring->sched);
4415
4416 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
4417 }
4418
4419 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
4420 drm_helper_resume_force_mode(tmp_adev->ddev);
4421 }
4422
4423 tmp_adev->asic_reset_res = 0;
4424
4425 if (r) {
4426 /* bad news, how to tell it to userspace ? */
4427 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
4428 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
4429 } else {
4430 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
4431 }
4432 }
4433
4434skip_sched_resume:
4435 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
4436 /*unlock kfd: SRIOV would do it separately */
4437 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
4438 amdgpu_amdkfd_post_reset(tmp_adev);
4439 if (audio_suspended)
4440 amdgpu_device_resume_display_audio(tmp_adev);
4441 amdgpu_device_unlock_adev(tmp_adev);
4442 }
4443
4444 if (hive) {
4445 mutex_unlock(&hive->reset_lock);
4446 mutex_unlock(&hive->hive_lock);
4447 }
4448
4449 if (r)
4450 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
4451 return r;
4452}
4453
4454/**
4455 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
4456 *
4457 * @adev: amdgpu_device pointer
4458 *
4459 * Fetchs and stores in the driver the PCIE capabilities (gen speed
4460 * and lanes) of the slot the device is in. Handles APUs and
4461 * virtualized environments where PCIE config space may not be available.
4462 */
4463static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
4464{
4465 struct pci_dev *pdev;
4466 enum pci_bus_speed speed_cap, platform_speed_cap;
4467 enum pcie_link_width platform_link_width;
4468
4469 if (amdgpu_pcie_gen_cap)
4470 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
4471
4472 if (amdgpu_pcie_lane_cap)
4473 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
4474
4475 /* covers APUs as well */
4476 if (pci_is_root_bus(adev->pdev->bus)) {
4477 if (adev->pm.pcie_gen_mask == 0)
4478 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
4479 if (adev->pm.pcie_mlw_mask == 0)
4480 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
4481 return;
4482 }
4483
4484 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
4485 return;
4486
4487 pcie_bandwidth_available(adev->pdev, NULL,
4488 &platform_speed_cap, &platform_link_width);
4489
4490 if (adev->pm.pcie_gen_mask == 0) {
4491 /* asic caps */
4492 pdev = adev->pdev;
4493 speed_cap = pcie_get_speed_cap(pdev);
4494 if (speed_cap == PCI_SPEED_UNKNOWN) {
4495 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4496 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4497 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4498 } else {
4499 if (speed_cap == PCIE_SPEED_16_0GT)
4500 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4501 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4502 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4503 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
4504 else if (speed_cap == PCIE_SPEED_8_0GT)
4505 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4506 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4507 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4508 else if (speed_cap == PCIE_SPEED_5_0GT)
4509 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4510 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
4511 else
4512 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
4513 }
4514 /* platform caps */
4515 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
4516 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4517 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4518 } else {
4519 if (platform_speed_cap == PCIE_SPEED_16_0GT)
4520 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4521 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4522 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4523 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
4524 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
4525 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4526 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4527 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
4528 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
4529 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4530 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4531 else
4532 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
4533
4534 }
4535 }
4536 if (adev->pm.pcie_mlw_mask == 0) {
4537 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
4538 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
4539 } else {
4540 switch (platform_link_width) {
4541 case PCIE_LNK_X32:
4542 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
4543 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4544 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4545 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4546 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4547 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4548 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4549 break;
4550 case PCIE_LNK_X16:
4551 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4552 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4553 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4554 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4555 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4556 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4557 break;
4558 case PCIE_LNK_X12:
4559 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4560 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4561 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4562 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4563 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4564 break;
4565 case PCIE_LNK_X8:
4566 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4567 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4568 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4569 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4570 break;
4571 case PCIE_LNK_X4:
4572 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4573 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4574 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4575 break;
4576 case PCIE_LNK_X2:
4577 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4578 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4579 break;
4580 case PCIE_LNK_X1:
4581 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
4582 break;
4583 default:
4584 break;
4585 }
4586 }
4587 }
4588}
4589
4590int amdgpu_device_baco_enter(struct drm_device *dev)
4591{
4592 struct amdgpu_device *adev = dev->dev_private;
4593 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4594
4595 if (!amdgpu_device_supports_baco(adev->ddev))
4596 return -ENOTSUPP;
4597
4598 if (ras && ras->supported)
4599 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
4600
4601 return amdgpu_dpm_baco_enter(adev);
4602}
4603
4604int amdgpu_device_baco_exit(struct drm_device *dev)
4605{
4606 struct amdgpu_device *adev = dev->dev_private;
4607 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4608 int ret = 0;
4609
4610 if (!amdgpu_device_supports_baco(adev->ddev))
4611 return -ENOTSUPP;
4612
4613 ret = amdgpu_dpm_baco_exit(adev);
4614 if (ret)
4615 return ret;
4616
4617 if (ras && ras->supported)
4618 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
4619
4620 return 0;
4621}
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28
29#include <linux/aperture.h>
30#include <linux/power_supply.h>
31#include <linux/kthread.h>
32#include <linux/module.h>
33#include <linux/console.h>
34#include <linux/slab.h>
35#include <linux/iommu.h>
36#include <linux/pci.h>
37#include <linux/pci-p2pdma.h>
38#include <linux/apple-gmux.h>
39
40#include <drm/drm_atomic_helper.h>
41#include <drm/drm_client_event.h>
42#include <drm/drm_crtc_helper.h>
43#include <drm/drm_probe_helper.h>
44#include <drm/amdgpu_drm.h>
45#include <linux/device.h>
46#include <linux/vgaarb.h>
47#include <linux/vga_switcheroo.h>
48#include <linux/efi.h>
49#include "amdgpu.h"
50#include "amdgpu_trace.h"
51#include "amdgpu_i2c.h"
52#include "atom.h"
53#include "amdgpu_atombios.h"
54#include "amdgpu_atomfirmware.h"
55#include "amd_pcie.h"
56#ifdef CONFIG_DRM_AMDGPU_SI
57#include "si.h"
58#endif
59#ifdef CONFIG_DRM_AMDGPU_CIK
60#include "cik.h"
61#endif
62#include "vi.h"
63#include "soc15.h"
64#include "nv.h"
65#include "bif/bif_4_1_d.h"
66#include <linux/firmware.h>
67#include "amdgpu_vf_error.h"
68
69#include "amdgpu_amdkfd.h"
70#include "amdgpu_pm.h"
71
72#include "amdgpu_xgmi.h"
73#include "amdgpu_ras.h"
74#include "amdgpu_pmu.h"
75#include "amdgpu_fru_eeprom.h"
76#include "amdgpu_reset.h"
77#include "amdgpu_virt.h"
78#include "amdgpu_dev_coredump.h"
79
80#include <linux/suspend.h>
81#include <drm/task_barrier.h>
82#include <linux/pm_runtime.h>
83
84#include <drm/drm_drv.h>
85
86#if IS_ENABLED(CONFIG_X86)
87#include <asm/intel-family.h>
88#endif
89
90MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
91MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
92MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
93MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
94MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
95MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
96MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
97
98#define AMDGPU_RESUME_MS 2000
99#define AMDGPU_MAX_RETRY_LIMIT 2
100#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
101#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
102#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
103#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
104
105static const struct drm_driver amdgpu_kms_driver;
106
107const char *amdgpu_asic_name[] = {
108 "TAHITI",
109 "PITCAIRN",
110 "VERDE",
111 "OLAND",
112 "HAINAN",
113 "BONAIRE",
114 "KAVERI",
115 "KABINI",
116 "HAWAII",
117 "MULLINS",
118 "TOPAZ",
119 "TONGA",
120 "FIJI",
121 "CARRIZO",
122 "STONEY",
123 "POLARIS10",
124 "POLARIS11",
125 "POLARIS12",
126 "VEGAM",
127 "VEGA10",
128 "VEGA12",
129 "VEGA20",
130 "RAVEN",
131 "ARCTURUS",
132 "RENOIR",
133 "ALDEBARAN",
134 "NAVI10",
135 "CYAN_SKILLFISH",
136 "NAVI14",
137 "NAVI12",
138 "SIENNA_CICHLID",
139 "NAVY_FLOUNDER",
140 "VANGOGH",
141 "DIMGREY_CAVEFISH",
142 "BEIGE_GOBY",
143 "YELLOW_CARP",
144 "IP DISCOVERY",
145 "LAST",
146};
147
148#define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0)
149/*
150 * Default init level where all blocks are expected to be initialized. This is
151 * the level of initialization expected by default and also after a full reset
152 * of the device.
153 */
154struct amdgpu_init_level amdgpu_init_default = {
155 .level = AMDGPU_INIT_LEVEL_DEFAULT,
156 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
157};
158
159struct amdgpu_init_level amdgpu_init_recovery = {
160 .level = AMDGPU_INIT_LEVEL_RESET_RECOVERY,
161 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
162};
163
164/*
165 * Minimal blocks needed to be initialized before a XGMI hive can be reset. This
166 * is used for cases like reset on initialization where the entire hive needs to
167 * be reset before first use.
168 */
169struct amdgpu_init_level amdgpu_init_minimal_xgmi = {
170 .level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI,
171 .hwini_ip_block_mask =
172 BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) |
173 BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) |
174 BIT(AMD_IP_BLOCK_TYPE_PSP)
175};
176
177static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,
178 enum amd_ip_block_type block)
179{
180 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0;
181}
182
183void amdgpu_set_init_level(struct amdgpu_device *adev,
184 enum amdgpu_init_lvl_id lvl)
185{
186 switch (lvl) {
187 case AMDGPU_INIT_LEVEL_MINIMAL_XGMI:
188 adev->init_lvl = &amdgpu_init_minimal_xgmi;
189 break;
190 case AMDGPU_INIT_LEVEL_RESET_RECOVERY:
191 adev->init_lvl = &amdgpu_init_recovery;
192 break;
193 case AMDGPU_INIT_LEVEL_DEFAULT:
194 fallthrough;
195 default:
196 adev->init_lvl = &amdgpu_init_default;
197 break;
198 }
199}
200
201static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
202
203/**
204 * DOC: pcie_replay_count
205 *
206 * The amdgpu driver provides a sysfs API for reporting the total number
207 * of PCIe replays (NAKs)
208 * The file pcie_replay_count is used for this and returns the total
209 * number of replays as a sum of the NAKs generated and NAKs received
210 */
211
212static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
213 struct device_attribute *attr, char *buf)
214{
215 struct drm_device *ddev = dev_get_drvdata(dev);
216 struct amdgpu_device *adev = drm_to_adev(ddev);
217 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
218
219 return sysfs_emit(buf, "%llu\n", cnt);
220}
221
222static DEVICE_ATTR(pcie_replay_count, 0444,
223 amdgpu_device_get_pcie_replay_count, NULL);
224
225static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
226 struct bin_attribute *attr, char *buf,
227 loff_t ppos, size_t count)
228{
229 struct device *dev = kobj_to_dev(kobj);
230 struct drm_device *ddev = dev_get_drvdata(dev);
231 struct amdgpu_device *adev = drm_to_adev(ddev);
232 ssize_t bytes_read;
233
234 switch (ppos) {
235 case AMDGPU_SYS_REG_STATE_XGMI:
236 bytes_read = amdgpu_asic_get_reg_state(
237 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
238 break;
239 case AMDGPU_SYS_REG_STATE_WAFL:
240 bytes_read = amdgpu_asic_get_reg_state(
241 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
242 break;
243 case AMDGPU_SYS_REG_STATE_PCIE:
244 bytes_read = amdgpu_asic_get_reg_state(
245 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
246 break;
247 case AMDGPU_SYS_REG_STATE_USR:
248 bytes_read = amdgpu_asic_get_reg_state(
249 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
250 break;
251 case AMDGPU_SYS_REG_STATE_USR_1:
252 bytes_read = amdgpu_asic_get_reg_state(
253 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
254 break;
255 default:
256 return -EINVAL;
257 }
258
259 return bytes_read;
260}
261
262BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
263 AMDGPU_SYS_REG_STATE_END);
264
265int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
266{
267 int ret;
268
269 if (!amdgpu_asic_get_reg_state_supported(adev))
270 return 0;
271
272 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
273
274 return ret;
275}
276
277void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
278{
279 if (!amdgpu_asic_get_reg_state_supported(adev))
280 return;
281 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
282}
283
284int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block)
285{
286 int r;
287
288 if (ip_block->version->funcs->suspend) {
289 r = ip_block->version->funcs->suspend(ip_block);
290 if (r) {
291 dev_err(ip_block->adev->dev,
292 "suspend of IP block <%s> failed %d\n",
293 ip_block->version->funcs->name, r);
294 return r;
295 }
296 }
297
298 ip_block->status.hw = false;
299 return 0;
300}
301
302int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block)
303{
304 int r;
305
306 if (ip_block->version->funcs->resume) {
307 r = ip_block->version->funcs->resume(ip_block);
308 if (r) {
309 dev_err(ip_block->adev->dev,
310 "resume of IP block <%s> failed %d\n",
311 ip_block->version->funcs->name, r);
312 return r;
313 }
314 }
315
316 ip_block->status.hw = true;
317 return 0;
318}
319
320/**
321 * DOC: board_info
322 *
323 * The amdgpu driver provides a sysfs API for giving board related information.
324 * It provides the form factor information in the format
325 *
326 * type : form factor
327 *
328 * Possible form factor values
329 *
330 * - "cem" - PCIE CEM card
331 * - "oam" - Open Compute Accelerator Module
332 * - "unknown" - Not known
333 *
334 */
335
336static ssize_t amdgpu_device_get_board_info(struct device *dev,
337 struct device_attribute *attr,
338 char *buf)
339{
340 struct drm_device *ddev = dev_get_drvdata(dev);
341 struct amdgpu_device *adev = drm_to_adev(ddev);
342 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
343 const char *pkg;
344
345 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
346 pkg_type = adev->smuio.funcs->get_pkg_type(adev);
347
348 switch (pkg_type) {
349 case AMDGPU_PKG_TYPE_CEM:
350 pkg = "cem";
351 break;
352 case AMDGPU_PKG_TYPE_OAM:
353 pkg = "oam";
354 break;
355 default:
356 pkg = "unknown";
357 break;
358 }
359
360 return sysfs_emit(buf, "%s : %s\n", "type", pkg);
361}
362
363static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
364
365static struct attribute *amdgpu_board_attrs[] = {
366 &dev_attr_board_info.attr,
367 NULL,
368};
369
370static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
371 struct attribute *attr, int n)
372{
373 struct device *dev = kobj_to_dev(kobj);
374 struct drm_device *ddev = dev_get_drvdata(dev);
375 struct amdgpu_device *adev = drm_to_adev(ddev);
376
377 if (adev->flags & AMD_IS_APU)
378 return 0;
379
380 return attr->mode;
381}
382
383static const struct attribute_group amdgpu_board_attrs_group = {
384 .attrs = amdgpu_board_attrs,
385 .is_visible = amdgpu_board_attrs_is_visible
386};
387
388static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
389
390
391/**
392 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
393 *
394 * @dev: drm_device pointer
395 *
396 * Returns true if the device is a dGPU with ATPX power control,
397 * otherwise return false.
398 */
399bool amdgpu_device_supports_px(struct drm_device *dev)
400{
401 struct amdgpu_device *adev = drm_to_adev(dev);
402
403 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
404 return true;
405 return false;
406}
407
408/**
409 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
410 *
411 * @dev: drm_device pointer
412 *
413 * Returns true if the device is a dGPU with ACPI power control,
414 * otherwise return false.
415 */
416bool amdgpu_device_supports_boco(struct drm_device *dev)
417{
418 struct amdgpu_device *adev = drm_to_adev(dev);
419
420 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
421 return false;
422
423 if (adev->has_pr3 ||
424 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
425 return true;
426 return false;
427}
428
429/**
430 * amdgpu_device_supports_baco - Does the device support BACO
431 *
432 * @dev: drm_device pointer
433 *
434 * Return:
435 * 1 if the device supporte BACO;
436 * 3 if the device support MACO (only works if BACO is supported)
437 * otherwise return 0.
438 */
439int amdgpu_device_supports_baco(struct drm_device *dev)
440{
441 struct amdgpu_device *adev = drm_to_adev(dev);
442
443 return amdgpu_asic_supports_baco(adev);
444}
445
446void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)
447{
448 struct drm_device *dev;
449 int bamaco_support;
450
451 dev = adev_to_drm(adev);
452
453 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
454 bamaco_support = amdgpu_device_supports_baco(dev);
455
456 switch (amdgpu_runtime_pm) {
457 case 2:
458 if (bamaco_support & MACO_SUPPORT) {
459 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
460 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");
461 } else if (bamaco_support == BACO_SUPPORT) {
462 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
463 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");
464 }
465 break;
466 case 1:
467 if (bamaco_support & BACO_SUPPORT) {
468 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
469 dev_info(adev->dev, "Forcing BACO for runtime pm\n");
470 }
471 break;
472 case -1:
473 case -2:
474 if (amdgpu_device_supports_px(dev)) { /* enable PX as runtime mode */
475 adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
476 dev_info(adev->dev, "Using ATPX for runtime pm\n");
477 } else if (amdgpu_device_supports_boco(dev)) { /* enable boco as runtime mode */
478 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
479 dev_info(adev->dev, "Using BOCO for runtime pm\n");
480 } else {
481 if (!bamaco_support)
482 goto no_runtime_pm;
483
484 switch (adev->asic_type) {
485 case CHIP_VEGA20:
486 case CHIP_ARCTURUS:
487 /* BACO are not supported on vega20 and arctrus */
488 break;
489 case CHIP_VEGA10:
490 /* enable BACO as runpm mode if noretry=0 */
491 if (!adev->gmc.noretry)
492 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
493 break;
494 default:
495 /* enable BACO as runpm mode on CI+ */
496 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
497 break;
498 }
499
500 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
501 if (bamaco_support & MACO_SUPPORT) {
502 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
503 dev_info(adev->dev, "Using BAMACO for runtime pm\n");
504 } else {
505 dev_info(adev->dev, "Using BACO for runtime pm\n");
506 }
507 }
508 }
509 break;
510 case 0:
511 dev_info(adev->dev, "runtime pm is manually disabled\n");
512 break;
513 default:
514 break;
515 }
516
517no_runtime_pm:
518 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)
519 dev_info(adev->dev, "Runtime PM not available\n");
520}
521/**
522 * amdgpu_device_supports_smart_shift - Is the device dGPU with
523 * smart shift support
524 *
525 * @dev: drm_device pointer
526 *
527 * Returns true if the device is a dGPU with Smart Shift support,
528 * otherwise returns false.
529 */
530bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
531{
532 return (amdgpu_device_supports_boco(dev) &&
533 amdgpu_acpi_is_power_shift_control_supported());
534}
535
536/*
537 * VRAM access helper functions
538 */
539
540/**
541 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
542 *
543 * @adev: amdgpu_device pointer
544 * @pos: offset of the buffer in vram
545 * @buf: virtual address of the buffer in system memory
546 * @size: read/write size, sizeof(@buf) must > @size
547 * @write: true - write to vram, otherwise - read from vram
548 */
549void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
550 void *buf, size_t size, bool write)
551{
552 unsigned long flags;
553 uint32_t hi = ~0, tmp = 0;
554 uint32_t *data = buf;
555 uint64_t last;
556 int idx;
557
558 if (!drm_dev_enter(adev_to_drm(adev), &idx))
559 return;
560
561 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
562
563 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
564 for (last = pos + size; pos < last; pos += 4) {
565 tmp = pos >> 31;
566
567 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
568 if (tmp != hi) {
569 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
570 hi = tmp;
571 }
572 if (write)
573 WREG32_NO_KIQ(mmMM_DATA, *data++);
574 else
575 *data++ = RREG32_NO_KIQ(mmMM_DATA);
576 }
577
578 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
579 drm_dev_exit(idx);
580}
581
582/**
583 * amdgpu_device_aper_access - access vram by vram aperature
584 *
585 * @adev: amdgpu_device pointer
586 * @pos: offset of the buffer in vram
587 * @buf: virtual address of the buffer in system memory
588 * @size: read/write size, sizeof(@buf) must > @size
589 * @write: true - write to vram, otherwise - read from vram
590 *
591 * The return value means how many bytes have been transferred.
592 */
593size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
594 void *buf, size_t size, bool write)
595{
596#ifdef CONFIG_64BIT
597 void __iomem *addr;
598 size_t count = 0;
599 uint64_t last;
600
601 if (!adev->mman.aper_base_kaddr)
602 return 0;
603
604 last = min(pos + size, adev->gmc.visible_vram_size);
605 if (last > pos) {
606 addr = adev->mman.aper_base_kaddr + pos;
607 count = last - pos;
608
609 if (write) {
610 memcpy_toio(addr, buf, count);
611 /* Make sure HDP write cache flush happens without any reordering
612 * after the system memory contents are sent over PCIe device
613 */
614 mb();
615 amdgpu_device_flush_hdp(adev, NULL);
616 } else {
617 amdgpu_device_invalidate_hdp(adev, NULL);
618 /* Make sure HDP read cache is invalidated before issuing a read
619 * to the PCIe device
620 */
621 mb();
622 memcpy_fromio(buf, addr, count);
623 }
624
625 }
626
627 return count;
628#else
629 return 0;
630#endif
631}
632
633/**
634 * amdgpu_device_vram_access - read/write a buffer in vram
635 *
636 * @adev: amdgpu_device pointer
637 * @pos: offset of the buffer in vram
638 * @buf: virtual address of the buffer in system memory
639 * @size: read/write size, sizeof(@buf) must > @size
640 * @write: true - write to vram, otherwise - read from vram
641 */
642void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
643 void *buf, size_t size, bool write)
644{
645 size_t count;
646
647 /* try to using vram apreature to access vram first */
648 count = amdgpu_device_aper_access(adev, pos, buf, size, write);
649 size -= count;
650 if (size) {
651 /* using MM to access rest vram */
652 pos += count;
653 buf += count;
654 amdgpu_device_mm_access(adev, pos, buf, size, write);
655 }
656}
657
658/*
659 * register access helper functions.
660 */
661
662/* Check if hw access should be skipped because of hotplug or device error */
663bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
664{
665 if (adev->no_hw_access)
666 return true;
667
668#ifdef CONFIG_LOCKDEP
669 /*
670 * This is a bit complicated to understand, so worth a comment. What we assert
671 * here is that the GPU reset is not running on another thread in parallel.
672 *
673 * For this we trylock the read side of the reset semaphore, if that succeeds
674 * we know that the reset is not running in paralell.
675 *
676 * If the trylock fails we assert that we are either already holding the read
677 * side of the lock or are the reset thread itself and hold the write side of
678 * the lock.
679 */
680 if (in_task()) {
681 if (down_read_trylock(&adev->reset_domain->sem))
682 up_read(&adev->reset_domain->sem);
683 else
684 lockdep_assert_held(&adev->reset_domain->sem);
685 }
686#endif
687 return false;
688}
689
690/**
691 * amdgpu_device_rreg - read a memory mapped IO or indirect register
692 *
693 * @adev: amdgpu_device pointer
694 * @reg: dword aligned register offset
695 * @acc_flags: access flags which require special behavior
696 *
697 * Returns the 32 bit value from the offset specified.
698 */
699uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
700 uint32_t reg, uint32_t acc_flags)
701{
702 uint32_t ret;
703
704 if (amdgpu_device_skip_hw_access(adev))
705 return 0;
706
707 if ((reg * 4) < adev->rmmio_size) {
708 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
709 amdgpu_sriov_runtime(adev) &&
710 down_read_trylock(&adev->reset_domain->sem)) {
711 ret = amdgpu_kiq_rreg(adev, reg, 0);
712 up_read(&adev->reset_domain->sem);
713 } else {
714 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
715 }
716 } else {
717 ret = adev->pcie_rreg(adev, reg * 4);
718 }
719
720 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
721
722 return ret;
723}
724
725/*
726 * MMIO register read with bytes helper functions
727 * @offset:bytes offset from MMIO start
728 */
729
730/**
731 * amdgpu_mm_rreg8 - read a memory mapped IO register
732 *
733 * @adev: amdgpu_device pointer
734 * @offset: byte aligned register offset
735 *
736 * Returns the 8 bit value from the offset specified.
737 */
738uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
739{
740 if (amdgpu_device_skip_hw_access(adev))
741 return 0;
742
743 if (offset < adev->rmmio_size)
744 return (readb(adev->rmmio + offset));
745 BUG();
746}
747
748
749/**
750 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
751 *
752 * @adev: amdgpu_device pointer
753 * @reg: dword aligned register offset
754 * @acc_flags: access flags which require special behavior
755 * @xcc_id: xcc accelerated compute core id
756 *
757 * Returns the 32 bit value from the offset specified.
758 */
759uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
760 uint32_t reg, uint32_t acc_flags,
761 uint32_t xcc_id)
762{
763 uint32_t ret, rlcg_flag;
764
765 if (amdgpu_device_skip_hw_access(adev))
766 return 0;
767
768 if ((reg * 4) < adev->rmmio_size) {
769 if (amdgpu_sriov_vf(adev) &&
770 !amdgpu_sriov_runtime(adev) &&
771 adev->gfx.rlc.rlcg_reg_access_supported &&
772 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
773 GC_HWIP, false,
774 &rlcg_flag)) {
775 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
776 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
777 amdgpu_sriov_runtime(adev) &&
778 down_read_trylock(&adev->reset_domain->sem)) {
779 ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
780 up_read(&adev->reset_domain->sem);
781 } else {
782 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
783 }
784 } else {
785 ret = adev->pcie_rreg(adev, reg * 4);
786 }
787
788 return ret;
789}
790
791/*
792 * MMIO register write with bytes helper functions
793 * @offset:bytes offset from MMIO start
794 * @value: the value want to be written to the register
795 */
796
797/**
798 * amdgpu_mm_wreg8 - read a memory mapped IO register
799 *
800 * @adev: amdgpu_device pointer
801 * @offset: byte aligned register offset
802 * @value: 8 bit value to write
803 *
804 * Writes the value specified to the offset specified.
805 */
806void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
807{
808 if (amdgpu_device_skip_hw_access(adev))
809 return;
810
811 if (offset < adev->rmmio_size)
812 writeb(value, adev->rmmio + offset);
813 else
814 BUG();
815}
816
817/**
818 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
819 *
820 * @adev: amdgpu_device pointer
821 * @reg: dword aligned register offset
822 * @v: 32 bit value to write to the register
823 * @acc_flags: access flags which require special behavior
824 *
825 * Writes the value specified to the offset specified.
826 */
827void amdgpu_device_wreg(struct amdgpu_device *adev,
828 uint32_t reg, uint32_t v,
829 uint32_t acc_flags)
830{
831 if (amdgpu_device_skip_hw_access(adev))
832 return;
833
834 if ((reg * 4) < adev->rmmio_size) {
835 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
836 amdgpu_sriov_runtime(adev) &&
837 down_read_trylock(&adev->reset_domain->sem)) {
838 amdgpu_kiq_wreg(adev, reg, v, 0);
839 up_read(&adev->reset_domain->sem);
840 } else {
841 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
842 }
843 } else {
844 adev->pcie_wreg(adev, reg * 4, v);
845 }
846
847 trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
848}
849
850/**
851 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range
852 *
853 * @adev: amdgpu_device pointer
854 * @reg: mmio/rlc register
855 * @v: value to write
856 * @xcc_id: xcc accelerated compute core id
857 *
858 * this function is invoked only for the debugfs register access
859 */
860void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
861 uint32_t reg, uint32_t v,
862 uint32_t xcc_id)
863{
864 if (amdgpu_device_skip_hw_access(adev))
865 return;
866
867 if (amdgpu_sriov_fullaccess(adev) &&
868 adev->gfx.rlc.funcs &&
869 adev->gfx.rlc.funcs->is_rlcg_access_range) {
870 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
871 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
872 } else if ((reg * 4) >= adev->rmmio_size) {
873 adev->pcie_wreg(adev, reg * 4, v);
874 } else {
875 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
876 }
877}
878
879/**
880 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
881 *
882 * @adev: amdgpu_device pointer
883 * @reg: dword aligned register offset
884 * @v: 32 bit value to write to the register
885 * @acc_flags: access flags which require special behavior
886 * @xcc_id: xcc accelerated compute core id
887 *
888 * Writes the value specified to the offset specified.
889 */
890void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
891 uint32_t reg, uint32_t v,
892 uint32_t acc_flags, uint32_t xcc_id)
893{
894 uint32_t rlcg_flag;
895
896 if (amdgpu_device_skip_hw_access(adev))
897 return;
898
899 if ((reg * 4) < adev->rmmio_size) {
900 if (amdgpu_sriov_vf(adev) &&
901 !amdgpu_sriov_runtime(adev) &&
902 adev->gfx.rlc.rlcg_reg_access_supported &&
903 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
904 GC_HWIP, true,
905 &rlcg_flag)) {
906 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
907 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
908 amdgpu_sriov_runtime(adev) &&
909 down_read_trylock(&adev->reset_domain->sem)) {
910 amdgpu_kiq_wreg(adev, reg, v, xcc_id);
911 up_read(&adev->reset_domain->sem);
912 } else {
913 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
914 }
915 } else {
916 adev->pcie_wreg(adev, reg * 4, v);
917 }
918}
919
920/**
921 * amdgpu_device_indirect_rreg - read an indirect register
922 *
923 * @adev: amdgpu_device pointer
924 * @reg_addr: indirect register address to read from
925 *
926 * Returns the value of indirect register @reg_addr
927 */
928u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
929 u32 reg_addr)
930{
931 unsigned long flags, pcie_index, pcie_data;
932 void __iomem *pcie_index_offset;
933 void __iomem *pcie_data_offset;
934 u32 r;
935
936 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
937 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
938
939 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
940 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
941 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
942
943 writel(reg_addr, pcie_index_offset);
944 readl(pcie_index_offset);
945 r = readl(pcie_data_offset);
946 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
947
948 return r;
949}
950
951u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
952 u64 reg_addr)
953{
954 unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
955 u32 r;
956 void __iomem *pcie_index_offset;
957 void __iomem *pcie_index_hi_offset;
958 void __iomem *pcie_data_offset;
959
960 if (unlikely(!adev->nbio.funcs)) {
961 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
962 pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
963 } else {
964 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
965 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
966 }
967
968 if (reg_addr >> 32) {
969 if (unlikely(!adev->nbio.funcs))
970 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
971 else
972 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
973 } else {
974 pcie_index_hi = 0;
975 }
976
977 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
978 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
979 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
980 if (pcie_index_hi != 0)
981 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
982 pcie_index_hi * 4;
983
984 writel(reg_addr, pcie_index_offset);
985 readl(pcie_index_offset);
986 if (pcie_index_hi != 0) {
987 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
988 readl(pcie_index_hi_offset);
989 }
990 r = readl(pcie_data_offset);
991
992 /* clear the high bits */
993 if (pcie_index_hi != 0) {
994 writel(0, pcie_index_hi_offset);
995 readl(pcie_index_hi_offset);
996 }
997
998 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
999
1000 return r;
1001}
1002
1003/**
1004 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
1005 *
1006 * @adev: amdgpu_device pointer
1007 * @reg_addr: indirect register address to read from
1008 *
1009 * Returns the value of indirect register @reg_addr
1010 */
1011u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
1012 u32 reg_addr)
1013{
1014 unsigned long flags, pcie_index, pcie_data;
1015 void __iomem *pcie_index_offset;
1016 void __iomem *pcie_data_offset;
1017 u64 r;
1018
1019 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1020 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1021
1022 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1023 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1024 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1025
1026 /* read low 32 bits */
1027 writel(reg_addr, pcie_index_offset);
1028 readl(pcie_index_offset);
1029 r = readl(pcie_data_offset);
1030 /* read high 32 bits */
1031 writel(reg_addr + 4, pcie_index_offset);
1032 readl(pcie_index_offset);
1033 r |= ((u64)readl(pcie_data_offset) << 32);
1034 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1035
1036 return r;
1037}
1038
1039u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
1040 u64 reg_addr)
1041{
1042 unsigned long flags, pcie_index, pcie_data;
1043 unsigned long pcie_index_hi = 0;
1044 void __iomem *pcie_index_offset;
1045 void __iomem *pcie_index_hi_offset;
1046 void __iomem *pcie_data_offset;
1047 u64 r;
1048
1049 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1050 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1051 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1052 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1053
1054 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1055 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1056 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1057 if (pcie_index_hi != 0)
1058 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1059 pcie_index_hi * 4;
1060
1061 /* read low 32 bits */
1062 writel(reg_addr, pcie_index_offset);
1063 readl(pcie_index_offset);
1064 if (pcie_index_hi != 0) {
1065 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1066 readl(pcie_index_hi_offset);
1067 }
1068 r = readl(pcie_data_offset);
1069 /* read high 32 bits */
1070 writel(reg_addr + 4, pcie_index_offset);
1071 readl(pcie_index_offset);
1072 if (pcie_index_hi != 0) {
1073 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1074 readl(pcie_index_hi_offset);
1075 }
1076 r |= ((u64)readl(pcie_data_offset) << 32);
1077
1078 /* clear the high bits */
1079 if (pcie_index_hi != 0) {
1080 writel(0, pcie_index_hi_offset);
1081 readl(pcie_index_hi_offset);
1082 }
1083
1084 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1085
1086 return r;
1087}
1088
1089/**
1090 * amdgpu_device_indirect_wreg - write an indirect register address
1091 *
1092 * @adev: amdgpu_device pointer
1093 * @reg_addr: indirect register offset
1094 * @reg_data: indirect register data
1095 *
1096 */
1097void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
1098 u32 reg_addr, u32 reg_data)
1099{
1100 unsigned long flags, pcie_index, pcie_data;
1101 void __iomem *pcie_index_offset;
1102 void __iomem *pcie_data_offset;
1103
1104 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1105 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1106
1107 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1108 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1109 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1110
1111 writel(reg_addr, pcie_index_offset);
1112 readl(pcie_index_offset);
1113 writel(reg_data, pcie_data_offset);
1114 readl(pcie_data_offset);
1115 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1116}
1117
1118void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
1119 u64 reg_addr, u32 reg_data)
1120{
1121 unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
1122 void __iomem *pcie_index_offset;
1123 void __iomem *pcie_index_hi_offset;
1124 void __iomem *pcie_data_offset;
1125
1126 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1127 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1128 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1129 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1130 else
1131 pcie_index_hi = 0;
1132
1133 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1134 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1135 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1136 if (pcie_index_hi != 0)
1137 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1138 pcie_index_hi * 4;
1139
1140 writel(reg_addr, pcie_index_offset);
1141 readl(pcie_index_offset);
1142 if (pcie_index_hi != 0) {
1143 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1144 readl(pcie_index_hi_offset);
1145 }
1146 writel(reg_data, pcie_data_offset);
1147 readl(pcie_data_offset);
1148
1149 /* clear the high bits */
1150 if (pcie_index_hi != 0) {
1151 writel(0, pcie_index_hi_offset);
1152 readl(pcie_index_hi_offset);
1153 }
1154
1155 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1156}
1157
1158/**
1159 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
1160 *
1161 * @adev: amdgpu_device pointer
1162 * @reg_addr: indirect register offset
1163 * @reg_data: indirect register data
1164 *
1165 */
1166void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
1167 u32 reg_addr, u64 reg_data)
1168{
1169 unsigned long flags, pcie_index, pcie_data;
1170 void __iomem *pcie_index_offset;
1171 void __iomem *pcie_data_offset;
1172
1173 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1174 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1175
1176 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1177 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1178 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1179
1180 /* write low 32 bits */
1181 writel(reg_addr, pcie_index_offset);
1182 readl(pcie_index_offset);
1183 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1184 readl(pcie_data_offset);
1185 /* write high 32 bits */
1186 writel(reg_addr + 4, pcie_index_offset);
1187 readl(pcie_index_offset);
1188 writel((u32)(reg_data >> 32), pcie_data_offset);
1189 readl(pcie_data_offset);
1190 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1191}
1192
1193void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1194 u64 reg_addr, u64 reg_data)
1195{
1196 unsigned long flags, pcie_index, pcie_data;
1197 unsigned long pcie_index_hi = 0;
1198 void __iomem *pcie_index_offset;
1199 void __iomem *pcie_index_hi_offset;
1200 void __iomem *pcie_data_offset;
1201
1202 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1203 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1204 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1205 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1206
1207 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1208 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1209 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1210 if (pcie_index_hi != 0)
1211 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1212 pcie_index_hi * 4;
1213
1214 /* write low 32 bits */
1215 writel(reg_addr, pcie_index_offset);
1216 readl(pcie_index_offset);
1217 if (pcie_index_hi != 0) {
1218 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1219 readl(pcie_index_hi_offset);
1220 }
1221 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1222 readl(pcie_data_offset);
1223 /* write high 32 bits */
1224 writel(reg_addr + 4, pcie_index_offset);
1225 readl(pcie_index_offset);
1226 if (pcie_index_hi != 0) {
1227 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1228 readl(pcie_index_hi_offset);
1229 }
1230 writel((u32)(reg_data >> 32), pcie_data_offset);
1231 readl(pcie_data_offset);
1232
1233 /* clear the high bits */
1234 if (pcie_index_hi != 0) {
1235 writel(0, pcie_index_hi_offset);
1236 readl(pcie_index_hi_offset);
1237 }
1238
1239 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1240}
1241
1242/**
1243 * amdgpu_device_get_rev_id - query device rev_id
1244 *
1245 * @adev: amdgpu_device pointer
1246 *
1247 * Return device rev_id
1248 */
1249u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1250{
1251 return adev->nbio.funcs->get_rev_id(adev);
1252}
1253
1254/**
1255 * amdgpu_invalid_rreg - dummy reg read function
1256 *
1257 * @adev: amdgpu_device pointer
1258 * @reg: offset of register
1259 *
1260 * Dummy register read function. Used for register blocks
1261 * that certain asics don't have (all asics).
1262 * Returns the value in the register.
1263 */
1264static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1265{
1266 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1267 BUG();
1268 return 0;
1269}
1270
1271static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1272{
1273 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1274 BUG();
1275 return 0;
1276}
1277
1278/**
1279 * amdgpu_invalid_wreg - dummy reg write function
1280 *
1281 * @adev: amdgpu_device pointer
1282 * @reg: offset of register
1283 * @v: value to write to the register
1284 *
1285 * Dummy register read function. Used for register blocks
1286 * that certain asics don't have (all asics).
1287 */
1288static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1289{
1290 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1291 reg, v);
1292 BUG();
1293}
1294
1295static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1296{
1297 DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1298 reg, v);
1299 BUG();
1300}
1301
1302/**
1303 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1304 *
1305 * @adev: amdgpu_device pointer
1306 * @reg: offset of register
1307 *
1308 * Dummy register read function. Used for register blocks
1309 * that certain asics don't have (all asics).
1310 * Returns the value in the register.
1311 */
1312static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1313{
1314 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1315 BUG();
1316 return 0;
1317}
1318
1319static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1320{
1321 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1322 BUG();
1323 return 0;
1324}
1325
1326/**
1327 * amdgpu_invalid_wreg64 - dummy reg write function
1328 *
1329 * @adev: amdgpu_device pointer
1330 * @reg: offset of register
1331 * @v: value to write to the register
1332 *
1333 * Dummy register read function. Used for register blocks
1334 * that certain asics don't have (all asics).
1335 */
1336static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1337{
1338 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1339 reg, v);
1340 BUG();
1341}
1342
1343static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1344{
1345 DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1346 reg, v);
1347 BUG();
1348}
1349
1350/**
1351 * amdgpu_block_invalid_rreg - dummy reg read function
1352 *
1353 * @adev: amdgpu_device pointer
1354 * @block: offset of instance
1355 * @reg: offset of register
1356 *
1357 * Dummy register read function. Used for register blocks
1358 * that certain asics don't have (all asics).
1359 * Returns the value in the register.
1360 */
1361static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1362 uint32_t block, uint32_t reg)
1363{
1364 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1365 reg, block);
1366 BUG();
1367 return 0;
1368}
1369
1370/**
1371 * amdgpu_block_invalid_wreg - dummy reg write function
1372 *
1373 * @adev: amdgpu_device pointer
1374 * @block: offset of instance
1375 * @reg: offset of register
1376 * @v: value to write to the register
1377 *
1378 * Dummy register read function. Used for register blocks
1379 * that certain asics don't have (all asics).
1380 */
1381static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1382 uint32_t block,
1383 uint32_t reg, uint32_t v)
1384{
1385 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1386 reg, block, v);
1387 BUG();
1388}
1389
1390/**
1391 * amdgpu_device_asic_init - Wrapper for atom asic_init
1392 *
1393 * @adev: amdgpu_device pointer
1394 *
1395 * Does any asic specific work and then calls atom asic init.
1396 */
1397static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1398{
1399 int ret;
1400
1401 amdgpu_asic_pre_asic_init(adev);
1402
1403 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1404 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
1405 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1406 amdgpu_psp_wait_for_bootloader(adev);
1407 ret = amdgpu_atomfirmware_asic_init(adev, true);
1408 return ret;
1409 } else {
1410 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1411 }
1412
1413 return 0;
1414}
1415
1416/**
1417 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1418 *
1419 * @adev: amdgpu_device pointer
1420 *
1421 * Allocates a scratch page of VRAM for use by various things in the
1422 * driver.
1423 */
1424static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1425{
1426 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1427 AMDGPU_GEM_DOMAIN_VRAM |
1428 AMDGPU_GEM_DOMAIN_GTT,
1429 &adev->mem_scratch.robj,
1430 &adev->mem_scratch.gpu_addr,
1431 (void **)&adev->mem_scratch.ptr);
1432}
1433
1434/**
1435 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1436 *
1437 * @adev: amdgpu_device pointer
1438 *
1439 * Frees the VRAM scratch page.
1440 */
1441static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1442{
1443 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1444}
1445
1446/**
1447 * amdgpu_device_program_register_sequence - program an array of registers.
1448 *
1449 * @adev: amdgpu_device pointer
1450 * @registers: pointer to the register array
1451 * @array_size: size of the register array
1452 *
1453 * Programs an array or registers with and or masks.
1454 * This is a helper for setting golden registers.
1455 */
1456void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1457 const u32 *registers,
1458 const u32 array_size)
1459{
1460 u32 tmp, reg, and_mask, or_mask;
1461 int i;
1462
1463 if (array_size % 3)
1464 return;
1465
1466 for (i = 0; i < array_size; i += 3) {
1467 reg = registers[i + 0];
1468 and_mask = registers[i + 1];
1469 or_mask = registers[i + 2];
1470
1471 if (and_mask == 0xffffffff) {
1472 tmp = or_mask;
1473 } else {
1474 tmp = RREG32(reg);
1475 tmp &= ~and_mask;
1476 if (adev->family >= AMDGPU_FAMILY_AI)
1477 tmp |= (or_mask & and_mask);
1478 else
1479 tmp |= or_mask;
1480 }
1481 WREG32(reg, tmp);
1482 }
1483}
1484
1485/**
1486 * amdgpu_device_pci_config_reset - reset the GPU
1487 *
1488 * @adev: amdgpu_device pointer
1489 *
1490 * Resets the GPU using the pci config reset sequence.
1491 * Only applicable to asics prior to vega10.
1492 */
1493void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1494{
1495 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1496}
1497
1498/**
1499 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1500 *
1501 * @adev: amdgpu_device pointer
1502 *
1503 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1504 */
1505int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1506{
1507 return pci_reset_function(adev->pdev);
1508}
1509
1510/*
1511 * amdgpu_device_wb_*()
1512 * Writeback is the method by which the GPU updates special pages in memory
1513 * with the status of certain GPU events (fences, ring pointers,etc.).
1514 */
1515
1516/**
1517 * amdgpu_device_wb_fini - Disable Writeback and free memory
1518 *
1519 * @adev: amdgpu_device pointer
1520 *
1521 * Disables Writeback and frees the Writeback memory (all asics).
1522 * Used at driver shutdown.
1523 */
1524static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1525{
1526 if (adev->wb.wb_obj) {
1527 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1528 &adev->wb.gpu_addr,
1529 (void **)&adev->wb.wb);
1530 adev->wb.wb_obj = NULL;
1531 }
1532}
1533
1534/**
1535 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1536 *
1537 * @adev: amdgpu_device pointer
1538 *
1539 * Initializes writeback and allocates writeback memory (all asics).
1540 * Used at driver startup.
1541 * Returns 0 on success or an -error on failure.
1542 */
1543static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1544{
1545 int r;
1546
1547 if (adev->wb.wb_obj == NULL) {
1548 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1549 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1550 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1551 &adev->wb.wb_obj, &adev->wb.gpu_addr,
1552 (void **)&adev->wb.wb);
1553 if (r) {
1554 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1555 return r;
1556 }
1557
1558 adev->wb.num_wb = AMDGPU_MAX_WB;
1559 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1560
1561 /* clear wb memory */
1562 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1563 }
1564
1565 return 0;
1566}
1567
1568/**
1569 * amdgpu_device_wb_get - Allocate a wb entry
1570 *
1571 * @adev: amdgpu_device pointer
1572 * @wb: wb index
1573 *
1574 * Allocate a wb slot for use by the driver (all asics).
1575 * Returns 0 on success or -EINVAL on failure.
1576 */
1577int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1578{
1579 unsigned long flags, offset;
1580
1581 spin_lock_irqsave(&adev->wb.lock, flags);
1582 offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1583 if (offset < adev->wb.num_wb) {
1584 __set_bit(offset, adev->wb.used);
1585 spin_unlock_irqrestore(&adev->wb.lock, flags);
1586 *wb = offset << 3; /* convert to dw offset */
1587 return 0;
1588 } else {
1589 spin_unlock_irqrestore(&adev->wb.lock, flags);
1590 return -EINVAL;
1591 }
1592}
1593
1594/**
1595 * amdgpu_device_wb_free - Free a wb entry
1596 *
1597 * @adev: amdgpu_device pointer
1598 * @wb: wb index
1599 *
1600 * Free a wb slot allocated for use by the driver (all asics)
1601 */
1602void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1603{
1604 unsigned long flags;
1605
1606 wb >>= 3;
1607 spin_lock_irqsave(&adev->wb.lock, flags);
1608 if (wb < adev->wb.num_wb)
1609 __clear_bit(wb, adev->wb.used);
1610 spin_unlock_irqrestore(&adev->wb.lock, flags);
1611}
1612
1613/**
1614 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1615 *
1616 * @adev: amdgpu_device pointer
1617 *
1618 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1619 * to fail, but if any of the BARs is not accessible after the size we abort
1620 * driver loading by returning -ENODEV.
1621 */
1622int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1623{
1624 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1625 struct pci_bus *root;
1626 struct resource *res;
1627 unsigned int i;
1628 u16 cmd;
1629 int r;
1630
1631 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1632 return 0;
1633
1634 /* Bypass for VF */
1635 if (amdgpu_sriov_vf(adev))
1636 return 0;
1637
1638 /* resizing on Dell G5 SE platforms causes problems with runtime pm */
1639 if ((amdgpu_runtime_pm != 0) &&
1640 adev->pdev->vendor == PCI_VENDOR_ID_ATI &&
1641 adev->pdev->device == 0x731f &&
1642 adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
1643 return 0;
1644
1645 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1646 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1647 DRM_WARN("System can't access extended configuration space, please check!!\n");
1648
1649 /* skip if the bios has already enabled large BAR */
1650 if (adev->gmc.real_vram_size &&
1651 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1652 return 0;
1653
1654 /* Check if the root BUS has 64bit memory resources */
1655 root = adev->pdev->bus;
1656 while (root->parent)
1657 root = root->parent;
1658
1659 pci_bus_for_each_resource(root, res, i) {
1660 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1661 res->start > 0x100000000ull)
1662 break;
1663 }
1664
1665 /* Trying to resize is pointless without a root hub window above 4GB */
1666 if (!res)
1667 return 0;
1668
1669 /* Limit the BAR size to what is available */
1670 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1671 rbar_size);
1672
1673 /* Disable memory decoding while we change the BAR addresses and size */
1674 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1675 pci_write_config_word(adev->pdev, PCI_COMMAND,
1676 cmd & ~PCI_COMMAND_MEMORY);
1677
1678 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1679 amdgpu_doorbell_fini(adev);
1680 if (adev->asic_type >= CHIP_BONAIRE)
1681 pci_release_resource(adev->pdev, 2);
1682
1683 pci_release_resource(adev->pdev, 0);
1684
1685 r = pci_resize_resource(adev->pdev, 0, rbar_size);
1686 if (r == -ENOSPC)
1687 DRM_INFO("Not enough PCI address space for a large BAR.");
1688 else if (r && r != -ENOTSUPP)
1689 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1690
1691 pci_assign_unassigned_bus_resources(adev->pdev->bus);
1692
1693 /* When the doorbell or fb BAR isn't available we have no chance of
1694 * using the device.
1695 */
1696 r = amdgpu_doorbell_init(adev);
1697 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1698 return -ENODEV;
1699
1700 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1701
1702 return 0;
1703}
1704
1705static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1706{
1707 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1708 return false;
1709
1710 return true;
1711}
1712
1713/*
1714 * GPU helpers function.
1715 */
1716/**
1717 * amdgpu_device_need_post - check if the hw need post or not
1718 *
1719 * @adev: amdgpu_device pointer
1720 *
1721 * Check if the asic has been initialized (all asics) at driver startup
1722 * or post is needed if hw reset is performed.
1723 * Returns true if need or false if not.
1724 */
1725bool amdgpu_device_need_post(struct amdgpu_device *adev)
1726{
1727 uint32_t reg;
1728
1729 if (amdgpu_sriov_vf(adev))
1730 return false;
1731
1732 if (!amdgpu_device_read_bios(adev))
1733 return false;
1734
1735 if (amdgpu_passthrough(adev)) {
1736 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1737 * some old smc fw still need driver do vPost otherwise gpu hang, while
1738 * those smc fw version above 22.15 doesn't have this flaw, so we force
1739 * vpost executed for smc version below 22.15
1740 */
1741 if (adev->asic_type == CHIP_FIJI) {
1742 int err;
1743 uint32_t fw_ver;
1744
1745 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1746 /* force vPost if error occured */
1747 if (err)
1748 return true;
1749
1750 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1751 release_firmware(adev->pm.fw);
1752 if (fw_ver < 0x00160e00)
1753 return true;
1754 }
1755 }
1756
1757 /* Don't post if we need to reset whole hive on init */
1758 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
1759 return false;
1760
1761 if (adev->has_hw_reset) {
1762 adev->has_hw_reset = false;
1763 return true;
1764 }
1765
1766 /* bios scratch used on CIK+ */
1767 if (adev->asic_type >= CHIP_BONAIRE)
1768 return amdgpu_atombios_scratch_need_asic_init(adev);
1769
1770 /* check MEM_SIZE for older asics */
1771 reg = amdgpu_asic_get_config_memsize(adev);
1772
1773 if ((reg != 0) && (reg != 0xffffffff))
1774 return false;
1775
1776 return true;
1777}
1778
1779/*
1780 * Check whether seamless boot is supported.
1781 *
1782 * So far we only support seamless boot on DCE 3.0 or later.
1783 * If users report that it works on older ASICS as well, we may
1784 * loosen this.
1785 */
1786bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1787{
1788 switch (amdgpu_seamless) {
1789 case -1:
1790 break;
1791 case 1:
1792 return true;
1793 case 0:
1794 return false;
1795 default:
1796 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1797 amdgpu_seamless);
1798 return false;
1799 }
1800
1801 if (!(adev->flags & AMD_IS_APU))
1802 return false;
1803
1804 if (adev->mman.keep_stolen_vga_memory)
1805 return false;
1806
1807 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1808}
1809
1810/*
1811 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1812 * don't support dynamic speed switching. Until we have confirmation from Intel
1813 * that a specific host supports it, it's safer that we keep it disabled for all.
1814 *
1815 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1816 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1817 */
1818static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1819{
1820#if IS_ENABLED(CONFIG_X86)
1821 struct cpuinfo_x86 *c = &cpu_data(0);
1822
1823 /* eGPU change speeds based on USB4 fabric conditions */
1824 if (dev_is_removable(adev->dev))
1825 return true;
1826
1827 if (c->x86_vendor == X86_VENDOR_INTEL)
1828 return false;
1829#endif
1830 return true;
1831}
1832
1833/**
1834 * amdgpu_device_should_use_aspm - check if the device should program ASPM
1835 *
1836 * @adev: amdgpu_device pointer
1837 *
1838 * Confirm whether the module parameter and pcie bridge agree that ASPM should
1839 * be set for this device.
1840 *
1841 * Returns true if it should be used or false if not.
1842 */
1843bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1844{
1845 switch (amdgpu_aspm) {
1846 case -1:
1847 break;
1848 case 0:
1849 return false;
1850 case 1:
1851 return true;
1852 default:
1853 return false;
1854 }
1855 if (adev->flags & AMD_IS_APU)
1856 return false;
1857 if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1858 return false;
1859 return pcie_aspm_enabled(adev->pdev);
1860}
1861
1862/* if we get transitioned to only one device, take VGA back */
1863/**
1864 * amdgpu_device_vga_set_decode - enable/disable vga decode
1865 *
1866 * @pdev: PCI device pointer
1867 * @state: enable/disable vga decode
1868 *
1869 * Enable/disable vga decode (all asics).
1870 * Returns VGA resource flags.
1871 */
1872static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1873 bool state)
1874{
1875 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1876
1877 amdgpu_asic_set_vga_state(adev, state);
1878 if (state)
1879 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1880 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1881 else
1882 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1883}
1884
1885/**
1886 * amdgpu_device_check_block_size - validate the vm block size
1887 *
1888 * @adev: amdgpu_device pointer
1889 *
1890 * Validates the vm block size specified via module parameter.
1891 * The vm block size defines number of bits in page table versus page directory,
1892 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1893 * page table and the remaining bits are in the page directory.
1894 */
1895static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1896{
1897 /* defines number of bits in page table versus page directory,
1898 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1899 * page table and the remaining bits are in the page directory
1900 */
1901 if (amdgpu_vm_block_size == -1)
1902 return;
1903
1904 if (amdgpu_vm_block_size < 9) {
1905 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1906 amdgpu_vm_block_size);
1907 amdgpu_vm_block_size = -1;
1908 }
1909}
1910
1911/**
1912 * amdgpu_device_check_vm_size - validate the vm size
1913 *
1914 * @adev: amdgpu_device pointer
1915 *
1916 * Validates the vm size in GB specified via module parameter.
1917 * The VM size is the size of the GPU virtual memory space in GB.
1918 */
1919static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1920{
1921 /* no need to check the default value */
1922 if (amdgpu_vm_size == -1)
1923 return;
1924
1925 if (amdgpu_vm_size < 1) {
1926 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1927 amdgpu_vm_size);
1928 amdgpu_vm_size = -1;
1929 }
1930}
1931
1932static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1933{
1934 struct sysinfo si;
1935 bool is_os_64 = (sizeof(void *) == 8);
1936 uint64_t total_memory;
1937 uint64_t dram_size_seven_GB = 0x1B8000000;
1938 uint64_t dram_size_three_GB = 0xB8000000;
1939
1940 if (amdgpu_smu_memory_pool_size == 0)
1941 return;
1942
1943 if (!is_os_64) {
1944 DRM_WARN("Not 64-bit OS, feature not supported\n");
1945 goto def_value;
1946 }
1947 si_meminfo(&si);
1948 total_memory = (uint64_t)si.totalram * si.mem_unit;
1949
1950 if ((amdgpu_smu_memory_pool_size == 1) ||
1951 (amdgpu_smu_memory_pool_size == 2)) {
1952 if (total_memory < dram_size_three_GB)
1953 goto def_value1;
1954 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1955 (amdgpu_smu_memory_pool_size == 8)) {
1956 if (total_memory < dram_size_seven_GB)
1957 goto def_value1;
1958 } else {
1959 DRM_WARN("Smu memory pool size not supported\n");
1960 goto def_value;
1961 }
1962 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1963
1964 return;
1965
1966def_value1:
1967 DRM_WARN("No enough system memory\n");
1968def_value:
1969 adev->pm.smu_prv_buffer_size = 0;
1970}
1971
1972static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1973{
1974 if (!(adev->flags & AMD_IS_APU) ||
1975 adev->asic_type < CHIP_RAVEN)
1976 return 0;
1977
1978 switch (adev->asic_type) {
1979 case CHIP_RAVEN:
1980 if (adev->pdev->device == 0x15dd)
1981 adev->apu_flags |= AMD_APU_IS_RAVEN;
1982 if (adev->pdev->device == 0x15d8)
1983 adev->apu_flags |= AMD_APU_IS_PICASSO;
1984 break;
1985 case CHIP_RENOIR:
1986 if ((adev->pdev->device == 0x1636) ||
1987 (adev->pdev->device == 0x164c))
1988 adev->apu_flags |= AMD_APU_IS_RENOIR;
1989 else
1990 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1991 break;
1992 case CHIP_VANGOGH:
1993 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1994 break;
1995 case CHIP_YELLOW_CARP:
1996 break;
1997 case CHIP_CYAN_SKILLFISH:
1998 if ((adev->pdev->device == 0x13FE) ||
1999 (adev->pdev->device == 0x143F))
2000 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
2001 break;
2002 default:
2003 break;
2004 }
2005
2006 return 0;
2007}
2008
2009/**
2010 * amdgpu_device_check_arguments - validate module params
2011 *
2012 * @adev: amdgpu_device pointer
2013 *
2014 * Validates certain module parameters and updates
2015 * the associated values used by the driver (all asics).
2016 */
2017static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
2018{
2019 int i;
2020
2021 if (amdgpu_sched_jobs < 4) {
2022 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
2023 amdgpu_sched_jobs);
2024 amdgpu_sched_jobs = 4;
2025 } else if (!is_power_of_2(amdgpu_sched_jobs)) {
2026 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
2027 amdgpu_sched_jobs);
2028 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
2029 }
2030
2031 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
2032 /* gart size must be greater or equal to 32M */
2033 dev_warn(adev->dev, "gart size (%d) too small\n",
2034 amdgpu_gart_size);
2035 amdgpu_gart_size = -1;
2036 }
2037
2038 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
2039 /* gtt size must be greater or equal to 32M */
2040 dev_warn(adev->dev, "gtt size (%d) too small\n",
2041 amdgpu_gtt_size);
2042 amdgpu_gtt_size = -1;
2043 }
2044
2045 /* valid range is between 4 and 9 inclusive */
2046 if (amdgpu_vm_fragment_size != -1 &&
2047 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
2048 dev_warn(adev->dev, "valid range is between 4 and 9\n");
2049 amdgpu_vm_fragment_size = -1;
2050 }
2051
2052 if (amdgpu_sched_hw_submission < 2) {
2053 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
2054 amdgpu_sched_hw_submission);
2055 amdgpu_sched_hw_submission = 2;
2056 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
2057 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
2058 amdgpu_sched_hw_submission);
2059 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
2060 }
2061
2062 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
2063 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
2064 amdgpu_reset_method = -1;
2065 }
2066
2067 amdgpu_device_check_smu_prv_buffer_size(adev);
2068
2069 amdgpu_device_check_vm_size(adev);
2070
2071 amdgpu_device_check_block_size(adev);
2072
2073 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
2074
2075 for (i = 0; i < MAX_XCP; i++)
2076 adev->enforce_isolation[i] = !!enforce_isolation;
2077
2078 return 0;
2079}
2080
2081/**
2082 * amdgpu_switcheroo_set_state - set switcheroo state
2083 *
2084 * @pdev: pci dev pointer
2085 * @state: vga_switcheroo state
2086 *
2087 * Callback for the switcheroo driver. Suspends or resumes
2088 * the asics before or after it is powered up using ACPI methods.
2089 */
2090static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
2091 enum vga_switcheroo_state state)
2092{
2093 struct drm_device *dev = pci_get_drvdata(pdev);
2094 int r;
2095
2096 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
2097 return;
2098
2099 if (state == VGA_SWITCHEROO_ON) {
2100 pr_info("switched on\n");
2101 /* don't suspend or resume card normally */
2102 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
2103
2104 pci_set_power_state(pdev, PCI_D0);
2105 amdgpu_device_load_pci_state(pdev);
2106 r = pci_enable_device(pdev);
2107 if (r)
2108 DRM_WARN("pci_enable_device failed (%d)\n", r);
2109 amdgpu_device_resume(dev, true);
2110
2111 dev->switch_power_state = DRM_SWITCH_POWER_ON;
2112 } else {
2113 pr_info("switched off\n");
2114 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
2115 amdgpu_device_prepare(dev);
2116 amdgpu_device_suspend(dev, true);
2117 amdgpu_device_cache_pci_state(pdev);
2118 /* Shut down the device */
2119 pci_disable_device(pdev);
2120 pci_set_power_state(pdev, PCI_D3cold);
2121 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
2122 }
2123}
2124
2125/**
2126 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
2127 *
2128 * @pdev: pci dev pointer
2129 *
2130 * Callback for the switcheroo driver. Check of the switcheroo
2131 * state can be changed.
2132 * Returns true if the state can be changed, false if not.
2133 */
2134static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
2135{
2136 struct drm_device *dev = pci_get_drvdata(pdev);
2137
2138 /*
2139 * FIXME: open_count is protected by drm_global_mutex but that would lead to
2140 * locking inversion with the driver load path. And the access here is
2141 * completely racy anyway. So don't bother with locking for now.
2142 */
2143 return atomic_read(&dev->open_count) == 0;
2144}
2145
2146static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
2147 .set_gpu_state = amdgpu_switcheroo_set_state,
2148 .reprobe = NULL,
2149 .can_switch = amdgpu_switcheroo_can_switch,
2150};
2151
2152/**
2153 * amdgpu_device_ip_set_clockgating_state - set the CG state
2154 *
2155 * @dev: amdgpu_device pointer
2156 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2157 * @state: clockgating state (gate or ungate)
2158 *
2159 * Sets the requested clockgating state for all instances of
2160 * the hardware IP specified.
2161 * Returns the error code from the last instance.
2162 */
2163int amdgpu_device_ip_set_clockgating_state(void *dev,
2164 enum amd_ip_block_type block_type,
2165 enum amd_clockgating_state state)
2166{
2167 struct amdgpu_device *adev = dev;
2168 int i, r = 0;
2169
2170 for (i = 0; i < adev->num_ip_blocks; i++) {
2171 if (!adev->ip_blocks[i].status.valid)
2172 continue;
2173 if (adev->ip_blocks[i].version->type != block_type)
2174 continue;
2175 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
2176 continue;
2177 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
2178 (void *)adev, state);
2179 if (r)
2180 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
2181 adev->ip_blocks[i].version->funcs->name, r);
2182 }
2183 return r;
2184}
2185
2186/**
2187 * amdgpu_device_ip_set_powergating_state - set the PG state
2188 *
2189 * @dev: amdgpu_device pointer
2190 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2191 * @state: powergating state (gate or ungate)
2192 *
2193 * Sets the requested powergating state for all instances of
2194 * the hardware IP specified.
2195 * Returns the error code from the last instance.
2196 */
2197int amdgpu_device_ip_set_powergating_state(void *dev,
2198 enum amd_ip_block_type block_type,
2199 enum amd_powergating_state state)
2200{
2201 struct amdgpu_device *adev = dev;
2202 int i, r = 0;
2203
2204 for (i = 0; i < adev->num_ip_blocks; i++) {
2205 if (!adev->ip_blocks[i].status.valid)
2206 continue;
2207 if (adev->ip_blocks[i].version->type != block_type)
2208 continue;
2209 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2210 continue;
2211 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2212 (void *)adev, state);
2213 if (r)
2214 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2215 adev->ip_blocks[i].version->funcs->name, r);
2216 }
2217 return r;
2218}
2219
2220/**
2221 * amdgpu_device_ip_get_clockgating_state - get the CG state
2222 *
2223 * @adev: amdgpu_device pointer
2224 * @flags: clockgating feature flags
2225 *
2226 * Walks the list of IPs on the device and updates the clockgating
2227 * flags for each IP.
2228 * Updates @flags with the feature flags for each hardware IP where
2229 * clockgating is enabled.
2230 */
2231void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2232 u64 *flags)
2233{
2234 int i;
2235
2236 for (i = 0; i < adev->num_ip_blocks; i++) {
2237 if (!adev->ip_blocks[i].status.valid)
2238 continue;
2239 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2240 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2241 }
2242}
2243
2244/**
2245 * amdgpu_device_ip_wait_for_idle - wait for idle
2246 *
2247 * @adev: amdgpu_device pointer
2248 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2249 *
2250 * Waits for the request hardware IP to be idle.
2251 * Returns 0 for success or a negative error code on failure.
2252 */
2253int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2254 enum amd_ip_block_type block_type)
2255{
2256 int i, r;
2257
2258 for (i = 0; i < adev->num_ip_blocks; i++) {
2259 if (!adev->ip_blocks[i].status.valid)
2260 continue;
2261 if (adev->ip_blocks[i].version->type == block_type) {
2262 if (adev->ip_blocks[i].version->funcs->wait_for_idle) {
2263 r = adev->ip_blocks[i].version->funcs->wait_for_idle(
2264 &adev->ip_blocks[i]);
2265 if (r)
2266 return r;
2267 }
2268 break;
2269 }
2270 }
2271 return 0;
2272
2273}
2274
2275/**
2276 * amdgpu_device_ip_is_valid - is the hardware IP enabled
2277 *
2278 * @adev: amdgpu_device pointer
2279 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2280 *
2281 * Check if the hardware IP is enable or not.
2282 * Returns true if it the IP is enable, false if not.
2283 */
2284bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
2285 enum amd_ip_block_type block_type)
2286{
2287 int i;
2288
2289 for (i = 0; i < adev->num_ip_blocks; i++) {
2290 if (adev->ip_blocks[i].version->type == block_type)
2291 return adev->ip_blocks[i].status.valid;
2292 }
2293 return false;
2294
2295}
2296
2297/**
2298 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2299 *
2300 * @adev: amdgpu_device pointer
2301 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2302 *
2303 * Returns a pointer to the hardware IP block structure
2304 * if it exists for the asic, otherwise NULL.
2305 */
2306struct amdgpu_ip_block *
2307amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2308 enum amd_ip_block_type type)
2309{
2310 int i;
2311
2312 for (i = 0; i < adev->num_ip_blocks; i++)
2313 if (adev->ip_blocks[i].version->type == type)
2314 return &adev->ip_blocks[i];
2315
2316 return NULL;
2317}
2318
2319/**
2320 * amdgpu_device_ip_block_version_cmp
2321 *
2322 * @adev: amdgpu_device pointer
2323 * @type: enum amd_ip_block_type
2324 * @major: major version
2325 * @minor: minor version
2326 *
2327 * return 0 if equal or greater
2328 * return 1 if smaller or the ip_block doesn't exist
2329 */
2330int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2331 enum amd_ip_block_type type,
2332 u32 major, u32 minor)
2333{
2334 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2335
2336 if (ip_block && ((ip_block->version->major > major) ||
2337 ((ip_block->version->major == major) &&
2338 (ip_block->version->minor >= minor))))
2339 return 0;
2340
2341 return 1;
2342}
2343
2344/**
2345 * amdgpu_device_ip_block_add
2346 *
2347 * @adev: amdgpu_device pointer
2348 * @ip_block_version: pointer to the IP to add
2349 *
2350 * Adds the IP block driver information to the collection of IPs
2351 * on the asic.
2352 */
2353int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2354 const struct amdgpu_ip_block_version *ip_block_version)
2355{
2356 if (!ip_block_version)
2357 return -EINVAL;
2358
2359 switch (ip_block_version->type) {
2360 case AMD_IP_BLOCK_TYPE_VCN:
2361 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2362 return 0;
2363 break;
2364 case AMD_IP_BLOCK_TYPE_JPEG:
2365 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2366 return 0;
2367 break;
2368 default:
2369 break;
2370 }
2371
2372 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2373 ip_block_version->funcs->name);
2374
2375 adev->ip_blocks[adev->num_ip_blocks].adev = adev;
2376
2377 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2378
2379 return 0;
2380}
2381
2382/**
2383 * amdgpu_device_enable_virtual_display - enable virtual display feature
2384 *
2385 * @adev: amdgpu_device pointer
2386 *
2387 * Enabled the virtual display feature if the user has enabled it via
2388 * the module parameter virtual_display. This feature provides a virtual
2389 * display hardware on headless boards or in virtualized environments.
2390 * This function parses and validates the configuration string specified by
2391 * the user and configues the virtual display configuration (number of
2392 * virtual connectors, crtcs, etc.) specified.
2393 */
2394static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2395{
2396 adev->enable_virtual_display = false;
2397
2398 if (amdgpu_virtual_display) {
2399 const char *pci_address_name = pci_name(adev->pdev);
2400 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2401
2402 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2403 pciaddstr_tmp = pciaddstr;
2404 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2405 pciaddname = strsep(&pciaddname_tmp, ",");
2406 if (!strcmp("all", pciaddname)
2407 || !strcmp(pci_address_name, pciaddname)) {
2408 long num_crtc;
2409 int res = -1;
2410
2411 adev->enable_virtual_display = true;
2412
2413 if (pciaddname_tmp)
2414 res = kstrtol(pciaddname_tmp, 10,
2415 &num_crtc);
2416
2417 if (!res) {
2418 if (num_crtc < 1)
2419 num_crtc = 1;
2420 if (num_crtc > 6)
2421 num_crtc = 6;
2422 adev->mode_info.num_crtc = num_crtc;
2423 } else {
2424 adev->mode_info.num_crtc = 1;
2425 }
2426 break;
2427 }
2428 }
2429
2430 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2431 amdgpu_virtual_display, pci_address_name,
2432 adev->enable_virtual_display, adev->mode_info.num_crtc);
2433
2434 kfree(pciaddstr);
2435 }
2436}
2437
2438void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2439{
2440 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2441 adev->mode_info.num_crtc = 1;
2442 adev->enable_virtual_display = true;
2443 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2444 adev->enable_virtual_display, adev->mode_info.num_crtc);
2445 }
2446}
2447
2448/**
2449 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2450 *
2451 * @adev: amdgpu_device pointer
2452 *
2453 * Parses the asic configuration parameters specified in the gpu info
2454 * firmware and makes them availale to the driver for use in configuring
2455 * the asic.
2456 * Returns 0 on success, -EINVAL on failure.
2457 */
2458static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2459{
2460 const char *chip_name;
2461 int err;
2462 const struct gpu_info_firmware_header_v1_0 *hdr;
2463
2464 adev->firmware.gpu_info_fw = NULL;
2465
2466 if (adev->mman.discovery_bin)
2467 return 0;
2468
2469 switch (adev->asic_type) {
2470 default:
2471 return 0;
2472 case CHIP_VEGA10:
2473 chip_name = "vega10";
2474 break;
2475 case CHIP_VEGA12:
2476 chip_name = "vega12";
2477 break;
2478 case CHIP_RAVEN:
2479 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2480 chip_name = "raven2";
2481 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2482 chip_name = "picasso";
2483 else
2484 chip_name = "raven";
2485 break;
2486 case CHIP_ARCTURUS:
2487 chip_name = "arcturus";
2488 break;
2489 case CHIP_NAVI12:
2490 chip_name = "navi12";
2491 break;
2492 }
2493
2494 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw,
2495 "amdgpu/%s_gpu_info.bin", chip_name);
2496 if (err) {
2497 dev_err(adev->dev,
2498 "Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n",
2499 chip_name);
2500 goto out;
2501 }
2502
2503 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2504 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2505
2506 switch (hdr->version_major) {
2507 case 1:
2508 {
2509 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2510 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2511 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2512
2513 /*
2514 * Should be droped when DAL no longer needs it.
2515 */
2516 if (adev->asic_type == CHIP_NAVI12)
2517 goto parse_soc_bounding_box;
2518
2519 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2520 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2521 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2522 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2523 adev->gfx.config.max_texture_channel_caches =
2524 le32_to_cpu(gpu_info_fw->gc_num_tccs);
2525 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2526 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2527 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2528 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2529 adev->gfx.config.double_offchip_lds_buf =
2530 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2531 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2532 adev->gfx.cu_info.max_waves_per_simd =
2533 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2534 adev->gfx.cu_info.max_scratch_slots_per_cu =
2535 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2536 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2537 if (hdr->version_minor >= 1) {
2538 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2539 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2540 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2541 adev->gfx.config.num_sc_per_sh =
2542 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2543 adev->gfx.config.num_packer_per_sc =
2544 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2545 }
2546
2547parse_soc_bounding_box:
2548 /*
2549 * soc bounding box info is not integrated in disocovery table,
2550 * we always need to parse it from gpu info firmware if needed.
2551 */
2552 if (hdr->version_minor == 2) {
2553 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2554 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2555 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2556 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2557 }
2558 break;
2559 }
2560 default:
2561 dev_err(adev->dev,
2562 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2563 err = -EINVAL;
2564 goto out;
2565 }
2566out:
2567 return err;
2568}
2569
2570/**
2571 * amdgpu_device_ip_early_init - run early init for hardware IPs
2572 *
2573 * @adev: amdgpu_device pointer
2574 *
2575 * Early initialization pass for hardware IPs. The hardware IPs that make
2576 * up each asic are discovered each IP's early_init callback is run. This
2577 * is the first stage in initializing the asic.
2578 * Returns 0 on success, negative error code on failure.
2579 */
2580static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2581{
2582 struct amdgpu_ip_block *ip_block;
2583 struct pci_dev *parent;
2584 int i, r;
2585 bool total;
2586
2587 amdgpu_device_enable_virtual_display(adev);
2588
2589 if (amdgpu_sriov_vf(adev)) {
2590 r = amdgpu_virt_request_full_gpu(adev, true);
2591 if (r)
2592 return r;
2593 }
2594
2595 switch (adev->asic_type) {
2596#ifdef CONFIG_DRM_AMDGPU_SI
2597 case CHIP_VERDE:
2598 case CHIP_TAHITI:
2599 case CHIP_PITCAIRN:
2600 case CHIP_OLAND:
2601 case CHIP_HAINAN:
2602 adev->family = AMDGPU_FAMILY_SI;
2603 r = si_set_ip_blocks(adev);
2604 if (r)
2605 return r;
2606 break;
2607#endif
2608#ifdef CONFIG_DRM_AMDGPU_CIK
2609 case CHIP_BONAIRE:
2610 case CHIP_HAWAII:
2611 case CHIP_KAVERI:
2612 case CHIP_KABINI:
2613 case CHIP_MULLINS:
2614 if (adev->flags & AMD_IS_APU)
2615 adev->family = AMDGPU_FAMILY_KV;
2616 else
2617 adev->family = AMDGPU_FAMILY_CI;
2618
2619 r = cik_set_ip_blocks(adev);
2620 if (r)
2621 return r;
2622 break;
2623#endif
2624 case CHIP_TOPAZ:
2625 case CHIP_TONGA:
2626 case CHIP_FIJI:
2627 case CHIP_POLARIS10:
2628 case CHIP_POLARIS11:
2629 case CHIP_POLARIS12:
2630 case CHIP_VEGAM:
2631 case CHIP_CARRIZO:
2632 case CHIP_STONEY:
2633 if (adev->flags & AMD_IS_APU)
2634 adev->family = AMDGPU_FAMILY_CZ;
2635 else
2636 adev->family = AMDGPU_FAMILY_VI;
2637
2638 r = vi_set_ip_blocks(adev);
2639 if (r)
2640 return r;
2641 break;
2642 default:
2643 r = amdgpu_discovery_set_ip_blocks(adev);
2644 if (r)
2645 return r;
2646 break;
2647 }
2648
2649 if (amdgpu_has_atpx() &&
2650 (amdgpu_is_atpx_hybrid() ||
2651 amdgpu_has_atpx_dgpu_power_cntl()) &&
2652 ((adev->flags & AMD_IS_APU) == 0) &&
2653 !dev_is_removable(&adev->pdev->dev))
2654 adev->flags |= AMD_IS_PX;
2655
2656 if (!(adev->flags & AMD_IS_APU)) {
2657 parent = pcie_find_root_port(adev->pdev);
2658 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2659 }
2660
2661
2662 adev->pm.pp_feature = amdgpu_pp_feature_mask;
2663 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2664 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2665 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2666 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2667 if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2668 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2669
2670 total = true;
2671 for (i = 0; i < adev->num_ip_blocks; i++) {
2672 ip_block = &adev->ip_blocks[i];
2673
2674 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2675 DRM_WARN("disabled ip block: %d <%s>\n",
2676 i, adev->ip_blocks[i].version->funcs->name);
2677 adev->ip_blocks[i].status.valid = false;
2678 } else if (ip_block->version->funcs->early_init) {
2679 r = ip_block->version->funcs->early_init(ip_block);
2680 if (r == -ENOENT) {
2681 adev->ip_blocks[i].status.valid = false;
2682 } else if (r) {
2683 DRM_ERROR("early_init of IP block <%s> failed %d\n",
2684 adev->ip_blocks[i].version->funcs->name, r);
2685 total = false;
2686 } else {
2687 adev->ip_blocks[i].status.valid = true;
2688 }
2689 } else {
2690 adev->ip_blocks[i].status.valid = true;
2691 }
2692 /* get the vbios after the asic_funcs are set up */
2693 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2694 r = amdgpu_device_parse_gpu_info_fw(adev);
2695 if (r)
2696 return r;
2697
2698 /* Read BIOS */
2699 if (amdgpu_device_read_bios(adev)) {
2700 if (!amdgpu_get_bios(adev))
2701 return -EINVAL;
2702
2703 r = amdgpu_atombios_init(adev);
2704 if (r) {
2705 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2706 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2707 return r;
2708 }
2709 }
2710
2711 /*get pf2vf msg info at it's earliest time*/
2712 if (amdgpu_sriov_vf(adev))
2713 amdgpu_virt_init_data_exchange(adev);
2714
2715 }
2716 }
2717 if (!total)
2718 return -ENODEV;
2719
2720 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
2721 if (ip_block->status.valid != false)
2722 amdgpu_amdkfd_device_probe(adev);
2723
2724 adev->cg_flags &= amdgpu_cg_mask;
2725 adev->pg_flags &= amdgpu_pg_mask;
2726
2727 return 0;
2728}
2729
2730static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2731{
2732 int i, r;
2733
2734 for (i = 0; i < adev->num_ip_blocks; i++) {
2735 if (!adev->ip_blocks[i].status.sw)
2736 continue;
2737 if (adev->ip_blocks[i].status.hw)
2738 continue;
2739 if (!amdgpu_ip_member_of_hwini(
2740 adev, adev->ip_blocks[i].version->type))
2741 continue;
2742 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2743 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2744 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2745 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2746 if (r) {
2747 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2748 adev->ip_blocks[i].version->funcs->name, r);
2749 return r;
2750 }
2751 adev->ip_blocks[i].status.hw = true;
2752 }
2753 }
2754
2755 return 0;
2756}
2757
2758static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2759{
2760 int i, r;
2761
2762 for (i = 0; i < adev->num_ip_blocks; i++) {
2763 if (!adev->ip_blocks[i].status.sw)
2764 continue;
2765 if (adev->ip_blocks[i].status.hw)
2766 continue;
2767 if (!amdgpu_ip_member_of_hwini(
2768 adev, adev->ip_blocks[i].version->type))
2769 continue;
2770 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2771 if (r) {
2772 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2773 adev->ip_blocks[i].version->funcs->name, r);
2774 return r;
2775 }
2776 adev->ip_blocks[i].status.hw = true;
2777 }
2778
2779 return 0;
2780}
2781
2782static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2783{
2784 int r = 0;
2785 int i;
2786 uint32_t smu_version;
2787
2788 if (adev->asic_type >= CHIP_VEGA10) {
2789 for (i = 0; i < adev->num_ip_blocks; i++) {
2790 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2791 continue;
2792
2793 if (!amdgpu_ip_member_of_hwini(adev,
2794 AMD_IP_BLOCK_TYPE_PSP))
2795 break;
2796
2797 if (!adev->ip_blocks[i].status.sw)
2798 continue;
2799
2800 /* no need to do the fw loading again if already done*/
2801 if (adev->ip_blocks[i].status.hw == true)
2802 break;
2803
2804 if (amdgpu_in_reset(adev) || adev->in_suspend) {
2805 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
2806 if (r)
2807 return r;
2808 } else {
2809 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2810 if (r) {
2811 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2812 adev->ip_blocks[i].version->funcs->name, r);
2813 return r;
2814 }
2815 adev->ip_blocks[i].status.hw = true;
2816 }
2817 break;
2818 }
2819 }
2820
2821 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2822 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2823
2824 return r;
2825}
2826
2827static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2828{
2829 long timeout;
2830 int r, i;
2831
2832 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2833 struct amdgpu_ring *ring = adev->rings[i];
2834
2835 /* No need to setup the GPU scheduler for rings that don't need it */
2836 if (!ring || ring->no_scheduler)
2837 continue;
2838
2839 switch (ring->funcs->type) {
2840 case AMDGPU_RING_TYPE_GFX:
2841 timeout = adev->gfx_timeout;
2842 break;
2843 case AMDGPU_RING_TYPE_COMPUTE:
2844 timeout = adev->compute_timeout;
2845 break;
2846 case AMDGPU_RING_TYPE_SDMA:
2847 timeout = adev->sdma_timeout;
2848 break;
2849 default:
2850 timeout = adev->video_timeout;
2851 break;
2852 }
2853
2854 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
2855 DRM_SCHED_PRIORITY_COUNT,
2856 ring->num_hw_submission, 0,
2857 timeout, adev->reset_domain->wq,
2858 ring->sched_score, ring->name,
2859 adev->dev);
2860 if (r) {
2861 DRM_ERROR("Failed to create scheduler on ring %s.\n",
2862 ring->name);
2863 return r;
2864 }
2865 r = amdgpu_uvd_entity_init(adev, ring);
2866 if (r) {
2867 DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2868 ring->name);
2869 return r;
2870 }
2871 r = amdgpu_vce_entity_init(adev, ring);
2872 if (r) {
2873 DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2874 ring->name);
2875 return r;
2876 }
2877 }
2878
2879 amdgpu_xcp_update_partition_sched_list(adev);
2880
2881 return 0;
2882}
2883
2884
2885/**
2886 * amdgpu_device_ip_init - run init for hardware IPs
2887 *
2888 * @adev: amdgpu_device pointer
2889 *
2890 * Main initialization pass for hardware IPs. The list of all the hardware
2891 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2892 * are run. sw_init initializes the software state associated with each IP
2893 * and hw_init initializes the hardware associated with each IP.
2894 * Returns 0 on success, negative error code on failure.
2895 */
2896static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2897{
2898 bool init_badpage;
2899 int i, r;
2900
2901 r = amdgpu_ras_init(adev);
2902 if (r)
2903 return r;
2904
2905 for (i = 0; i < adev->num_ip_blocks; i++) {
2906 if (!adev->ip_blocks[i].status.valid)
2907 continue;
2908 if (adev->ip_blocks[i].version->funcs->sw_init) {
2909 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]);
2910 if (r) {
2911 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2912 adev->ip_blocks[i].version->funcs->name, r);
2913 goto init_failed;
2914 }
2915 }
2916 adev->ip_blocks[i].status.sw = true;
2917
2918 if (!amdgpu_ip_member_of_hwini(
2919 adev, adev->ip_blocks[i].version->type))
2920 continue;
2921
2922 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2923 /* need to do common hw init early so everything is set up for gmc */
2924 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2925 if (r) {
2926 DRM_ERROR("hw_init %d failed %d\n", i, r);
2927 goto init_failed;
2928 }
2929 adev->ip_blocks[i].status.hw = true;
2930 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2931 /* need to do gmc hw init early so we can allocate gpu mem */
2932 /* Try to reserve bad pages early */
2933 if (amdgpu_sriov_vf(adev))
2934 amdgpu_virt_exchange_data(adev);
2935
2936 r = amdgpu_device_mem_scratch_init(adev);
2937 if (r) {
2938 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2939 goto init_failed;
2940 }
2941 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2942 if (r) {
2943 DRM_ERROR("hw_init %d failed %d\n", i, r);
2944 goto init_failed;
2945 }
2946 r = amdgpu_device_wb_init(adev);
2947 if (r) {
2948 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2949 goto init_failed;
2950 }
2951 adev->ip_blocks[i].status.hw = true;
2952
2953 /* right after GMC hw init, we create CSA */
2954 if (adev->gfx.mcbp) {
2955 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2956 AMDGPU_GEM_DOMAIN_VRAM |
2957 AMDGPU_GEM_DOMAIN_GTT,
2958 AMDGPU_CSA_SIZE);
2959 if (r) {
2960 DRM_ERROR("allocate CSA failed %d\n", r);
2961 goto init_failed;
2962 }
2963 }
2964
2965 r = amdgpu_seq64_init(adev);
2966 if (r) {
2967 DRM_ERROR("allocate seq64 failed %d\n", r);
2968 goto init_failed;
2969 }
2970 }
2971 }
2972
2973 if (amdgpu_sriov_vf(adev))
2974 amdgpu_virt_init_data_exchange(adev);
2975
2976 r = amdgpu_ib_pool_init(adev);
2977 if (r) {
2978 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2979 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2980 goto init_failed;
2981 }
2982
2983 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2984 if (r)
2985 goto init_failed;
2986
2987 r = amdgpu_device_ip_hw_init_phase1(adev);
2988 if (r)
2989 goto init_failed;
2990
2991 r = amdgpu_device_fw_loading(adev);
2992 if (r)
2993 goto init_failed;
2994
2995 r = amdgpu_device_ip_hw_init_phase2(adev);
2996 if (r)
2997 goto init_failed;
2998
2999 /*
3000 * retired pages will be loaded from eeprom and reserved here,
3001 * it should be called after amdgpu_device_ip_hw_init_phase2 since
3002 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
3003 * for I2C communication which only true at this point.
3004 *
3005 * amdgpu_ras_recovery_init may fail, but the upper only cares the
3006 * failure from bad gpu situation and stop amdgpu init process
3007 * accordingly. For other failed cases, it will still release all
3008 * the resource and print error message, rather than returning one
3009 * negative value to upper level.
3010 *
3011 * Note: theoretically, this should be called before all vram allocations
3012 * to protect retired page from abusing
3013 */
3014 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
3015 r = amdgpu_ras_recovery_init(adev, init_badpage);
3016 if (r)
3017 goto init_failed;
3018
3019 /**
3020 * In case of XGMI grab extra reference for reset domain for this device
3021 */
3022 if (adev->gmc.xgmi.num_physical_nodes > 1) {
3023 if (amdgpu_xgmi_add_device(adev) == 0) {
3024 if (!amdgpu_sriov_vf(adev)) {
3025 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3026
3027 if (WARN_ON(!hive)) {
3028 r = -ENOENT;
3029 goto init_failed;
3030 }
3031
3032 if (!hive->reset_domain ||
3033 !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
3034 r = -ENOENT;
3035 amdgpu_put_xgmi_hive(hive);
3036 goto init_failed;
3037 }
3038
3039 /* Drop the early temporary reset domain we created for device */
3040 amdgpu_reset_put_reset_domain(adev->reset_domain);
3041 adev->reset_domain = hive->reset_domain;
3042 amdgpu_put_xgmi_hive(hive);
3043 }
3044 }
3045 }
3046
3047 r = amdgpu_device_init_schedulers(adev);
3048 if (r)
3049 goto init_failed;
3050
3051 if (adev->mman.buffer_funcs_ring->sched.ready)
3052 amdgpu_ttm_set_buffer_funcs_status(adev, true);
3053
3054 /* Don't init kfd if whole hive need to be reset during init */
3055 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
3056 kgd2kfd_init_zone_device(adev);
3057 amdgpu_amdkfd_device_init(adev);
3058 }
3059
3060 amdgpu_fru_get_product_info(adev);
3061
3062init_failed:
3063
3064 return r;
3065}
3066
3067/**
3068 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
3069 *
3070 * @adev: amdgpu_device pointer
3071 *
3072 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
3073 * this function before a GPU reset. If the value is retained after a
3074 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
3075 */
3076static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
3077{
3078 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
3079}
3080
3081/**
3082 * amdgpu_device_check_vram_lost - check if vram is valid
3083 *
3084 * @adev: amdgpu_device pointer
3085 *
3086 * Checks the reset magic value written to the gart pointer in VRAM.
3087 * The driver calls this after a GPU reset to see if the contents of
3088 * VRAM is lost or now.
3089 * returns true if vram is lost, false if not.
3090 */
3091static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
3092{
3093 if (memcmp(adev->gart.ptr, adev->reset_magic,
3094 AMDGPU_RESET_MAGIC_NUM))
3095 return true;
3096
3097 if (!amdgpu_in_reset(adev))
3098 return false;
3099
3100 /*
3101 * For all ASICs with baco/mode1 reset, the VRAM is
3102 * always assumed to be lost.
3103 */
3104 switch (amdgpu_asic_reset_method(adev)) {
3105 case AMD_RESET_METHOD_BACO:
3106 case AMD_RESET_METHOD_MODE1:
3107 return true;
3108 default:
3109 return false;
3110 }
3111}
3112
3113/**
3114 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
3115 *
3116 * @adev: amdgpu_device pointer
3117 * @state: clockgating state (gate or ungate)
3118 *
3119 * The list of all the hardware IPs that make up the asic is walked and the
3120 * set_clockgating_state callbacks are run.
3121 * Late initialization pass enabling clockgating for hardware IPs.
3122 * Fini or suspend, pass disabling clockgating for hardware IPs.
3123 * Returns 0 on success, negative error code on failure.
3124 */
3125
3126int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
3127 enum amd_clockgating_state state)
3128{
3129 int i, j, r;
3130
3131 if (amdgpu_emu_mode == 1)
3132 return 0;
3133
3134 for (j = 0; j < adev->num_ip_blocks; j++) {
3135 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
3136 if (!adev->ip_blocks[i].status.late_initialized)
3137 continue;
3138 /* skip CG for GFX, SDMA on S0ix */
3139 if (adev->in_s0ix &&
3140 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3141 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3142 continue;
3143 /* skip CG for VCE/UVD, it's handled specially */
3144 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
3145 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
3146 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
3147 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
3148 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
3149 /* enable clockgating to save power */
3150 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
3151 state);
3152 if (r) {
3153 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
3154 adev->ip_blocks[i].version->funcs->name, r);
3155 return r;
3156 }
3157 }
3158 }
3159
3160 return 0;
3161}
3162
3163int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
3164 enum amd_powergating_state state)
3165{
3166 int i, j, r;
3167
3168 if (amdgpu_emu_mode == 1)
3169 return 0;
3170
3171 for (j = 0; j < adev->num_ip_blocks; j++) {
3172 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
3173 if (!adev->ip_blocks[i].status.late_initialized)
3174 continue;
3175 /* skip PG for GFX, SDMA on S0ix */
3176 if (adev->in_s0ix &&
3177 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3178 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3179 continue;
3180 /* skip CG for VCE/UVD, it's handled specially */
3181 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
3182 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
3183 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
3184 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
3185 adev->ip_blocks[i].version->funcs->set_powergating_state) {
3186 /* enable powergating to save power */
3187 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
3188 state);
3189 if (r) {
3190 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
3191 adev->ip_blocks[i].version->funcs->name, r);
3192 return r;
3193 }
3194 }
3195 }
3196 return 0;
3197}
3198
3199static int amdgpu_device_enable_mgpu_fan_boost(void)
3200{
3201 struct amdgpu_gpu_instance *gpu_ins;
3202 struct amdgpu_device *adev;
3203 int i, ret = 0;
3204
3205 mutex_lock(&mgpu_info.mutex);
3206
3207 /*
3208 * MGPU fan boost feature should be enabled
3209 * only when there are two or more dGPUs in
3210 * the system
3211 */
3212 if (mgpu_info.num_dgpu < 2)
3213 goto out;
3214
3215 for (i = 0; i < mgpu_info.num_dgpu; i++) {
3216 gpu_ins = &(mgpu_info.gpu_ins[i]);
3217 adev = gpu_ins->adev;
3218 if (!(adev->flags & AMD_IS_APU) &&
3219 !gpu_ins->mgpu_fan_enabled) {
3220 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3221 if (ret)
3222 break;
3223
3224 gpu_ins->mgpu_fan_enabled = 1;
3225 }
3226 }
3227
3228out:
3229 mutex_unlock(&mgpu_info.mutex);
3230
3231 return ret;
3232}
3233
3234/**
3235 * amdgpu_device_ip_late_init - run late init for hardware IPs
3236 *
3237 * @adev: amdgpu_device pointer
3238 *
3239 * Late initialization pass for hardware IPs. The list of all the hardware
3240 * IPs that make up the asic is walked and the late_init callbacks are run.
3241 * late_init covers any special initialization that an IP requires
3242 * after all of the have been initialized or something that needs to happen
3243 * late in the init process.
3244 * Returns 0 on success, negative error code on failure.
3245 */
3246static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3247{
3248 struct amdgpu_gpu_instance *gpu_instance;
3249 int i = 0, r;
3250
3251 for (i = 0; i < adev->num_ip_blocks; i++) {
3252 if (!adev->ip_blocks[i].status.hw)
3253 continue;
3254 if (adev->ip_blocks[i].version->funcs->late_init) {
3255 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);
3256 if (r) {
3257 DRM_ERROR("late_init of IP block <%s> failed %d\n",
3258 adev->ip_blocks[i].version->funcs->name, r);
3259 return r;
3260 }
3261 }
3262 adev->ip_blocks[i].status.late_initialized = true;
3263 }
3264
3265 r = amdgpu_ras_late_init(adev);
3266 if (r) {
3267 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3268 return r;
3269 }
3270
3271 if (!amdgpu_reset_in_recovery(adev))
3272 amdgpu_ras_set_error_query_ready(adev, true);
3273
3274 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3275 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3276
3277 amdgpu_device_fill_reset_magic(adev);
3278
3279 r = amdgpu_device_enable_mgpu_fan_boost();
3280 if (r)
3281 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3282
3283 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3284 if (amdgpu_passthrough(adev) &&
3285 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3286 adev->asic_type == CHIP_ALDEBARAN))
3287 amdgpu_dpm_handle_passthrough_sbr(adev, true);
3288
3289 if (adev->gmc.xgmi.num_physical_nodes > 1) {
3290 mutex_lock(&mgpu_info.mutex);
3291
3292 /*
3293 * Reset device p-state to low as this was booted with high.
3294 *
3295 * This should be performed only after all devices from the same
3296 * hive get initialized.
3297 *
3298 * However, it's unknown how many device in the hive in advance.
3299 * As this is counted one by one during devices initializations.
3300 *
3301 * So, we wait for all XGMI interlinked devices initialized.
3302 * This may bring some delays as those devices may come from
3303 * different hives. But that should be OK.
3304 */
3305 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3306 for (i = 0; i < mgpu_info.num_gpu; i++) {
3307 gpu_instance = &(mgpu_info.gpu_ins[i]);
3308 if (gpu_instance->adev->flags & AMD_IS_APU)
3309 continue;
3310
3311 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3312 AMDGPU_XGMI_PSTATE_MIN);
3313 if (r) {
3314 DRM_ERROR("pstate setting failed (%d).\n", r);
3315 break;
3316 }
3317 }
3318 }
3319
3320 mutex_unlock(&mgpu_info.mutex);
3321 }
3322
3323 return 0;
3324}
3325
3326static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block)
3327{
3328 int r;
3329
3330 if (!ip_block->version->funcs->hw_fini) {
3331 DRM_ERROR("hw_fini of IP block <%s> not defined\n",
3332 ip_block->version->funcs->name);
3333 } else {
3334 r = ip_block->version->funcs->hw_fini(ip_block);
3335 /* XXX handle errors */
3336 if (r) {
3337 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3338 ip_block->version->funcs->name, r);
3339 }
3340 }
3341
3342 ip_block->status.hw = false;
3343}
3344
3345/**
3346 * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3347 *
3348 * @adev: amdgpu_device pointer
3349 *
3350 * For ASICs need to disable SMC first
3351 */
3352static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3353{
3354 int i;
3355
3356 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3357 return;
3358
3359 for (i = 0; i < adev->num_ip_blocks; i++) {
3360 if (!adev->ip_blocks[i].status.hw)
3361 continue;
3362 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3363 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
3364 break;
3365 }
3366 }
3367}
3368
3369static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3370{
3371 int i, r;
3372
3373 for (i = 0; i < adev->num_ip_blocks; i++) {
3374 if (!adev->ip_blocks[i].version->funcs->early_fini)
3375 continue;
3376
3377 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]);
3378 if (r) {
3379 DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3380 adev->ip_blocks[i].version->funcs->name, r);
3381 }
3382 }
3383
3384 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3385 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3386
3387 amdgpu_amdkfd_suspend(adev, false);
3388
3389 /* Workaroud for ASICs need to disable SMC first */
3390 amdgpu_device_smu_fini_early(adev);
3391
3392 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3393 if (!adev->ip_blocks[i].status.hw)
3394 continue;
3395
3396 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
3397 }
3398
3399 if (amdgpu_sriov_vf(adev)) {
3400 if (amdgpu_virt_release_full_gpu(adev, false))
3401 DRM_ERROR("failed to release exclusive mode on fini\n");
3402 }
3403
3404 return 0;
3405}
3406
3407/**
3408 * amdgpu_device_ip_fini - run fini for hardware IPs
3409 *
3410 * @adev: amdgpu_device pointer
3411 *
3412 * Main teardown pass for hardware IPs. The list of all the hardware
3413 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3414 * are run. hw_fini tears down the hardware associated with each IP
3415 * and sw_fini tears down any software state associated with each IP.
3416 * Returns 0 on success, negative error code on failure.
3417 */
3418static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3419{
3420 int i, r;
3421
3422 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3423 amdgpu_virt_release_ras_err_handler_data(adev);
3424
3425 if (adev->gmc.xgmi.num_physical_nodes > 1)
3426 amdgpu_xgmi_remove_device(adev);
3427
3428 amdgpu_amdkfd_device_fini_sw(adev);
3429
3430 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3431 if (!adev->ip_blocks[i].status.sw)
3432 continue;
3433
3434 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3435 amdgpu_ucode_free_bo(adev);
3436 amdgpu_free_static_csa(&adev->virt.csa_obj);
3437 amdgpu_device_wb_fini(adev);
3438 amdgpu_device_mem_scratch_fini(adev);
3439 amdgpu_ib_pool_fini(adev);
3440 amdgpu_seq64_fini(adev);
3441 }
3442 if (adev->ip_blocks[i].version->funcs->sw_fini) {
3443 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]);
3444 /* XXX handle errors */
3445 if (r) {
3446 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3447 adev->ip_blocks[i].version->funcs->name, r);
3448 }
3449 }
3450 adev->ip_blocks[i].status.sw = false;
3451 adev->ip_blocks[i].status.valid = false;
3452 }
3453
3454 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3455 if (!adev->ip_blocks[i].status.late_initialized)
3456 continue;
3457 if (adev->ip_blocks[i].version->funcs->late_fini)
3458 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]);
3459 adev->ip_blocks[i].status.late_initialized = false;
3460 }
3461
3462 amdgpu_ras_fini(adev);
3463
3464 return 0;
3465}
3466
3467/**
3468 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3469 *
3470 * @work: work_struct.
3471 */
3472static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3473{
3474 struct amdgpu_device *adev =
3475 container_of(work, struct amdgpu_device, delayed_init_work.work);
3476 int r;
3477
3478 r = amdgpu_ib_ring_tests(adev);
3479 if (r)
3480 DRM_ERROR("ib ring test failed (%d).\n", r);
3481}
3482
3483static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3484{
3485 struct amdgpu_device *adev =
3486 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3487
3488 WARN_ON_ONCE(adev->gfx.gfx_off_state);
3489 WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3490
3491 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3492 adev->gfx.gfx_off_state = true;
3493}
3494
3495/**
3496 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3497 *
3498 * @adev: amdgpu_device pointer
3499 *
3500 * Main suspend function for hardware IPs. The list of all the hardware
3501 * IPs that make up the asic is walked, clockgating is disabled and the
3502 * suspend callbacks are run. suspend puts the hardware and software state
3503 * in each IP into a state suitable for suspend.
3504 * Returns 0 on success, negative error code on failure.
3505 */
3506static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3507{
3508 int i, r;
3509
3510 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3511 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3512
3513 /*
3514 * Per PMFW team's suggestion, driver needs to handle gfxoff
3515 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3516 * scenario. Add the missing df cstate disablement here.
3517 */
3518 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3519 dev_warn(adev->dev, "Failed to disallow df cstate");
3520
3521 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3522 if (!adev->ip_blocks[i].status.valid)
3523 continue;
3524
3525 /* displays are handled separately */
3526 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3527 continue;
3528
3529 /* XXX handle errors */
3530 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3531 if (r)
3532 return r;
3533 }
3534
3535 return 0;
3536}
3537
3538/**
3539 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3540 *
3541 * @adev: amdgpu_device pointer
3542 *
3543 * Main suspend function for hardware IPs. The list of all the hardware
3544 * IPs that make up the asic is walked, clockgating is disabled and the
3545 * suspend callbacks are run. suspend puts the hardware and software state
3546 * in each IP into a state suitable for suspend.
3547 * Returns 0 on success, negative error code on failure.
3548 */
3549static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3550{
3551 int i, r;
3552
3553 if (adev->in_s0ix)
3554 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3555
3556 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3557 if (!adev->ip_blocks[i].status.valid)
3558 continue;
3559 /* displays are handled in phase1 */
3560 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3561 continue;
3562 /* PSP lost connection when err_event_athub occurs */
3563 if (amdgpu_ras_intr_triggered() &&
3564 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3565 adev->ip_blocks[i].status.hw = false;
3566 continue;
3567 }
3568
3569 /* skip unnecessary suspend if we do not initialize them yet */
3570 if (!amdgpu_ip_member_of_hwini(
3571 adev, adev->ip_blocks[i].version->type))
3572 continue;
3573
3574 /* skip suspend of gfx/mes and psp for S0ix
3575 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3576 * like at runtime. PSP is also part of the always on hardware
3577 * so no need to suspend it.
3578 */
3579 if (adev->in_s0ix &&
3580 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3581 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3582 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3583 continue;
3584
3585 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3586 if (adev->in_s0ix &&
3587 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3588 IP_VERSION(5, 0, 0)) &&
3589 (adev->ip_blocks[i].version->type ==
3590 AMD_IP_BLOCK_TYPE_SDMA))
3591 continue;
3592
3593 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3594 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3595 * from this location and RLC Autoload automatically also gets loaded
3596 * from here based on PMFW -> PSP message during re-init sequence.
3597 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3598 * the TMR and reload FWs again for IMU enabled APU ASICs.
3599 */
3600 if (amdgpu_in_reset(adev) &&
3601 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3602 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3603 continue;
3604
3605 /* XXX handle errors */
3606 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3607 adev->ip_blocks[i].status.hw = false;
3608
3609 /* handle putting the SMC in the appropriate state */
3610 if (!amdgpu_sriov_vf(adev)) {
3611 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3612 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3613 if (r) {
3614 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3615 adev->mp1_state, r);
3616 return r;
3617 }
3618 }
3619 }
3620 }
3621
3622 return 0;
3623}
3624
3625/**
3626 * amdgpu_device_ip_suspend - run suspend for hardware IPs
3627 *
3628 * @adev: amdgpu_device pointer
3629 *
3630 * Main suspend function for hardware IPs. The list of all the hardware
3631 * IPs that make up the asic is walked, clockgating is disabled and the
3632 * suspend callbacks are run. suspend puts the hardware and software state
3633 * in each IP into a state suitable for suspend.
3634 * Returns 0 on success, negative error code on failure.
3635 */
3636int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3637{
3638 int r;
3639
3640 if (amdgpu_sriov_vf(adev)) {
3641 amdgpu_virt_fini_data_exchange(adev);
3642 amdgpu_virt_request_full_gpu(adev, false);
3643 }
3644
3645 amdgpu_ttm_set_buffer_funcs_status(adev, false);
3646
3647 r = amdgpu_device_ip_suspend_phase1(adev);
3648 if (r)
3649 return r;
3650 r = amdgpu_device_ip_suspend_phase2(adev);
3651
3652 if (amdgpu_sriov_vf(adev))
3653 amdgpu_virt_release_full_gpu(adev, false);
3654
3655 return r;
3656}
3657
3658static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3659{
3660 int i, r;
3661
3662 static enum amd_ip_block_type ip_order[] = {
3663 AMD_IP_BLOCK_TYPE_COMMON,
3664 AMD_IP_BLOCK_TYPE_GMC,
3665 AMD_IP_BLOCK_TYPE_PSP,
3666 AMD_IP_BLOCK_TYPE_IH,
3667 };
3668
3669 for (i = 0; i < adev->num_ip_blocks; i++) {
3670 int j;
3671 struct amdgpu_ip_block *block;
3672
3673 block = &adev->ip_blocks[i];
3674 block->status.hw = false;
3675
3676 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3677
3678 if (block->version->type != ip_order[j] ||
3679 !block->status.valid)
3680 continue;
3681
3682 r = block->version->funcs->hw_init(&adev->ip_blocks[i]);
3683 if (r) {
3684 dev_err(adev->dev, "RE-INIT-early: %s failed\n",
3685 block->version->funcs->name);
3686 return r;
3687 }
3688 block->status.hw = true;
3689 }
3690 }
3691
3692 return 0;
3693}
3694
3695static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3696{
3697 struct amdgpu_ip_block *block;
3698 int i, r = 0;
3699
3700 static enum amd_ip_block_type ip_order[] = {
3701 AMD_IP_BLOCK_TYPE_SMC,
3702 AMD_IP_BLOCK_TYPE_DCE,
3703 AMD_IP_BLOCK_TYPE_GFX,
3704 AMD_IP_BLOCK_TYPE_SDMA,
3705 AMD_IP_BLOCK_TYPE_MES,
3706 AMD_IP_BLOCK_TYPE_UVD,
3707 AMD_IP_BLOCK_TYPE_VCE,
3708 AMD_IP_BLOCK_TYPE_VCN,
3709 AMD_IP_BLOCK_TYPE_JPEG
3710 };
3711
3712 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3713 block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]);
3714
3715 if (!block)
3716 continue;
3717
3718 if (block->status.valid && !block->status.hw) {
3719 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) {
3720 r = amdgpu_ip_block_resume(block);
3721 } else {
3722 r = block->version->funcs->hw_init(block);
3723 }
3724
3725 if (r) {
3726 dev_err(adev->dev, "RE-INIT-late: %s failed\n",
3727 block->version->funcs->name);
3728 break;
3729 }
3730 block->status.hw = true;
3731 }
3732 }
3733
3734 return r;
3735}
3736
3737/**
3738 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3739 *
3740 * @adev: amdgpu_device pointer
3741 *
3742 * First resume function for hardware IPs. The list of all the hardware
3743 * IPs that make up the asic is walked and the resume callbacks are run for
3744 * COMMON, GMC, and IH. resume puts the hardware into a functional state
3745 * after a suspend and updates the software state as necessary. This
3746 * function is also used for restoring the GPU after a GPU reset.
3747 * Returns 0 on success, negative error code on failure.
3748 */
3749static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3750{
3751 int i, r;
3752
3753 for (i = 0; i < adev->num_ip_blocks; i++) {
3754 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3755 continue;
3756 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3757 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3758 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3759 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3760
3761 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3762 if (r)
3763 return r;
3764 }
3765 }
3766
3767 return 0;
3768}
3769
3770/**
3771 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3772 *
3773 * @adev: amdgpu_device pointer
3774 *
3775 * Second resume function for hardware IPs. The list of all the hardware
3776 * IPs that make up the asic is walked and the resume callbacks are run for
3777 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
3778 * functional state after a suspend and updates the software state as
3779 * necessary. This function is also used for restoring the GPU after a GPU
3780 * reset.
3781 * Returns 0 on success, negative error code on failure.
3782 */
3783static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3784{
3785 int i, r;
3786
3787 for (i = 0; i < adev->num_ip_blocks; i++) {
3788 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3789 continue;
3790 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3791 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3792 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3793 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE ||
3794 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3795 continue;
3796 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3797 if (r)
3798 return r;
3799 }
3800
3801 return 0;
3802}
3803
3804/**
3805 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs
3806 *
3807 * @adev: amdgpu_device pointer
3808 *
3809 * Third resume function for hardware IPs. The list of all the hardware
3810 * IPs that make up the asic is walked and the resume callbacks are run for
3811 * all DCE. resume puts the hardware into a functional state after a suspend
3812 * and updates the software state as necessary. This function is also used
3813 * for restoring the GPU after a GPU reset.
3814 *
3815 * Returns 0 on success, negative error code on failure.
3816 */
3817static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)
3818{
3819 int i, r;
3820
3821 for (i = 0; i < adev->num_ip_blocks; i++) {
3822 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3823 continue;
3824 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
3825 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3826 if (r)
3827 return r;
3828 }
3829 }
3830
3831 return 0;
3832}
3833
3834/**
3835 * amdgpu_device_ip_resume - run resume for hardware IPs
3836 *
3837 * @adev: amdgpu_device pointer
3838 *
3839 * Main resume function for hardware IPs. The hardware IPs
3840 * are split into two resume functions because they are
3841 * also used in recovering from a GPU reset and some additional
3842 * steps need to be take between them. In this case (S3/S4) they are
3843 * run sequentially.
3844 * Returns 0 on success, negative error code on failure.
3845 */
3846static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3847{
3848 int r;
3849
3850 r = amdgpu_device_ip_resume_phase1(adev);
3851 if (r)
3852 return r;
3853
3854 r = amdgpu_device_fw_loading(adev);
3855 if (r)
3856 return r;
3857
3858 r = amdgpu_device_ip_resume_phase2(adev);
3859
3860 if (adev->mman.buffer_funcs_ring->sched.ready)
3861 amdgpu_ttm_set_buffer_funcs_status(adev, true);
3862
3863 if (r)
3864 return r;
3865
3866 amdgpu_fence_driver_hw_init(adev);
3867
3868 r = amdgpu_device_ip_resume_phase3(adev);
3869
3870 return r;
3871}
3872
3873/**
3874 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3875 *
3876 * @adev: amdgpu_device pointer
3877 *
3878 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3879 */
3880static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3881{
3882 if (amdgpu_sriov_vf(adev)) {
3883 if (adev->is_atom_fw) {
3884 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3885 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3886 } else {
3887 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3888 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3889 }
3890
3891 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3892 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3893 }
3894}
3895
3896/**
3897 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3898 *
3899 * @asic_type: AMD asic type
3900 *
3901 * Check if there is DC (new modesetting infrastructre) support for an asic.
3902 * returns true if DC has support, false if not.
3903 */
3904bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3905{
3906 switch (asic_type) {
3907#ifdef CONFIG_DRM_AMDGPU_SI
3908 case CHIP_HAINAN:
3909#endif
3910 case CHIP_TOPAZ:
3911 /* chips with no display hardware */
3912 return false;
3913#if defined(CONFIG_DRM_AMD_DC)
3914 case CHIP_TAHITI:
3915 case CHIP_PITCAIRN:
3916 case CHIP_VERDE:
3917 case CHIP_OLAND:
3918 /*
3919 * We have systems in the wild with these ASICs that require
3920 * LVDS and VGA support which is not supported with DC.
3921 *
3922 * Fallback to the non-DC driver here by default so as not to
3923 * cause regressions.
3924 */
3925#if defined(CONFIG_DRM_AMD_DC_SI)
3926 return amdgpu_dc > 0;
3927#else
3928 return false;
3929#endif
3930 case CHIP_BONAIRE:
3931 case CHIP_KAVERI:
3932 case CHIP_KABINI:
3933 case CHIP_MULLINS:
3934 /*
3935 * We have systems in the wild with these ASICs that require
3936 * VGA support which is not supported with DC.
3937 *
3938 * Fallback to the non-DC driver here by default so as not to
3939 * cause regressions.
3940 */
3941 return amdgpu_dc > 0;
3942 default:
3943 return amdgpu_dc != 0;
3944#else
3945 default:
3946 if (amdgpu_dc > 0)
3947 DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3948 return false;
3949#endif
3950 }
3951}
3952
3953/**
3954 * amdgpu_device_has_dc_support - check if dc is supported
3955 *
3956 * @adev: amdgpu_device pointer
3957 *
3958 * Returns true for supported, false for not supported
3959 */
3960bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3961{
3962 if (adev->enable_virtual_display ||
3963 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3964 return false;
3965
3966 return amdgpu_device_asic_has_dc_support(adev->asic_type);
3967}
3968
3969static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3970{
3971 struct amdgpu_device *adev =
3972 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3973 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3974
3975 /* It's a bug to not have a hive within this function */
3976 if (WARN_ON(!hive))
3977 return;
3978
3979 /*
3980 * Use task barrier to synchronize all xgmi reset works across the
3981 * hive. task_barrier_enter and task_barrier_exit will block
3982 * until all the threads running the xgmi reset works reach
3983 * those points. task_barrier_full will do both blocks.
3984 */
3985 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3986
3987 task_barrier_enter(&hive->tb);
3988 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3989
3990 if (adev->asic_reset_res)
3991 goto fail;
3992
3993 task_barrier_exit(&hive->tb);
3994 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3995
3996 if (adev->asic_reset_res)
3997 goto fail;
3998
3999 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
4000 } else {
4001
4002 task_barrier_full(&hive->tb);
4003 adev->asic_reset_res = amdgpu_asic_reset(adev);
4004 }
4005
4006fail:
4007 if (adev->asic_reset_res)
4008 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
4009 adev->asic_reset_res, adev_to_drm(adev)->unique);
4010 amdgpu_put_xgmi_hive(hive);
4011}
4012
4013static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
4014{
4015 char *input = amdgpu_lockup_timeout;
4016 char *timeout_setting = NULL;
4017 int index = 0;
4018 long timeout;
4019 int ret = 0;
4020
4021 /*
4022 * By default timeout for non compute jobs is 10000
4023 * and 60000 for compute jobs.
4024 * In SR-IOV or passthrough mode, timeout for compute
4025 * jobs are 60000 by default.
4026 */
4027 adev->gfx_timeout = msecs_to_jiffies(10000);
4028 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
4029 if (amdgpu_sriov_vf(adev))
4030 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
4031 msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
4032 else
4033 adev->compute_timeout = msecs_to_jiffies(60000);
4034
4035 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
4036 while ((timeout_setting = strsep(&input, ",")) &&
4037 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
4038 ret = kstrtol(timeout_setting, 0, &timeout);
4039 if (ret)
4040 return ret;
4041
4042 if (timeout == 0) {
4043 index++;
4044 continue;
4045 } else if (timeout < 0) {
4046 timeout = MAX_SCHEDULE_TIMEOUT;
4047 dev_warn(adev->dev, "lockup timeout disabled");
4048 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
4049 } else {
4050 timeout = msecs_to_jiffies(timeout);
4051 }
4052
4053 switch (index++) {
4054 case 0:
4055 adev->gfx_timeout = timeout;
4056 break;
4057 case 1:
4058 adev->compute_timeout = timeout;
4059 break;
4060 case 2:
4061 adev->sdma_timeout = timeout;
4062 break;
4063 case 3:
4064 adev->video_timeout = timeout;
4065 break;
4066 default:
4067 break;
4068 }
4069 }
4070 /*
4071 * There is only one value specified and
4072 * it should apply to all non-compute jobs.
4073 */
4074 if (index == 1) {
4075 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
4076 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
4077 adev->compute_timeout = adev->gfx_timeout;
4078 }
4079 }
4080
4081 return ret;
4082}
4083
4084/**
4085 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
4086 *
4087 * @adev: amdgpu_device pointer
4088 *
4089 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
4090 */
4091static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
4092{
4093 struct iommu_domain *domain;
4094
4095 domain = iommu_get_domain_for_dev(adev->dev);
4096 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
4097 adev->ram_is_direct_mapped = true;
4098}
4099
4100#if defined(CONFIG_HSA_AMD_P2P)
4101/**
4102 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
4103 *
4104 * @adev: amdgpu_device pointer
4105 *
4106 * return if IOMMU remapping bar address
4107 */
4108static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)
4109{
4110 struct iommu_domain *domain;
4111
4112 domain = iommu_get_domain_for_dev(adev->dev);
4113 if (domain && (domain->type == IOMMU_DOMAIN_DMA ||
4114 domain->type == IOMMU_DOMAIN_DMA_FQ))
4115 return true;
4116
4117 return false;
4118}
4119#endif
4120
4121static const struct attribute *amdgpu_dev_attributes[] = {
4122 &dev_attr_pcie_replay_count.attr,
4123 NULL
4124};
4125
4126static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
4127{
4128 if (amdgpu_mcbp == 1)
4129 adev->gfx.mcbp = true;
4130 else if (amdgpu_mcbp == 0)
4131 adev->gfx.mcbp = false;
4132
4133 if (amdgpu_sriov_vf(adev))
4134 adev->gfx.mcbp = true;
4135
4136 if (adev->gfx.mcbp)
4137 DRM_INFO("MCBP is enabled\n");
4138}
4139
4140/**
4141 * amdgpu_device_init - initialize the driver
4142 *
4143 * @adev: amdgpu_device pointer
4144 * @flags: driver flags
4145 *
4146 * Initializes the driver info and hw (all asics).
4147 * Returns 0 for success or an error on failure.
4148 * Called at driver startup.
4149 */
4150int amdgpu_device_init(struct amdgpu_device *adev,
4151 uint32_t flags)
4152{
4153 struct drm_device *ddev = adev_to_drm(adev);
4154 struct pci_dev *pdev = adev->pdev;
4155 int r, i;
4156 bool px = false;
4157 u32 max_MBps;
4158 int tmp;
4159
4160 adev->shutdown = false;
4161 adev->flags = flags;
4162
4163 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
4164 adev->asic_type = amdgpu_force_asic_type;
4165 else
4166 adev->asic_type = flags & AMD_ASIC_MASK;
4167
4168 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
4169 if (amdgpu_emu_mode == 1)
4170 adev->usec_timeout *= 10;
4171 adev->gmc.gart_size = 512 * 1024 * 1024;
4172 adev->accel_working = false;
4173 adev->num_rings = 0;
4174 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
4175 adev->mman.buffer_funcs = NULL;
4176 adev->mman.buffer_funcs_ring = NULL;
4177 adev->vm_manager.vm_pte_funcs = NULL;
4178 adev->vm_manager.vm_pte_num_scheds = 0;
4179 adev->gmc.gmc_funcs = NULL;
4180 adev->harvest_ip_mask = 0x0;
4181 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
4182 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
4183
4184 adev->smc_rreg = &amdgpu_invalid_rreg;
4185 adev->smc_wreg = &amdgpu_invalid_wreg;
4186 adev->pcie_rreg = &amdgpu_invalid_rreg;
4187 adev->pcie_wreg = &amdgpu_invalid_wreg;
4188 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
4189 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
4190 adev->pciep_rreg = &amdgpu_invalid_rreg;
4191 adev->pciep_wreg = &amdgpu_invalid_wreg;
4192 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
4193 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
4194 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
4195 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
4196 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
4197 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
4198 adev->didt_rreg = &amdgpu_invalid_rreg;
4199 adev->didt_wreg = &amdgpu_invalid_wreg;
4200 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
4201 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
4202 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
4203 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
4204
4205 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
4206 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
4207 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
4208
4209 /* mutex initialization are all done here so we
4210 * can recall function without having locking issues
4211 */
4212 mutex_init(&adev->firmware.mutex);
4213 mutex_init(&adev->pm.mutex);
4214 mutex_init(&adev->gfx.gpu_clock_mutex);
4215 mutex_init(&adev->srbm_mutex);
4216 mutex_init(&adev->gfx.pipe_reserve_mutex);
4217 mutex_init(&adev->gfx.gfx_off_mutex);
4218 mutex_init(&adev->gfx.partition_mutex);
4219 mutex_init(&adev->grbm_idx_mutex);
4220 mutex_init(&adev->mn_lock);
4221 mutex_init(&adev->virt.vf_errors.lock);
4222 mutex_init(&adev->virt.rlcg_reg_lock);
4223 hash_init(adev->mn_hash);
4224 mutex_init(&adev->psp.mutex);
4225 mutex_init(&adev->notifier_lock);
4226 mutex_init(&adev->pm.stable_pstate_ctx_lock);
4227 mutex_init(&adev->benchmark_mutex);
4228 mutex_init(&adev->gfx.reset_sem_mutex);
4229 /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */
4230 mutex_init(&adev->enforce_isolation_mutex);
4231 mutex_init(&adev->gfx.kfd_sch_mutex);
4232
4233 amdgpu_device_init_apu_flags(adev);
4234
4235 r = amdgpu_device_check_arguments(adev);
4236 if (r)
4237 return r;
4238
4239 spin_lock_init(&adev->mmio_idx_lock);
4240 spin_lock_init(&adev->smc_idx_lock);
4241 spin_lock_init(&adev->pcie_idx_lock);
4242 spin_lock_init(&adev->uvd_ctx_idx_lock);
4243 spin_lock_init(&adev->didt_idx_lock);
4244 spin_lock_init(&adev->gc_cac_idx_lock);
4245 spin_lock_init(&adev->se_cac_idx_lock);
4246 spin_lock_init(&adev->audio_endpt_idx_lock);
4247 spin_lock_init(&adev->mm_stats.lock);
4248 spin_lock_init(&adev->wb.lock);
4249
4250 INIT_LIST_HEAD(&adev->reset_list);
4251
4252 INIT_LIST_HEAD(&adev->ras_list);
4253
4254 INIT_LIST_HEAD(&adev->pm.od_kobj_list);
4255
4256 INIT_DELAYED_WORK(&adev->delayed_init_work,
4257 amdgpu_device_delayed_init_work_handler);
4258 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
4259 amdgpu_device_delay_enable_gfx_off);
4260 /*
4261 * Initialize the enforce_isolation work structures for each XCP
4262 * partition. This work handler is responsible for enforcing shader
4263 * isolation on AMD GPUs. It counts the number of emitted fences for
4264 * each GFX and compute ring. If there are any fences, it schedules
4265 * the `enforce_isolation_work` to be run after a delay. If there are
4266 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the
4267 * runqueue.
4268 */
4269 for (i = 0; i < MAX_XCP; i++) {
4270 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work,
4271 amdgpu_gfx_enforce_isolation_handler);
4272 adev->gfx.enforce_isolation[i].adev = adev;
4273 adev->gfx.enforce_isolation[i].xcp_id = i;
4274 }
4275
4276 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
4277
4278 adev->gfx.gfx_off_req_count = 1;
4279 adev->gfx.gfx_off_residency = 0;
4280 adev->gfx.gfx_off_entrycount = 0;
4281 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4282
4283 atomic_set(&adev->throttling_logging_enabled, 1);
4284 /*
4285 * If throttling continues, logging will be performed every minute
4286 * to avoid log flooding. "-1" is subtracted since the thermal
4287 * throttling interrupt comes every second. Thus, the total logging
4288 * interval is 59 seconds(retelimited printk interval) + 1(waiting
4289 * for throttling interrupt) = 60 seconds.
4290 */
4291 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
4292 ratelimit_state_init(&adev->virt.ras_telemetry_rs, 5 * HZ, 1);
4293
4294 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4295 ratelimit_set_flags(&adev->virt.ras_telemetry_rs, RATELIMIT_MSG_ON_RELEASE);
4296
4297 /* Registers mapping */
4298 /* TODO: block userspace mapping of io register */
4299 if (adev->asic_type >= CHIP_BONAIRE) {
4300 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4301 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4302 } else {
4303 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4304 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4305 }
4306
4307 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4308 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
4309
4310 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
4311 if (!adev->rmmio)
4312 return -ENOMEM;
4313
4314 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4315 DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4316
4317 /*
4318 * Reset domain needs to be present early, before XGMI hive discovered
4319 * (if any) and intitialized to use reset sem and in_gpu reset flag
4320 * early on during init and before calling to RREG32.
4321 */
4322 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
4323 if (!adev->reset_domain)
4324 return -ENOMEM;
4325
4326 /* detect hw virtualization here */
4327 amdgpu_detect_virtualization(adev);
4328
4329 amdgpu_device_get_pcie_info(adev);
4330
4331 r = amdgpu_device_get_job_timeout_settings(adev);
4332 if (r) {
4333 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4334 return r;
4335 }
4336
4337 amdgpu_device_set_mcbp(adev);
4338
4339 /*
4340 * By default, use default mode where all blocks are expected to be
4341 * initialized. At present a 'swinit' of blocks is required to be
4342 * completed before the need for a different level is detected.
4343 */
4344 amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
4345 /* early init functions */
4346 r = amdgpu_device_ip_early_init(adev);
4347 if (r)
4348 return r;
4349
4350 /* Get rid of things like offb */
4351 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);
4352 if (r)
4353 return r;
4354
4355 /* Enable TMZ based on IP_VERSION */
4356 amdgpu_gmc_tmz_set(adev);
4357
4358 if (amdgpu_sriov_vf(adev) &&
4359 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
4360 /* VF MMIO access (except mailbox range) from CPU
4361 * will be blocked during sriov runtime
4362 */
4363 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
4364
4365 amdgpu_gmc_noretry_set(adev);
4366 /* Need to get xgmi info early to decide the reset behavior*/
4367 if (adev->gmc.xgmi.supported) {
4368 r = adev->gfxhub.funcs->get_xgmi_info(adev);
4369 if (r)
4370 return r;
4371 }
4372
4373 /* enable PCIE atomic ops */
4374 if (amdgpu_sriov_vf(adev)) {
4375 if (adev->virt.fw_reserve.p_pf2vf)
4376 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4377 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4378 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4379 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4380 * internal path natively support atomics, set have_atomics_support to true.
4381 */
4382 } else if ((adev->flags & AMD_IS_APU) &&
4383 (amdgpu_ip_version(adev, GC_HWIP, 0) >
4384 IP_VERSION(9, 0, 0))) {
4385 adev->have_atomics_support = true;
4386 } else {
4387 adev->have_atomics_support =
4388 !pci_enable_atomic_ops_to_root(adev->pdev,
4389 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4390 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4391 }
4392
4393 if (!adev->have_atomics_support)
4394 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4395
4396 /* doorbell bar mapping and doorbell index init*/
4397 amdgpu_doorbell_init(adev);
4398
4399 if (amdgpu_emu_mode == 1) {
4400 /* post the asic on emulation mode */
4401 emu_soc_asic_init(adev);
4402 goto fence_driver_init;
4403 }
4404
4405 amdgpu_reset_init(adev);
4406
4407 /* detect if we are with an SRIOV vbios */
4408 if (adev->bios)
4409 amdgpu_device_detect_sriov_bios(adev);
4410
4411 /* check if we need to reset the asic
4412 * E.g., driver was not cleanly unloaded previously, etc.
4413 */
4414 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4415 if (adev->gmc.xgmi.num_physical_nodes) {
4416 dev_info(adev->dev, "Pending hive reset.\n");
4417 amdgpu_set_init_level(adev,
4418 AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
4419 } else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) &&
4420 !amdgpu_device_has_display_hardware(adev)) {
4421 r = psp_gpu_reset(adev);
4422 } else {
4423 tmp = amdgpu_reset_method;
4424 /* It should do a default reset when loading or reloading the driver,
4425 * regardless of the module parameter reset_method.
4426 */
4427 amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4428 r = amdgpu_asic_reset(adev);
4429 amdgpu_reset_method = tmp;
4430 }
4431
4432 if (r) {
4433 dev_err(adev->dev, "asic reset on init failed\n");
4434 goto failed;
4435 }
4436 }
4437
4438 /* Post card if necessary */
4439 if (amdgpu_device_need_post(adev)) {
4440 if (!adev->bios) {
4441 dev_err(adev->dev, "no vBIOS found\n");
4442 r = -EINVAL;
4443 goto failed;
4444 }
4445 DRM_INFO("GPU posting now...\n");
4446 r = amdgpu_device_asic_init(adev);
4447 if (r) {
4448 dev_err(adev->dev, "gpu post error!\n");
4449 goto failed;
4450 }
4451 }
4452
4453 if (adev->bios) {
4454 if (adev->is_atom_fw) {
4455 /* Initialize clocks */
4456 r = amdgpu_atomfirmware_get_clock_info(adev);
4457 if (r) {
4458 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4459 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4460 goto failed;
4461 }
4462 } else {
4463 /* Initialize clocks */
4464 r = amdgpu_atombios_get_clock_info(adev);
4465 if (r) {
4466 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4467 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4468 goto failed;
4469 }
4470 /* init i2c buses */
4471 if (!amdgpu_device_has_dc_support(adev))
4472 amdgpu_atombios_i2c_init(adev);
4473 }
4474 }
4475
4476fence_driver_init:
4477 /* Fence driver */
4478 r = amdgpu_fence_driver_sw_init(adev);
4479 if (r) {
4480 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4481 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4482 goto failed;
4483 }
4484
4485 /* init the mode config */
4486 drm_mode_config_init(adev_to_drm(adev));
4487
4488 r = amdgpu_device_ip_init(adev);
4489 if (r) {
4490 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4491 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4492 goto release_ras_con;
4493 }
4494
4495 amdgpu_fence_driver_hw_init(adev);
4496
4497 dev_info(adev->dev,
4498 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4499 adev->gfx.config.max_shader_engines,
4500 adev->gfx.config.max_sh_per_se,
4501 adev->gfx.config.max_cu_per_sh,
4502 adev->gfx.cu_info.number);
4503
4504 adev->accel_working = true;
4505
4506 amdgpu_vm_check_compute_bug(adev);
4507
4508 /* Initialize the buffer migration limit. */
4509 if (amdgpu_moverate >= 0)
4510 max_MBps = amdgpu_moverate;
4511 else
4512 max_MBps = 8; /* Allow 8 MB/s. */
4513 /* Get a log2 for easy divisions. */
4514 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4515
4516 /*
4517 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4518 * Otherwise the mgpu fan boost feature will be skipped due to the
4519 * gpu instance is counted less.
4520 */
4521 amdgpu_register_gpu_instance(adev);
4522
4523 /* enable clockgating, etc. after ib tests, etc. since some blocks require
4524 * explicit gating rather than handling it automatically.
4525 */
4526 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4527 r = amdgpu_device_ip_late_init(adev);
4528 if (r) {
4529 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4530 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4531 goto release_ras_con;
4532 }
4533 /* must succeed. */
4534 amdgpu_ras_resume(adev);
4535 queue_delayed_work(system_wq, &adev->delayed_init_work,
4536 msecs_to_jiffies(AMDGPU_RESUME_MS));
4537 }
4538
4539 if (amdgpu_sriov_vf(adev)) {
4540 amdgpu_virt_release_full_gpu(adev, true);
4541 flush_delayed_work(&adev->delayed_init_work);
4542 }
4543
4544 /*
4545 * Place those sysfs registering after `late_init`. As some of those
4546 * operations performed in `late_init` might affect the sysfs
4547 * interfaces creating.
4548 */
4549 r = amdgpu_atombios_sysfs_init(adev);
4550 if (r)
4551 drm_err(&adev->ddev,
4552 "registering atombios sysfs failed (%d).\n", r);
4553
4554 r = amdgpu_pm_sysfs_init(adev);
4555 if (r)
4556 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4557
4558 r = amdgpu_ucode_sysfs_init(adev);
4559 if (r) {
4560 adev->ucode_sysfs_en = false;
4561 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4562 } else
4563 adev->ucode_sysfs_en = true;
4564
4565 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4566 if (r)
4567 dev_err(adev->dev, "Could not create amdgpu device attr\n");
4568
4569 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4570 if (r)
4571 dev_err(adev->dev,
4572 "Could not create amdgpu board attributes\n");
4573
4574 amdgpu_fru_sysfs_init(adev);
4575 amdgpu_reg_state_sysfs_init(adev);
4576 amdgpu_xcp_cfg_sysfs_init(adev);
4577
4578 if (IS_ENABLED(CONFIG_PERF_EVENTS))
4579 r = amdgpu_pmu_init(adev);
4580 if (r)
4581 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4582
4583 /* Have stored pci confspace at hand for restore in sudden PCI error */
4584 if (amdgpu_device_cache_pci_state(adev->pdev))
4585 pci_restore_state(pdev);
4586
4587 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4588 /* this will fail for cards that aren't VGA class devices, just
4589 * ignore it
4590 */
4591 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4592 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4593
4594 px = amdgpu_device_supports_px(ddev);
4595
4596 if (px || (!dev_is_removable(&adev->pdev->dev) &&
4597 apple_gmux_detect(NULL, NULL)))
4598 vga_switcheroo_register_client(adev->pdev,
4599 &amdgpu_switcheroo_ops, px);
4600
4601 if (px)
4602 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4603
4604 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
4605 amdgpu_xgmi_reset_on_init(adev);
4606
4607 amdgpu_device_check_iommu_direct_map(adev);
4608
4609 return 0;
4610
4611release_ras_con:
4612 if (amdgpu_sriov_vf(adev))
4613 amdgpu_virt_release_full_gpu(adev, true);
4614
4615 /* failed in exclusive mode due to timeout */
4616 if (amdgpu_sriov_vf(adev) &&
4617 !amdgpu_sriov_runtime(adev) &&
4618 amdgpu_virt_mmio_blocked(adev) &&
4619 !amdgpu_virt_wait_reset(adev)) {
4620 dev_err(adev->dev, "VF exclusive mode timeout\n");
4621 /* Don't send request since VF is inactive. */
4622 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4623 adev->virt.ops = NULL;
4624 r = -EAGAIN;
4625 }
4626 amdgpu_release_ras_context(adev);
4627
4628failed:
4629 amdgpu_vf_error_trans_all(adev);
4630
4631 return r;
4632}
4633
4634static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4635{
4636
4637 /* Clear all CPU mappings pointing to this device */
4638 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4639
4640 /* Unmap all mapped bars - Doorbell, registers and VRAM */
4641 amdgpu_doorbell_fini(adev);
4642
4643 iounmap(adev->rmmio);
4644 adev->rmmio = NULL;
4645 if (adev->mman.aper_base_kaddr)
4646 iounmap(adev->mman.aper_base_kaddr);
4647 adev->mman.aper_base_kaddr = NULL;
4648
4649 /* Memory manager related */
4650 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4651 arch_phys_wc_del(adev->gmc.vram_mtrr);
4652 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4653 }
4654}
4655
4656/**
4657 * amdgpu_device_fini_hw - tear down the driver
4658 *
4659 * @adev: amdgpu_device pointer
4660 *
4661 * Tear down the driver info (all asics).
4662 * Called at driver shutdown.
4663 */
4664void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4665{
4666 dev_info(adev->dev, "amdgpu: finishing device.\n");
4667 flush_delayed_work(&adev->delayed_init_work);
4668
4669 if (adev->mman.initialized)
4670 drain_workqueue(adev->mman.bdev.wq);
4671 adev->shutdown = true;
4672
4673 /* make sure IB test finished before entering exclusive mode
4674 * to avoid preemption on IB test
4675 */
4676 if (amdgpu_sriov_vf(adev)) {
4677 amdgpu_virt_request_full_gpu(adev, false);
4678 amdgpu_virt_fini_data_exchange(adev);
4679 }
4680
4681 /* disable all interrupts */
4682 amdgpu_irq_disable_all(adev);
4683 if (adev->mode_info.mode_config_initialized) {
4684 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4685 drm_helper_force_disable_all(adev_to_drm(adev));
4686 else
4687 drm_atomic_helper_shutdown(adev_to_drm(adev));
4688 }
4689 amdgpu_fence_driver_hw_fini(adev);
4690
4691 if (adev->pm.sysfs_initialized)
4692 amdgpu_pm_sysfs_fini(adev);
4693 if (adev->ucode_sysfs_en)
4694 amdgpu_ucode_sysfs_fini(adev);
4695 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4696 amdgpu_fru_sysfs_fini(adev);
4697
4698 amdgpu_reg_state_sysfs_fini(adev);
4699 amdgpu_xcp_cfg_sysfs_fini(adev);
4700
4701 /* disable ras feature must before hw fini */
4702 amdgpu_ras_pre_fini(adev);
4703
4704 amdgpu_ttm_set_buffer_funcs_status(adev, false);
4705
4706 amdgpu_device_ip_fini_early(adev);
4707
4708 amdgpu_irq_fini_hw(adev);
4709
4710 if (adev->mman.initialized)
4711 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4712
4713 amdgpu_gart_dummy_page_fini(adev);
4714
4715 if (drm_dev_is_unplugged(adev_to_drm(adev)))
4716 amdgpu_device_unmap_mmio(adev);
4717
4718}
4719
4720void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4721{
4722 int idx;
4723 bool px;
4724
4725 amdgpu_device_ip_fini(adev);
4726 amdgpu_fence_driver_sw_fini(adev);
4727 amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4728 adev->accel_working = false;
4729 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4730
4731 amdgpu_reset_fini(adev);
4732
4733 /* free i2c buses */
4734 if (!amdgpu_device_has_dc_support(adev))
4735 amdgpu_i2c_fini(adev);
4736
4737 if (amdgpu_emu_mode != 1)
4738 amdgpu_atombios_fini(adev);
4739
4740 kfree(adev->bios);
4741 adev->bios = NULL;
4742
4743 kfree(adev->fru_info);
4744 adev->fru_info = NULL;
4745
4746 px = amdgpu_device_supports_px(adev_to_drm(adev));
4747
4748 if (px || (!dev_is_removable(&adev->pdev->dev) &&
4749 apple_gmux_detect(NULL, NULL)))
4750 vga_switcheroo_unregister_client(adev->pdev);
4751
4752 if (px)
4753 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4754
4755 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4756 vga_client_unregister(adev->pdev);
4757
4758 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4759
4760 iounmap(adev->rmmio);
4761 adev->rmmio = NULL;
4762 amdgpu_doorbell_fini(adev);
4763 drm_dev_exit(idx);
4764 }
4765
4766 if (IS_ENABLED(CONFIG_PERF_EVENTS))
4767 amdgpu_pmu_fini(adev);
4768 if (adev->mman.discovery_bin)
4769 amdgpu_discovery_fini(adev);
4770
4771 amdgpu_reset_put_reset_domain(adev->reset_domain);
4772 adev->reset_domain = NULL;
4773
4774 kfree(adev->pci_state);
4775
4776}
4777
4778/**
4779 * amdgpu_device_evict_resources - evict device resources
4780 * @adev: amdgpu device object
4781 *
4782 * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4783 * of the vram memory type. Mainly used for evicting device resources
4784 * at suspend time.
4785 *
4786 */
4787static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4788{
4789 int ret;
4790
4791 /* No need to evict vram on APUs for suspend to ram or s2idle */
4792 if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4793 return 0;
4794
4795 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4796 if (ret)
4797 DRM_WARN("evicting device resources failed\n");
4798 return ret;
4799}
4800
4801/*
4802 * Suspend & resume.
4803 */
4804/**
4805 * amdgpu_device_prepare - prepare for device suspend
4806 *
4807 * @dev: drm dev pointer
4808 *
4809 * Prepare to put the hw in the suspend state (all asics).
4810 * Returns 0 for success or an error on failure.
4811 * Called at driver suspend.
4812 */
4813int amdgpu_device_prepare(struct drm_device *dev)
4814{
4815 struct amdgpu_device *adev = drm_to_adev(dev);
4816 int i, r;
4817
4818 amdgpu_choose_low_power_state(adev);
4819
4820 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4821 return 0;
4822
4823 /* Evict the majority of BOs before starting suspend sequence */
4824 r = amdgpu_device_evict_resources(adev);
4825 if (r)
4826 goto unprepare;
4827
4828 flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4829
4830 for (i = 0; i < adev->num_ip_blocks; i++) {
4831 if (!adev->ip_blocks[i].status.valid)
4832 continue;
4833 if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4834 continue;
4835 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]);
4836 if (r)
4837 goto unprepare;
4838 }
4839
4840 return 0;
4841
4842unprepare:
4843 adev->in_s0ix = adev->in_s3 = false;
4844
4845 return r;
4846}
4847
4848/**
4849 * amdgpu_device_suspend - initiate device suspend
4850 *
4851 * @dev: drm dev pointer
4852 * @notify_clients: notify in-kernel DRM clients
4853 *
4854 * Puts the hw in the suspend state (all asics).
4855 * Returns 0 for success or an error on failure.
4856 * Called at driver suspend.
4857 */
4858int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
4859{
4860 struct amdgpu_device *adev = drm_to_adev(dev);
4861 int r = 0;
4862
4863 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4864 return 0;
4865
4866 adev->in_suspend = true;
4867
4868 if (amdgpu_sriov_vf(adev)) {
4869 amdgpu_virt_fini_data_exchange(adev);
4870 r = amdgpu_virt_request_full_gpu(adev, false);
4871 if (r)
4872 return r;
4873 }
4874
4875 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4876 DRM_WARN("smart shift update failed\n");
4877
4878 if (notify_clients)
4879 drm_client_dev_suspend(adev_to_drm(adev), false);
4880
4881 cancel_delayed_work_sync(&adev->delayed_init_work);
4882
4883 amdgpu_ras_suspend(adev);
4884
4885 amdgpu_device_ip_suspend_phase1(adev);
4886
4887 if (!adev->in_s0ix)
4888 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4889
4890 r = amdgpu_device_evict_resources(adev);
4891 if (r)
4892 return r;
4893
4894 amdgpu_ttm_set_buffer_funcs_status(adev, false);
4895
4896 amdgpu_fence_driver_hw_fini(adev);
4897
4898 amdgpu_device_ip_suspend_phase2(adev);
4899
4900 if (amdgpu_sriov_vf(adev))
4901 amdgpu_virt_release_full_gpu(adev, false);
4902
4903 r = amdgpu_dpm_notify_rlc_state(adev, false);
4904 if (r)
4905 return r;
4906
4907 return 0;
4908}
4909
4910/**
4911 * amdgpu_device_resume - initiate device resume
4912 *
4913 * @dev: drm dev pointer
4914 * @notify_clients: notify in-kernel DRM clients
4915 *
4916 * Bring the hw back to operating state (all asics).
4917 * Returns 0 for success or an error on failure.
4918 * Called at driver resume.
4919 */
4920int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
4921{
4922 struct amdgpu_device *adev = drm_to_adev(dev);
4923 int r = 0;
4924
4925 if (amdgpu_sriov_vf(adev)) {
4926 r = amdgpu_virt_request_full_gpu(adev, true);
4927 if (r)
4928 return r;
4929 }
4930
4931 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4932 return 0;
4933
4934 if (adev->in_s0ix)
4935 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4936
4937 /* post card */
4938 if (amdgpu_device_need_post(adev)) {
4939 r = amdgpu_device_asic_init(adev);
4940 if (r)
4941 dev_err(adev->dev, "amdgpu asic init failed\n");
4942 }
4943
4944 r = amdgpu_device_ip_resume(adev);
4945
4946 if (r) {
4947 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4948 goto exit;
4949 }
4950
4951 if (!adev->in_s0ix) {
4952 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4953 if (r)
4954 goto exit;
4955 }
4956
4957 r = amdgpu_device_ip_late_init(adev);
4958 if (r)
4959 goto exit;
4960
4961 queue_delayed_work(system_wq, &adev->delayed_init_work,
4962 msecs_to_jiffies(AMDGPU_RESUME_MS));
4963exit:
4964 if (amdgpu_sriov_vf(adev)) {
4965 amdgpu_virt_init_data_exchange(adev);
4966 amdgpu_virt_release_full_gpu(adev, true);
4967 }
4968
4969 if (r)
4970 return r;
4971
4972 /* Make sure IB tests flushed */
4973 flush_delayed_work(&adev->delayed_init_work);
4974
4975 if (notify_clients)
4976 drm_client_dev_resume(adev_to_drm(adev), false);
4977
4978 amdgpu_ras_resume(adev);
4979
4980 if (adev->mode_info.num_crtc) {
4981 /*
4982 * Most of the connector probing functions try to acquire runtime pm
4983 * refs to ensure that the GPU is powered on when connector polling is
4984 * performed. Since we're calling this from a runtime PM callback,
4985 * trying to acquire rpm refs will cause us to deadlock.
4986 *
4987 * Since we're guaranteed to be holding the rpm lock, it's safe to
4988 * temporarily disable the rpm helpers so this doesn't deadlock us.
4989 */
4990#ifdef CONFIG_PM
4991 dev->dev->power.disable_depth++;
4992#endif
4993 if (!adev->dc_enabled)
4994 drm_helper_hpd_irq_event(dev);
4995 else
4996 drm_kms_helper_hotplug_event(dev);
4997#ifdef CONFIG_PM
4998 dev->dev->power.disable_depth--;
4999#endif
5000 }
5001 adev->in_suspend = false;
5002
5003 if (adev->enable_mes)
5004 amdgpu_mes_self_test(adev);
5005
5006 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
5007 DRM_WARN("smart shift update failed\n");
5008
5009 return 0;
5010}
5011
5012/**
5013 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
5014 *
5015 * @adev: amdgpu_device pointer
5016 *
5017 * The list of all the hardware IPs that make up the asic is walked and
5018 * the check_soft_reset callbacks are run. check_soft_reset determines
5019 * if the asic is still hung or not.
5020 * Returns true if any of the IPs are still in a hung state, false if not.
5021 */
5022static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
5023{
5024 int i;
5025 bool asic_hang = false;
5026
5027 if (amdgpu_sriov_vf(adev))
5028 return true;
5029
5030 if (amdgpu_asic_need_full_reset(adev))
5031 return true;
5032
5033 for (i = 0; i < adev->num_ip_blocks; i++) {
5034 if (!adev->ip_blocks[i].status.valid)
5035 continue;
5036 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
5037 adev->ip_blocks[i].status.hang =
5038 adev->ip_blocks[i].version->funcs->check_soft_reset(
5039 &adev->ip_blocks[i]);
5040 if (adev->ip_blocks[i].status.hang) {
5041 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
5042 asic_hang = true;
5043 }
5044 }
5045 return asic_hang;
5046}
5047
5048/**
5049 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
5050 *
5051 * @adev: amdgpu_device pointer
5052 *
5053 * The list of all the hardware IPs that make up the asic is walked and the
5054 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
5055 * handles any IP specific hardware or software state changes that are
5056 * necessary for a soft reset to succeed.
5057 * Returns 0 on success, negative error code on failure.
5058 */
5059static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
5060{
5061 int i, r = 0;
5062
5063 for (i = 0; i < adev->num_ip_blocks; i++) {
5064 if (!adev->ip_blocks[i].status.valid)
5065 continue;
5066 if (adev->ip_blocks[i].status.hang &&
5067 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
5068 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]);
5069 if (r)
5070 return r;
5071 }
5072 }
5073
5074 return 0;
5075}
5076
5077/**
5078 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
5079 *
5080 * @adev: amdgpu_device pointer
5081 *
5082 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
5083 * reset is necessary to recover.
5084 * Returns true if a full asic reset is required, false if not.
5085 */
5086static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
5087{
5088 int i;
5089
5090 if (amdgpu_asic_need_full_reset(adev))
5091 return true;
5092
5093 for (i = 0; i < adev->num_ip_blocks; i++) {
5094 if (!adev->ip_blocks[i].status.valid)
5095 continue;
5096 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
5097 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
5098 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
5099 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
5100 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
5101 if (adev->ip_blocks[i].status.hang) {
5102 dev_info(adev->dev, "Some block need full reset!\n");
5103 return true;
5104 }
5105 }
5106 }
5107 return false;
5108}
5109
5110/**
5111 * amdgpu_device_ip_soft_reset - do a soft reset
5112 *
5113 * @adev: amdgpu_device pointer
5114 *
5115 * The list of all the hardware IPs that make up the asic is walked and the
5116 * soft_reset callbacks are run if the block is hung. soft_reset handles any
5117 * IP specific hardware or software state changes that are necessary to soft
5118 * reset the IP.
5119 * Returns 0 on success, negative error code on failure.
5120 */
5121static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
5122{
5123 int i, r = 0;
5124
5125 for (i = 0; i < adev->num_ip_blocks; i++) {
5126 if (!adev->ip_blocks[i].status.valid)
5127 continue;
5128 if (adev->ip_blocks[i].status.hang &&
5129 adev->ip_blocks[i].version->funcs->soft_reset) {
5130 r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]);
5131 if (r)
5132 return r;
5133 }
5134 }
5135
5136 return 0;
5137}
5138
5139/**
5140 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
5141 *
5142 * @adev: amdgpu_device pointer
5143 *
5144 * The list of all the hardware IPs that make up the asic is walked and the
5145 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
5146 * handles any IP specific hardware or software state changes that are
5147 * necessary after the IP has been soft reset.
5148 * Returns 0 on success, negative error code on failure.
5149 */
5150static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
5151{
5152 int i, r = 0;
5153
5154 for (i = 0; i < adev->num_ip_blocks; i++) {
5155 if (!adev->ip_blocks[i].status.valid)
5156 continue;
5157 if (adev->ip_blocks[i].status.hang &&
5158 adev->ip_blocks[i].version->funcs->post_soft_reset)
5159 r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]);
5160 if (r)
5161 return r;
5162 }
5163
5164 return 0;
5165}
5166
5167/**
5168 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
5169 *
5170 * @adev: amdgpu_device pointer
5171 * @reset_context: amdgpu reset context pointer
5172 *
5173 * do VF FLR and reinitialize Asic
5174 * return 0 means succeeded otherwise failed
5175 */
5176static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
5177 struct amdgpu_reset_context *reset_context)
5178{
5179 int r;
5180 struct amdgpu_hive_info *hive = NULL;
5181
5182 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) {
5183 if (!amdgpu_ras_get_fed_status(adev))
5184 amdgpu_virt_ready_to_reset(adev);
5185 amdgpu_virt_wait_reset(adev);
5186 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags);
5187 r = amdgpu_virt_request_full_gpu(adev, true);
5188 } else {
5189 r = amdgpu_virt_reset_gpu(adev);
5190 }
5191 if (r)
5192 return r;
5193
5194 amdgpu_ras_set_fed(adev, false);
5195 amdgpu_irq_gpu_reset_resume_helper(adev);
5196
5197 /* some sw clean up VF needs to do before recover */
5198 amdgpu_virt_post_reset(adev);
5199
5200 /* Resume IP prior to SMC */
5201 r = amdgpu_device_ip_reinit_early_sriov(adev);
5202 if (r)
5203 return r;
5204
5205 amdgpu_virt_init_data_exchange(adev);
5206
5207 r = amdgpu_device_fw_loading(adev);
5208 if (r)
5209 return r;
5210
5211 /* now we are okay to resume SMC/CP/SDMA */
5212 r = amdgpu_device_ip_reinit_late_sriov(adev);
5213 if (r)
5214 return r;
5215
5216 hive = amdgpu_get_xgmi_hive(adev);
5217 /* Update PSP FW topology after reset */
5218 if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5219 r = amdgpu_xgmi_update_topology(hive, adev);
5220 if (hive)
5221 amdgpu_put_xgmi_hive(hive);
5222 if (r)
5223 return r;
5224
5225 r = amdgpu_ib_ring_tests(adev);
5226 if (r)
5227 return r;
5228
5229 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST)
5230 amdgpu_inc_vram_lost(adev);
5231
5232 /* need to be called during full access so we can't do it later like
5233 * bare-metal does.
5234 */
5235 amdgpu_amdkfd_post_reset(adev);
5236 amdgpu_virt_release_full_gpu(adev, true);
5237
5238 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5239 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
5240 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
5241 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
5242 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5243 amdgpu_ras_resume(adev);
5244
5245 amdgpu_virt_ras_telemetry_post_reset(adev);
5246
5247 return 0;
5248}
5249
5250/**
5251 * amdgpu_device_has_job_running - check if there is any job in mirror list
5252 *
5253 * @adev: amdgpu_device pointer
5254 *
5255 * check if there is any job in mirror list
5256 */
5257bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5258{
5259 int i;
5260 struct drm_sched_job *job;
5261
5262 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5263 struct amdgpu_ring *ring = adev->rings[i];
5264
5265 if (!amdgpu_ring_sched_ready(ring))
5266 continue;
5267
5268 spin_lock(&ring->sched.job_list_lock);
5269 job = list_first_entry_or_null(&ring->sched.pending_list,
5270 struct drm_sched_job, list);
5271 spin_unlock(&ring->sched.job_list_lock);
5272 if (job)
5273 return true;
5274 }
5275 return false;
5276}
5277
5278/**
5279 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5280 *
5281 * @adev: amdgpu_device pointer
5282 *
5283 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5284 * a hung GPU.
5285 */
5286bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5287{
5288
5289 if (amdgpu_gpu_recovery == 0)
5290 goto disabled;
5291
5292 /* Skip soft reset check in fatal error mode */
5293 if (!amdgpu_ras_is_poison_mode_supported(adev))
5294 return true;
5295
5296 if (amdgpu_sriov_vf(adev))
5297 return true;
5298
5299 if (amdgpu_gpu_recovery == -1) {
5300 switch (adev->asic_type) {
5301#ifdef CONFIG_DRM_AMDGPU_SI
5302 case CHIP_VERDE:
5303 case CHIP_TAHITI:
5304 case CHIP_PITCAIRN:
5305 case CHIP_OLAND:
5306 case CHIP_HAINAN:
5307#endif
5308#ifdef CONFIG_DRM_AMDGPU_CIK
5309 case CHIP_KAVERI:
5310 case CHIP_KABINI:
5311 case CHIP_MULLINS:
5312#endif
5313 case CHIP_CARRIZO:
5314 case CHIP_STONEY:
5315 case CHIP_CYAN_SKILLFISH:
5316 goto disabled;
5317 default:
5318 break;
5319 }
5320 }
5321
5322 return true;
5323
5324disabled:
5325 dev_info(adev->dev, "GPU recovery disabled.\n");
5326 return false;
5327}
5328
5329int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5330{
5331 u32 i;
5332 int ret = 0;
5333
5334 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5335
5336 dev_info(adev->dev, "GPU mode1 reset\n");
5337
5338 /* Cache the state before bus master disable. The saved config space
5339 * values are used in other cases like restore after mode-2 reset.
5340 */
5341 amdgpu_device_cache_pci_state(adev->pdev);
5342
5343 /* disable BM */
5344 pci_clear_master(adev->pdev);
5345
5346 if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5347 dev_info(adev->dev, "GPU smu mode1 reset\n");
5348 ret = amdgpu_dpm_mode1_reset(adev);
5349 } else {
5350 dev_info(adev->dev, "GPU psp mode1 reset\n");
5351 ret = psp_gpu_reset(adev);
5352 }
5353
5354 if (ret)
5355 goto mode1_reset_failed;
5356
5357 amdgpu_device_load_pci_state(adev->pdev);
5358 ret = amdgpu_psp_wait_for_bootloader(adev);
5359 if (ret)
5360 goto mode1_reset_failed;
5361
5362 /* wait for asic to come out of reset */
5363 for (i = 0; i < adev->usec_timeout; i++) {
5364 u32 memsize = adev->nbio.funcs->get_memsize(adev);
5365
5366 if (memsize != 0xffffffff)
5367 break;
5368 udelay(1);
5369 }
5370
5371 if (i >= adev->usec_timeout) {
5372 ret = -ETIMEDOUT;
5373 goto mode1_reset_failed;
5374 }
5375
5376 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5377
5378 return 0;
5379
5380mode1_reset_failed:
5381 dev_err(adev->dev, "GPU mode1 reset failed\n");
5382 return ret;
5383}
5384
5385int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5386 struct amdgpu_reset_context *reset_context)
5387{
5388 int i, r = 0;
5389 struct amdgpu_job *job = NULL;
5390 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev;
5391 bool need_full_reset =
5392 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5393
5394 if (reset_context->reset_req_dev == adev)
5395 job = reset_context->job;
5396
5397 if (amdgpu_sriov_vf(adev))
5398 amdgpu_virt_pre_reset(adev);
5399
5400 amdgpu_fence_driver_isr_toggle(adev, true);
5401
5402 /* block all schedulers and reset given job's ring */
5403 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5404 struct amdgpu_ring *ring = adev->rings[i];
5405
5406 if (!amdgpu_ring_sched_ready(ring))
5407 continue;
5408
5409 /* Clear job fence from fence drv to avoid force_completion
5410 * leave NULL and vm flush fence in fence drv
5411 */
5412 amdgpu_fence_driver_clear_job_fences(ring);
5413
5414 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5415 amdgpu_fence_driver_force_completion(ring);
5416 }
5417
5418 amdgpu_fence_driver_isr_toggle(adev, false);
5419
5420 if (job && job->vm)
5421 drm_sched_increase_karma(&job->base);
5422
5423 r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5424 /* If reset handler not implemented, continue; otherwise return */
5425 if (r == -EOPNOTSUPP)
5426 r = 0;
5427 else
5428 return r;
5429
5430 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5431 if (!amdgpu_sriov_vf(adev)) {
5432
5433 if (!need_full_reset)
5434 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5435
5436 if (!need_full_reset && amdgpu_gpu_recovery &&
5437 amdgpu_device_ip_check_soft_reset(adev)) {
5438 amdgpu_device_ip_pre_soft_reset(adev);
5439 r = amdgpu_device_ip_soft_reset(adev);
5440 amdgpu_device_ip_post_soft_reset(adev);
5441 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5442 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5443 need_full_reset = true;
5444 }
5445 }
5446
5447 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
5448 dev_info(tmp_adev->dev, "Dumping IP State\n");
5449 /* Trigger ip dump before we reset the asic */
5450 for (i = 0; i < tmp_adev->num_ip_blocks; i++)
5451 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state)
5452 tmp_adev->ip_blocks[i].version->funcs
5453 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]);
5454 dev_info(tmp_adev->dev, "Dumping IP State Completed\n");
5455 }
5456
5457 if (need_full_reset)
5458 r = amdgpu_device_ip_suspend(adev);
5459 if (need_full_reset)
5460 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5461 else
5462 clear_bit(AMDGPU_NEED_FULL_RESET,
5463 &reset_context->flags);
5464 }
5465
5466 return r;
5467}
5468
5469int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
5470{
5471 struct list_head *device_list_handle;
5472 bool full_reset, vram_lost = false;
5473 struct amdgpu_device *tmp_adev;
5474 int r, init_level;
5475
5476 device_list_handle = reset_context->reset_device_list;
5477
5478 if (!device_list_handle)
5479 return -EINVAL;
5480
5481 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5482
5483 /**
5484 * If it's reset on init, it's default init level, otherwise keep level
5485 * as recovery level.
5486 */
5487 if (reset_context->method == AMD_RESET_METHOD_ON_INIT)
5488 init_level = AMDGPU_INIT_LEVEL_DEFAULT;
5489 else
5490 init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY;
5491
5492 r = 0;
5493 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5494 amdgpu_set_init_level(tmp_adev, init_level);
5495 if (full_reset) {
5496 /* post card */
5497 amdgpu_ras_set_fed(tmp_adev, false);
5498 r = amdgpu_device_asic_init(tmp_adev);
5499 if (r) {
5500 dev_warn(tmp_adev->dev, "asic atom init failed!");
5501 } else {
5502 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5503
5504 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5505 if (r)
5506 goto out;
5507
5508 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5509
5510 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags))
5511 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job);
5512
5513 if (vram_lost) {
5514 DRM_INFO("VRAM is lost due to GPU reset!\n");
5515 amdgpu_inc_vram_lost(tmp_adev);
5516 }
5517
5518 r = amdgpu_device_fw_loading(tmp_adev);
5519 if (r)
5520 return r;
5521
5522 r = amdgpu_xcp_restore_partition_mode(
5523 tmp_adev->xcp_mgr);
5524 if (r)
5525 goto out;
5526
5527 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5528 if (r)
5529 goto out;
5530
5531 if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5532 amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5533
5534 r = amdgpu_device_ip_resume_phase3(tmp_adev);
5535 if (r)
5536 goto out;
5537
5538 if (vram_lost)
5539 amdgpu_device_fill_reset_magic(tmp_adev);
5540
5541 /*
5542 * Add this ASIC as tracked as reset was already
5543 * complete successfully.
5544 */
5545 amdgpu_register_gpu_instance(tmp_adev);
5546
5547 if (!reset_context->hive &&
5548 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5549 amdgpu_xgmi_add_device(tmp_adev);
5550
5551 r = amdgpu_device_ip_late_init(tmp_adev);
5552 if (r)
5553 goto out;
5554
5555 drm_client_dev_resume(adev_to_drm(tmp_adev), false);
5556
5557 /*
5558 * The GPU enters bad state once faulty pages
5559 * by ECC has reached the threshold, and ras
5560 * recovery is scheduled next. So add one check
5561 * here to break recovery if it indeed exceeds
5562 * bad page threshold, and remind user to
5563 * retire this GPU or setting one bigger
5564 * bad_page_threshold value to fix this once
5565 * probing driver again.
5566 */
5567 if (!amdgpu_ras_is_rma(tmp_adev)) {
5568 /* must succeed. */
5569 amdgpu_ras_resume(tmp_adev);
5570 } else {
5571 r = -EINVAL;
5572 goto out;
5573 }
5574
5575 /* Update PSP FW topology after reset */
5576 if (reset_context->hive &&
5577 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5578 r = amdgpu_xgmi_update_topology(
5579 reset_context->hive, tmp_adev);
5580 }
5581 }
5582
5583out:
5584 if (!r) {
5585 /* IP init is complete now, set level as default */
5586 amdgpu_set_init_level(tmp_adev,
5587 AMDGPU_INIT_LEVEL_DEFAULT);
5588 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5589 r = amdgpu_ib_ring_tests(tmp_adev);
5590 if (r) {
5591 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5592 r = -EAGAIN;
5593 goto end;
5594 }
5595 }
5596
5597 if (r)
5598 tmp_adev->asic_reset_res = r;
5599 }
5600
5601end:
5602 return r;
5603}
5604
5605int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5606 struct amdgpu_reset_context *reset_context)
5607{
5608 struct amdgpu_device *tmp_adev = NULL;
5609 bool need_full_reset, skip_hw_reset;
5610 int r = 0;
5611
5612 /* Try reset handler method first */
5613 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5614 reset_list);
5615
5616 reset_context->reset_device_list = device_list_handle;
5617 r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5618 /* If reset handler not implemented, continue; otherwise return */
5619 if (r == -EOPNOTSUPP)
5620 r = 0;
5621 else
5622 return r;
5623
5624 /* Reset handler not implemented, use the default method */
5625 need_full_reset =
5626 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5627 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5628
5629 /*
5630 * ASIC reset has to be done on all XGMI hive nodes ASAP
5631 * to allow proper links negotiation in FW (within 1 sec)
5632 */
5633 if (!skip_hw_reset && need_full_reset) {
5634 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5635 /* For XGMI run all resets in parallel to speed up the process */
5636 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5637 if (!queue_work(system_unbound_wq,
5638 &tmp_adev->xgmi_reset_work))
5639 r = -EALREADY;
5640 } else
5641 r = amdgpu_asic_reset(tmp_adev);
5642
5643 if (r) {
5644 dev_err(tmp_adev->dev,
5645 "ASIC reset failed with error, %d for drm dev, %s",
5646 r, adev_to_drm(tmp_adev)->unique);
5647 goto out;
5648 }
5649 }
5650
5651 /* For XGMI wait for all resets to complete before proceed */
5652 if (!r) {
5653 list_for_each_entry(tmp_adev, device_list_handle,
5654 reset_list) {
5655 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5656 flush_work(&tmp_adev->xgmi_reset_work);
5657 r = tmp_adev->asic_reset_res;
5658 if (r)
5659 break;
5660 }
5661 }
5662 }
5663 }
5664
5665 if (!r && amdgpu_ras_intr_triggered()) {
5666 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5667 amdgpu_ras_reset_error_count(tmp_adev,
5668 AMDGPU_RAS_BLOCK__MMHUB);
5669 }
5670
5671 amdgpu_ras_intr_cleared();
5672 }
5673
5674 r = amdgpu_device_reinit_after_reset(reset_context);
5675 if (r == -EAGAIN)
5676 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5677 else
5678 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5679
5680out:
5681 return r;
5682}
5683
5684static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5685{
5686
5687 switch (amdgpu_asic_reset_method(adev)) {
5688 case AMD_RESET_METHOD_MODE1:
5689 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5690 break;
5691 case AMD_RESET_METHOD_MODE2:
5692 adev->mp1_state = PP_MP1_STATE_RESET;
5693 break;
5694 default:
5695 adev->mp1_state = PP_MP1_STATE_NONE;
5696 break;
5697 }
5698}
5699
5700static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5701{
5702 amdgpu_vf_error_trans_all(adev);
5703 adev->mp1_state = PP_MP1_STATE_NONE;
5704}
5705
5706static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5707{
5708 struct pci_dev *p = NULL;
5709
5710 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5711 adev->pdev->bus->number, 1);
5712 if (p) {
5713 pm_runtime_enable(&(p->dev));
5714 pm_runtime_resume(&(p->dev));
5715 }
5716
5717 pci_dev_put(p);
5718}
5719
5720static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5721{
5722 enum amd_reset_method reset_method;
5723 struct pci_dev *p = NULL;
5724 u64 expires;
5725
5726 /*
5727 * For now, only BACO and mode1 reset are confirmed
5728 * to suffer the audio issue without proper suspended.
5729 */
5730 reset_method = amdgpu_asic_reset_method(adev);
5731 if ((reset_method != AMD_RESET_METHOD_BACO) &&
5732 (reset_method != AMD_RESET_METHOD_MODE1))
5733 return -EINVAL;
5734
5735 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5736 adev->pdev->bus->number, 1);
5737 if (!p)
5738 return -ENODEV;
5739
5740 expires = pm_runtime_autosuspend_expiration(&(p->dev));
5741 if (!expires)
5742 /*
5743 * If we cannot get the audio device autosuspend delay,
5744 * a fixed 4S interval will be used. Considering 3S is
5745 * the audio controller default autosuspend delay setting.
5746 * 4S used here is guaranteed to cover that.
5747 */
5748 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5749
5750 while (!pm_runtime_status_suspended(&(p->dev))) {
5751 if (!pm_runtime_suspend(&(p->dev)))
5752 break;
5753
5754 if (expires < ktime_get_mono_fast_ns()) {
5755 dev_warn(adev->dev, "failed to suspend display audio\n");
5756 pci_dev_put(p);
5757 /* TODO: abort the succeeding gpu reset? */
5758 return -ETIMEDOUT;
5759 }
5760 }
5761
5762 pm_runtime_disable(&(p->dev));
5763
5764 pci_dev_put(p);
5765 return 0;
5766}
5767
5768static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5769{
5770 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5771
5772#if defined(CONFIG_DEBUG_FS)
5773 if (!amdgpu_sriov_vf(adev))
5774 cancel_work(&adev->reset_work);
5775#endif
5776
5777 if (adev->kfd.dev)
5778 cancel_work(&adev->kfd.reset_work);
5779
5780 if (amdgpu_sriov_vf(adev))
5781 cancel_work(&adev->virt.flr_work);
5782
5783 if (con && adev->ras_enabled)
5784 cancel_work(&con->recovery_work);
5785
5786}
5787
5788static int amdgpu_device_health_check(struct list_head *device_list_handle)
5789{
5790 struct amdgpu_device *tmp_adev;
5791 int ret = 0;
5792 u32 status;
5793
5794 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5795 pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status);
5796 if (PCI_POSSIBLE_ERROR(status)) {
5797 dev_err(tmp_adev->dev, "device lost from bus!");
5798 ret = -ENODEV;
5799 }
5800 }
5801
5802 return ret;
5803}
5804
5805/**
5806 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5807 *
5808 * @adev: amdgpu_device pointer
5809 * @job: which job trigger hang
5810 * @reset_context: amdgpu reset context pointer
5811 *
5812 * Attempt to reset the GPU if it has hung (all asics).
5813 * Attempt to do soft-reset or full-reset and reinitialize Asic
5814 * Returns 0 for success or an error on failure.
5815 */
5816
5817int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5818 struct amdgpu_job *job,
5819 struct amdgpu_reset_context *reset_context)
5820{
5821 struct list_head device_list, *device_list_handle = NULL;
5822 bool job_signaled = false;
5823 struct amdgpu_hive_info *hive = NULL;
5824 struct amdgpu_device *tmp_adev = NULL;
5825 int i, r = 0;
5826 bool need_emergency_restart = false;
5827 bool audio_suspended = false;
5828 int retry_limit = AMDGPU_MAX_RETRY_LIMIT;
5829
5830 /*
5831 * Special case: RAS triggered and full reset isn't supported
5832 */
5833 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5834
5835 /*
5836 * Flush RAM to disk so that after reboot
5837 * the user can read log and see why the system rebooted.
5838 */
5839 if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5840 amdgpu_ras_get_context(adev)->reboot) {
5841 DRM_WARN("Emergency reboot.");
5842
5843 ksys_sync_helper();
5844 emergency_restart();
5845 }
5846
5847 dev_info(adev->dev, "GPU %s begin!\n",
5848 need_emergency_restart ? "jobs stop":"reset");
5849
5850 if (!amdgpu_sriov_vf(adev))
5851 hive = amdgpu_get_xgmi_hive(adev);
5852 if (hive)
5853 mutex_lock(&hive->hive_lock);
5854
5855 reset_context->job = job;
5856 reset_context->hive = hive;
5857 /*
5858 * Build list of devices to reset.
5859 * In case we are in XGMI hive mode, resort the device list
5860 * to put adev in the 1st position.
5861 */
5862 INIT_LIST_HEAD(&device_list);
5863 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
5864 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5865 list_add_tail(&tmp_adev->reset_list, &device_list);
5866 if (adev->shutdown)
5867 tmp_adev->shutdown = true;
5868 }
5869 if (!list_is_first(&adev->reset_list, &device_list))
5870 list_rotate_to_front(&adev->reset_list, &device_list);
5871 device_list_handle = &device_list;
5872 } else {
5873 list_add_tail(&adev->reset_list, &device_list);
5874 device_list_handle = &device_list;
5875 }
5876
5877 if (!amdgpu_sriov_vf(adev)) {
5878 r = amdgpu_device_health_check(device_list_handle);
5879 if (r)
5880 goto end_reset;
5881 }
5882
5883 /* We need to lock reset domain only once both for XGMI and single device */
5884 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5885 reset_list);
5886 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5887
5888 /* block all schedulers and reset given job's ring */
5889 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5890
5891 amdgpu_device_set_mp1_state(tmp_adev);
5892
5893 /*
5894 * Try to put the audio codec into suspend state
5895 * before gpu reset started.
5896 *
5897 * Due to the power domain of the graphics device
5898 * is shared with AZ power domain. Without this,
5899 * we may change the audio hardware from behind
5900 * the audio driver's back. That will trigger
5901 * some audio codec errors.
5902 */
5903 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5904 audio_suspended = true;
5905
5906 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5907
5908 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5909
5910 amdgpu_amdkfd_pre_reset(tmp_adev, reset_context);
5911
5912 /*
5913 * Mark these ASICs to be reseted as untracked first
5914 * And add them back after reset completed
5915 */
5916 amdgpu_unregister_gpu_instance(tmp_adev);
5917
5918 drm_client_dev_suspend(adev_to_drm(tmp_adev), false);
5919
5920 /* disable ras on ALL IPs */
5921 if (!need_emergency_restart &&
5922 amdgpu_device_ip_need_full_reset(tmp_adev))
5923 amdgpu_ras_suspend(tmp_adev);
5924
5925 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5926 struct amdgpu_ring *ring = tmp_adev->rings[i];
5927
5928 if (!amdgpu_ring_sched_ready(ring))
5929 continue;
5930
5931 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5932
5933 if (need_emergency_restart)
5934 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5935 }
5936 atomic_inc(&tmp_adev->gpu_reset_counter);
5937 }
5938
5939 if (need_emergency_restart)
5940 goto skip_sched_resume;
5941
5942 /*
5943 * Must check guilty signal here since after this point all old
5944 * HW fences are force signaled.
5945 *
5946 * job->base holds a reference to parent fence
5947 */
5948 if (job && dma_fence_is_signaled(&job->hw_fence)) {
5949 job_signaled = true;
5950 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5951 goto skip_hw_reset;
5952 }
5953
5954retry: /* Rest of adevs pre asic reset from XGMI hive. */
5955 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5956 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5957 /*TODO Should we stop ?*/
5958 if (r) {
5959 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5960 r, adev_to_drm(tmp_adev)->unique);
5961 tmp_adev->asic_reset_res = r;
5962 }
5963 }
5964
5965 /* Actual ASIC resets if needed.*/
5966 /* Host driver will handle XGMI hive reset for SRIOV */
5967 if (amdgpu_sriov_vf(adev)) {
5968 if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
5969 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
5970 amdgpu_ras_set_fed(adev, true);
5971 set_bit(AMDGPU_HOST_FLR, &reset_context->flags);
5972 }
5973
5974 r = amdgpu_device_reset_sriov(adev, reset_context);
5975 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) {
5976 amdgpu_virt_release_full_gpu(adev, true);
5977 goto retry;
5978 }
5979 if (r)
5980 adev->asic_reset_res = r;
5981 } else {
5982 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5983 if (r && r == -EAGAIN)
5984 goto retry;
5985 }
5986
5987 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5988 /*
5989 * Drop any pending non scheduler resets queued before reset is done.
5990 * Any reset scheduled after this point would be valid. Scheduler resets
5991 * were already dropped during drm_sched_stop and no new ones can come
5992 * in before drm_sched_start.
5993 */
5994 amdgpu_device_stop_pending_resets(tmp_adev);
5995 }
5996
5997skip_hw_reset:
5998
5999 /* Post ASIC reset for all devs .*/
6000 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6001
6002 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6003 struct amdgpu_ring *ring = tmp_adev->rings[i];
6004
6005 if (!amdgpu_ring_sched_ready(ring))
6006 continue;
6007
6008 drm_sched_start(&ring->sched, 0);
6009 }
6010
6011 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
6012 drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
6013
6014 if (tmp_adev->asic_reset_res)
6015 r = tmp_adev->asic_reset_res;
6016
6017 tmp_adev->asic_reset_res = 0;
6018
6019 if (r) {
6020 /* bad news, how to tell it to userspace ?
6021 * for ras error, we should report GPU bad status instead of
6022 * reset failure
6023 */
6024 if (reset_context->src != AMDGPU_RESET_SRC_RAS ||
6025 !amdgpu_ras_eeprom_check_err_threshold(tmp_adev))
6026 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n",
6027 atomic_read(&tmp_adev->gpu_reset_counter));
6028 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
6029 } else {
6030 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
6031 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
6032 DRM_WARN("smart shift update failed\n");
6033 }
6034 }
6035
6036skip_sched_resume:
6037 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6038 /* unlock kfd: SRIOV would do it separately */
6039 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
6040 amdgpu_amdkfd_post_reset(tmp_adev);
6041
6042 /* kfd_post_reset will do nothing if kfd device is not initialized,
6043 * need to bring up kfd here if it's not be initialized before
6044 */
6045 if (!adev->kfd.init_complete)
6046 amdgpu_amdkfd_device_init(adev);
6047
6048 if (audio_suspended)
6049 amdgpu_device_resume_display_audio(tmp_adev);
6050
6051 amdgpu_device_unset_mp1_state(tmp_adev);
6052
6053 amdgpu_ras_set_error_query_ready(tmp_adev, true);
6054 }
6055
6056 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
6057 reset_list);
6058 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
6059
6060end_reset:
6061 if (hive) {
6062 mutex_unlock(&hive->hive_lock);
6063 amdgpu_put_xgmi_hive(hive);
6064 }
6065
6066 if (r)
6067 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
6068
6069 atomic_set(&adev->reset_domain->reset_res, r);
6070 return r;
6071}
6072
6073/**
6074 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
6075 *
6076 * @adev: amdgpu_device pointer
6077 * @speed: pointer to the speed of the link
6078 * @width: pointer to the width of the link
6079 *
6080 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
6081 * first physical partner to an AMD dGPU.
6082 * This will exclude any virtual switches and links.
6083 */
6084static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
6085 enum pci_bus_speed *speed,
6086 enum pcie_link_width *width)
6087{
6088 struct pci_dev *parent = adev->pdev;
6089
6090 if (!speed || !width)
6091 return;
6092
6093 *speed = PCI_SPEED_UNKNOWN;
6094 *width = PCIE_LNK_WIDTH_UNKNOWN;
6095
6096 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
6097 while ((parent = pci_upstream_bridge(parent))) {
6098 /* skip upstream/downstream switches internal to dGPU*/
6099 if (parent->vendor == PCI_VENDOR_ID_ATI)
6100 continue;
6101 *speed = pcie_get_speed_cap(parent);
6102 *width = pcie_get_width_cap(parent);
6103 break;
6104 }
6105 } else {
6106 /* use the current speeds rather than max if switching is not supported */
6107 pcie_bandwidth_available(adev->pdev, NULL, speed, width);
6108 }
6109}
6110
6111/**
6112 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
6113 *
6114 * @adev: amdgpu_device pointer
6115 *
6116 * Fetchs and stores in the driver the PCIE capabilities (gen speed
6117 * and lanes) of the slot the device is in. Handles APUs and
6118 * virtualized environments where PCIE config space may not be available.
6119 */
6120static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
6121{
6122 struct pci_dev *pdev;
6123 enum pci_bus_speed speed_cap, platform_speed_cap;
6124 enum pcie_link_width platform_link_width;
6125
6126 if (amdgpu_pcie_gen_cap)
6127 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
6128
6129 if (amdgpu_pcie_lane_cap)
6130 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
6131
6132 /* covers APUs as well */
6133 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
6134 if (adev->pm.pcie_gen_mask == 0)
6135 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
6136 if (adev->pm.pcie_mlw_mask == 0)
6137 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
6138 return;
6139 }
6140
6141 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
6142 return;
6143
6144 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
6145 &platform_link_width);
6146
6147 if (adev->pm.pcie_gen_mask == 0) {
6148 /* asic caps */
6149 pdev = adev->pdev;
6150 speed_cap = pcie_get_speed_cap(pdev);
6151 if (speed_cap == PCI_SPEED_UNKNOWN) {
6152 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6153 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6154 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
6155 } else {
6156 if (speed_cap == PCIE_SPEED_32_0GT)
6157 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6158 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6159 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6160 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
6161 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
6162 else if (speed_cap == PCIE_SPEED_16_0GT)
6163 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6164 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6165 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6166 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
6167 else if (speed_cap == PCIE_SPEED_8_0GT)
6168 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6169 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6170 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
6171 else if (speed_cap == PCIE_SPEED_5_0GT)
6172 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6173 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
6174 else
6175 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
6176 }
6177 /* platform caps */
6178 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
6179 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6180 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6181 } else {
6182 if (platform_speed_cap == PCIE_SPEED_32_0GT)
6183 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6184 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6185 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6186 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
6187 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
6188 else if (platform_speed_cap == PCIE_SPEED_16_0GT)
6189 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6190 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6191 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6192 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
6193 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
6194 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6195 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6196 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
6197 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
6198 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6199 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6200 else
6201 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
6202
6203 }
6204 }
6205 if (adev->pm.pcie_mlw_mask == 0) {
6206 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6207 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
6208 } else {
6209 switch (platform_link_width) {
6210 case PCIE_LNK_X32:
6211 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
6212 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6213 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6214 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6215 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6216 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6217 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6218 break;
6219 case PCIE_LNK_X16:
6220 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6221 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6222 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6223 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6224 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6225 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6226 break;
6227 case PCIE_LNK_X12:
6228 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6229 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6230 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6231 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6232 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6233 break;
6234 case PCIE_LNK_X8:
6235 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6236 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6237 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6238 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6239 break;
6240 case PCIE_LNK_X4:
6241 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6242 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6243 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6244 break;
6245 case PCIE_LNK_X2:
6246 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6247 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6248 break;
6249 case PCIE_LNK_X1:
6250 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
6251 break;
6252 default:
6253 break;
6254 }
6255 }
6256 }
6257}
6258
6259/**
6260 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6261 *
6262 * @adev: amdgpu_device pointer
6263 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6264 *
6265 * Return true if @peer_adev can access (DMA) @adev through the PCIe
6266 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
6267 * @peer_adev.
6268 */
6269bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
6270 struct amdgpu_device *peer_adev)
6271{
6272#ifdef CONFIG_HSA_AMD_P2P
6273 bool p2p_access =
6274 !adev->gmc.xgmi.connected_to_cpu &&
6275 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
6276 if (!p2p_access)
6277 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n",
6278 pci_name(peer_adev->pdev));
6279
6280 bool is_large_bar = adev->gmc.visible_vram_size &&
6281 adev->gmc.real_vram_size == adev->gmc.visible_vram_size;
6282 bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev);
6283
6284 if (!p2p_addressable) {
6285 uint64_t address_mask = peer_adev->dev->dma_mask ?
6286 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
6287 resource_size_t aper_limit =
6288 adev->gmc.aper_base + adev->gmc.aper_size - 1;
6289
6290 p2p_addressable = !(adev->gmc.aper_base & address_mask ||
6291 aper_limit & address_mask);
6292 }
6293 return pcie_p2p && is_large_bar && p2p_access && p2p_addressable;
6294#else
6295 return false;
6296#endif
6297}
6298
6299int amdgpu_device_baco_enter(struct drm_device *dev)
6300{
6301 struct amdgpu_device *adev = drm_to_adev(dev);
6302 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6303
6304 if (!amdgpu_device_supports_baco(dev))
6305 return -ENOTSUPP;
6306
6307 if (ras && adev->ras_enabled &&
6308 adev->nbio.funcs->enable_doorbell_interrupt)
6309 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6310
6311 return amdgpu_dpm_baco_enter(adev);
6312}
6313
6314int amdgpu_device_baco_exit(struct drm_device *dev)
6315{
6316 struct amdgpu_device *adev = drm_to_adev(dev);
6317 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6318 int ret = 0;
6319
6320 if (!amdgpu_device_supports_baco(dev))
6321 return -ENOTSUPP;
6322
6323 ret = amdgpu_dpm_baco_exit(adev);
6324 if (ret)
6325 return ret;
6326
6327 if (ras && adev->ras_enabled &&
6328 adev->nbio.funcs->enable_doorbell_interrupt)
6329 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6330
6331 if (amdgpu_passthrough(adev) && adev->nbio.funcs &&
6332 adev->nbio.funcs->clear_doorbell_interrupt)
6333 adev->nbio.funcs->clear_doorbell_interrupt(adev);
6334
6335 return 0;
6336}
6337
6338/**
6339 * amdgpu_pci_error_detected - Called when a PCI error is detected.
6340 * @pdev: PCI device struct
6341 * @state: PCI channel state
6342 *
6343 * Description: Called when a PCI error is detected.
6344 *
6345 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6346 */
6347pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6348{
6349 struct drm_device *dev = pci_get_drvdata(pdev);
6350 struct amdgpu_device *adev = drm_to_adev(dev);
6351 int i;
6352
6353 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6354
6355 if (adev->gmc.xgmi.num_physical_nodes > 1) {
6356 DRM_WARN("No support for XGMI hive yet...");
6357 return PCI_ERS_RESULT_DISCONNECT;
6358 }
6359
6360 adev->pci_channel_state = state;
6361
6362 switch (state) {
6363 case pci_channel_io_normal:
6364 return PCI_ERS_RESULT_CAN_RECOVER;
6365 /* Fatal error, prepare for slot reset */
6366 case pci_channel_io_frozen:
6367 /*
6368 * Locking adev->reset_domain->sem will prevent any external access
6369 * to GPU during PCI error recovery
6370 */
6371 amdgpu_device_lock_reset_domain(adev->reset_domain);
6372 amdgpu_device_set_mp1_state(adev);
6373
6374 /*
6375 * Block any work scheduling as we do for regular GPU reset
6376 * for the duration of the recovery
6377 */
6378 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6379 struct amdgpu_ring *ring = adev->rings[i];
6380
6381 if (!amdgpu_ring_sched_ready(ring))
6382 continue;
6383
6384 drm_sched_stop(&ring->sched, NULL);
6385 }
6386 atomic_inc(&adev->gpu_reset_counter);
6387 return PCI_ERS_RESULT_NEED_RESET;
6388 case pci_channel_io_perm_failure:
6389 /* Permanent error, prepare for device removal */
6390 return PCI_ERS_RESULT_DISCONNECT;
6391 }
6392
6393 return PCI_ERS_RESULT_NEED_RESET;
6394}
6395
6396/**
6397 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6398 * @pdev: pointer to PCI device
6399 */
6400pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6401{
6402
6403 DRM_INFO("PCI error: mmio enabled callback!!\n");
6404
6405 /* TODO - dump whatever for debugging purposes */
6406
6407 /* This called only if amdgpu_pci_error_detected returns
6408 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6409 * works, no need to reset slot.
6410 */
6411
6412 return PCI_ERS_RESULT_RECOVERED;
6413}
6414
6415/**
6416 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6417 * @pdev: PCI device struct
6418 *
6419 * Description: This routine is called by the pci error recovery
6420 * code after the PCI slot has been reset, just before we
6421 * should resume normal operations.
6422 */
6423pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6424{
6425 struct drm_device *dev = pci_get_drvdata(pdev);
6426 struct amdgpu_device *adev = drm_to_adev(dev);
6427 int r, i;
6428 struct amdgpu_reset_context reset_context;
6429 u32 memsize;
6430 struct list_head device_list;
6431
6432 /* PCI error slot reset should be skipped During RAS recovery */
6433 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
6434 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) &&
6435 amdgpu_ras_in_recovery(adev))
6436 return PCI_ERS_RESULT_RECOVERED;
6437
6438 DRM_INFO("PCI error: slot reset callback!!\n");
6439
6440 memset(&reset_context, 0, sizeof(reset_context));
6441
6442 INIT_LIST_HEAD(&device_list);
6443 list_add_tail(&adev->reset_list, &device_list);
6444
6445 /* wait for asic to come out of reset */
6446 msleep(500);
6447
6448 /* Restore PCI confspace */
6449 amdgpu_device_load_pci_state(pdev);
6450
6451 /* confirm ASIC came out of reset */
6452 for (i = 0; i < adev->usec_timeout; i++) {
6453 memsize = amdgpu_asic_get_config_memsize(adev);
6454
6455 if (memsize != 0xffffffff)
6456 break;
6457 udelay(1);
6458 }
6459 if (memsize == 0xffffffff) {
6460 r = -ETIME;
6461 goto out;
6462 }
6463
6464 reset_context.method = AMD_RESET_METHOD_NONE;
6465 reset_context.reset_req_dev = adev;
6466 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6467 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6468
6469 adev->no_hw_access = true;
6470 r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6471 adev->no_hw_access = false;
6472 if (r)
6473 goto out;
6474
6475 r = amdgpu_do_asic_reset(&device_list, &reset_context);
6476
6477out:
6478 if (!r) {
6479 if (amdgpu_device_cache_pci_state(adev->pdev))
6480 pci_restore_state(adev->pdev);
6481
6482 DRM_INFO("PCIe error recovery succeeded\n");
6483 } else {
6484 DRM_ERROR("PCIe error recovery failed, err:%d", r);
6485 amdgpu_device_unset_mp1_state(adev);
6486 amdgpu_device_unlock_reset_domain(adev->reset_domain);
6487 }
6488
6489 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6490}
6491
6492/**
6493 * amdgpu_pci_resume() - resume normal ops after PCI reset
6494 * @pdev: pointer to PCI device
6495 *
6496 * Called when the error recovery driver tells us that its
6497 * OK to resume normal operation.
6498 */
6499void amdgpu_pci_resume(struct pci_dev *pdev)
6500{
6501 struct drm_device *dev = pci_get_drvdata(pdev);
6502 struct amdgpu_device *adev = drm_to_adev(dev);
6503 int i;
6504
6505
6506 DRM_INFO("PCI error: resume callback!!\n");
6507
6508 /* Only continue execution for the case of pci_channel_io_frozen */
6509 if (adev->pci_channel_state != pci_channel_io_frozen)
6510 return;
6511
6512 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6513 struct amdgpu_ring *ring = adev->rings[i];
6514
6515 if (!amdgpu_ring_sched_ready(ring))
6516 continue;
6517
6518 drm_sched_start(&ring->sched, 0);
6519 }
6520
6521 amdgpu_device_unset_mp1_state(adev);
6522 amdgpu_device_unlock_reset_domain(adev->reset_domain);
6523}
6524
6525bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6526{
6527 struct drm_device *dev = pci_get_drvdata(pdev);
6528 struct amdgpu_device *adev = drm_to_adev(dev);
6529 int r;
6530
6531 if (amdgpu_sriov_vf(adev))
6532 return false;
6533
6534 r = pci_save_state(pdev);
6535 if (!r) {
6536 kfree(adev->pci_state);
6537
6538 adev->pci_state = pci_store_saved_state(pdev);
6539
6540 if (!adev->pci_state) {
6541 DRM_ERROR("Failed to store PCI saved state");
6542 return false;
6543 }
6544 } else {
6545 DRM_WARN("Failed to save PCI state, err:%d\n", r);
6546 return false;
6547 }
6548
6549 return true;
6550}
6551
6552bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6553{
6554 struct drm_device *dev = pci_get_drvdata(pdev);
6555 struct amdgpu_device *adev = drm_to_adev(dev);
6556 int r;
6557
6558 if (!adev->pci_state)
6559 return false;
6560
6561 r = pci_load_saved_state(pdev, adev->pci_state);
6562
6563 if (!r) {
6564 pci_restore_state(pdev);
6565 } else {
6566 DRM_WARN("Failed to load PCI state, err:%d\n", r);
6567 return false;
6568 }
6569
6570 return true;
6571}
6572
6573void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6574 struct amdgpu_ring *ring)
6575{
6576#ifdef CONFIG_X86_64
6577 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6578 return;
6579#endif
6580 if (adev->gmc.xgmi.connected_to_cpu)
6581 return;
6582
6583 if (ring && ring->funcs->emit_hdp_flush)
6584 amdgpu_ring_emit_hdp_flush(ring);
6585 else
6586 amdgpu_asic_flush_hdp(adev, ring);
6587}
6588
6589void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6590 struct amdgpu_ring *ring)
6591{
6592#ifdef CONFIG_X86_64
6593 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6594 return;
6595#endif
6596 if (adev->gmc.xgmi.connected_to_cpu)
6597 return;
6598
6599 amdgpu_asic_invalidate_hdp(adev, ring);
6600}
6601
6602int amdgpu_in_reset(struct amdgpu_device *adev)
6603{
6604 return atomic_read(&adev->reset_domain->in_gpu_reset);
6605}
6606
6607/**
6608 * amdgpu_device_halt() - bring hardware to some kind of halt state
6609 *
6610 * @adev: amdgpu_device pointer
6611 *
6612 * Bring hardware to some kind of halt state so that no one can touch it
6613 * any more. It will help to maintain error context when error occurred.
6614 * Compare to a simple hang, the system will keep stable at least for SSH
6615 * access. Then it should be trivial to inspect the hardware state and
6616 * see what's going on. Implemented as following:
6617 *
6618 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6619 * clears all CPU mappings to device, disallows remappings through page faults
6620 * 2. amdgpu_irq_disable_all() disables all interrupts
6621 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6622 * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6623 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6624 * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6625 * flush any in flight DMA operations
6626 */
6627void amdgpu_device_halt(struct amdgpu_device *adev)
6628{
6629 struct pci_dev *pdev = adev->pdev;
6630 struct drm_device *ddev = adev_to_drm(adev);
6631
6632 amdgpu_xcp_dev_unplug(adev);
6633 drm_dev_unplug(ddev);
6634
6635 amdgpu_irq_disable_all(adev);
6636
6637 amdgpu_fence_driver_hw_fini(adev);
6638
6639 adev->no_hw_access = true;
6640
6641 amdgpu_device_unmap_mmio(adev);
6642
6643 pci_disable_device(pdev);
6644 pci_wait_for_pending_transaction(pdev);
6645}
6646
6647u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6648 u32 reg)
6649{
6650 unsigned long flags, address, data;
6651 u32 r;
6652
6653 address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6654 data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6655
6656 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6657 WREG32(address, reg * 4);
6658 (void)RREG32(address);
6659 r = RREG32(data);
6660 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6661 return r;
6662}
6663
6664void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6665 u32 reg, u32 v)
6666{
6667 unsigned long flags, address, data;
6668
6669 address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6670 data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6671
6672 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6673 WREG32(address, reg * 4);
6674 (void)RREG32(address);
6675 WREG32(data, v);
6676 (void)RREG32(data);
6677 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6678}
6679
6680/**
6681 * amdgpu_device_get_gang - return a reference to the current gang
6682 * @adev: amdgpu_device pointer
6683 *
6684 * Returns: A new reference to the current gang leader.
6685 */
6686struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)
6687{
6688 struct dma_fence *fence;
6689
6690 rcu_read_lock();
6691 fence = dma_fence_get_rcu_safe(&adev->gang_submit);
6692 rcu_read_unlock();
6693 return fence;
6694}
6695
6696/**
6697 * amdgpu_device_switch_gang - switch to a new gang
6698 * @adev: amdgpu_device pointer
6699 * @gang: the gang to switch to
6700 *
6701 * Try to switch to a new gang.
6702 * Returns: NULL if we switched to the new gang or a reference to the current
6703 * gang leader.
6704 */
6705struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6706 struct dma_fence *gang)
6707{
6708 struct dma_fence *old = NULL;
6709
6710 do {
6711 dma_fence_put(old);
6712 old = amdgpu_device_get_gang(adev);
6713 if (old == gang)
6714 break;
6715
6716 if (!dma_fence_is_signaled(old))
6717 return old;
6718
6719 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6720 old, gang) != old);
6721
6722 dma_fence_put(old);
6723 return NULL;
6724}
6725
6726bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6727{
6728 switch (adev->asic_type) {
6729#ifdef CONFIG_DRM_AMDGPU_SI
6730 case CHIP_HAINAN:
6731#endif
6732 case CHIP_TOPAZ:
6733 /* chips with no display hardware */
6734 return false;
6735#ifdef CONFIG_DRM_AMDGPU_SI
6736 case CHIP_TAHITI:
6737 case CHIP_PITCAIRN:
6738 case CHIP_VERDE:
6739 case CHIP_OLAND:
6740#endif
6741#ifdef CONFIG_DRM_AMDGPU_CIK
6742 case CHIP_BONAIRE:
6743 case CHIP_HAWAII:
6744 case CHIP_KAVERI:
6745 case CHIP_KABINI:
6746 case CHIP_MULLINS:
6747#endif
6748 case CHIP_TONGA:
6749 case CHIP_FIJI:
6750 case CHIP_POLARIS10:
6751 case CHIP_POLARIS11:
6752 case CHIP_POLARIS12:
6753 case CHIP_VEGAM:
6754 case CHIP_CARRIZO:
6755 case CHIP_STONEY:
6756 /* chips with display hardware */
6757 return true;
6758 default:
6759 /* IP discovery */
6760 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6761 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6762 return false;
6763 return true;
6764 }
6765}
6766
6767uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6768 uint32_t inst, uint32_t reg_addr, char reg_name[],
6769 uint32_t expected_value, uint32_t mask)
6770{
6771 uint32_t ret = 0;
6772 uint32_t old_ = 0;
6773 uint32_t tmp_ = RREG32(reg_addr);
6774 uint32_t loop = adev->usec_timeout;
6775
6776 while ((tmp_ & (mask)) != (expected_value)) {
6777 if (old_ != tmp_) {
6778 loop = adev->usec_timeout;
6779 old_ = tmp_;
6780 } else
6781 udelay(1);
6782 tmp_ = RREG32(reg_addr);
6783 loop--;
6784 if (!loop) {
6785 DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6786 inst, reg_name, (uint32_t)expected_value,
6787 (uint32_t)(tmp_ & (mask)));
6788 ret = -ETIMEDOUT;
6789 break;
6790 }
6791 }
6792 return ret;
6793}
6794
6795ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
6796{
6797 ssize_t size = 0;
6798
6799 if (!ring || !ring->adev)
6800 return size;
6801
6802 if (amdgpu_device_should_recover_gpu(ring->adev))
6803 size |= AMDGPU_RESET_TYPE_FULL;
6804
6805 if (unlikely(!ring->adev->debug_disable_soft_recovery) &&
6806 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)
6807 size |= AMDGPU_RESET_TYPE_SOFT_RESET;
6808
6809 return size;
6810}
6811
6812ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset)
6813{
6814 ssize_t size = 0;
6815
6816 if (supported_reset == 0) {
6817 size += sysfs_emit_at(buf, size, "unsupported");
6818 size += sysfs_emit_at(buf, size, "\n");
6819 return size;
6820
6821 }
6822
6823 if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET)
6824 size += sysfs_emit_at(buf, size, "soft ");
6825
6826 if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)
6827 size += sysfs_emit_at(buf, size, "queue ");
6828
6829 if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE)
6830 size += sysfs_emit_at(buf, size, "pipe ");
6831
6832 if (supported_reset & AMDGPU_RESET_TYPE_FULL)
6833 size += sysfs_emit_at(buf, size, "full ");
6834
6835 size += sysfs_emit_at(buf, size, "\n");
6836 return size;
6837}