Loading...
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/slab.h>
26#include <drm/drmP.h>
27#include "radeon.h"
28#include "radeon_asic.h"
29#include "radeon_audio.h"
30#include <drm/radeon_drm.h>
31#include "evergreend.h"
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
35#include "evergreen_blit_shaders.h"
36#include "radeon_ucode.h"
37
38/*
39 * Indirect registers accessor
40 */
41u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
42{
43 unsigned long flags;
44 u32 r;
45
46 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
47 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
48 r = RREG32(EVERGREEN_CG_IND_DATA);
49 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
50 return r;
51}
52
53void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
54{
55 unsigned long flags;
56
57 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
58 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
59 WREG32(EVERGREEN_CG_IND_DATA, (v));
60 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
61}
62
63u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
64{
65 unsigned long flags;
66 u32 r;
67
68 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
69 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
70 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
71 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
72 return r;
73}
74
75void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
76{
77 unsigned long flags;
78
79 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
80 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
81 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
82 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
83}
84
85u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
86{
87 unsigned long flags;
88 u32 r;
89
90 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
91 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
92 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
93 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
94 return r;
95}
96
97void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
98{
99 unsigned long flags;
100
101 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
102 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
103 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
104 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
105}
106
107static const u32 crtc_offsets[6] =
108{
109 EVERGREEN_CRTC0_REGISTER_OFFSET,
110 EVERGREEN_CRTC1_REGISTER_OFFSET,
111 EVERGREEN_CRTC2_REGISTER_OFFSET,
112 EVERGREEN_CRTC3_REGISTER_OFFSET,
113 EVERGREEN_CRTC4_REGISTER_OFFSET,
114 EVERGREEN_CRTC5_REGISTER_OFFSET
115};
116
117#include "clearstate_evergreen.h"
118
119static const u32 sumo_rlc_save_restore_register_list[] =
120{
121 0x98fc,
122 0x9830,
123 0x9834,
124 0x9838,
125 0x9870,
126 0x9874,
127 0x8a14,
128 0x8b24,
129 0x8bcc,
130 0x8b10,
131 0x8d00,
132 0x8d04,
133 0x8c00,
134 0x8c04,
135 0x8c08,
136 0x8c0c,
137 0x8d8c,
138 0x8c20,
139 0x8c24,
140 0x8c28,
141 0x8c18,
142 0x8c1c,
143 0x8cf0,
144 0x8e2c,
145 0x8e38,
146 0x8c30,
147 0x9508,
148 0x9688,
149 0x9608,
150 0x960c,
151 0x9610,
152 0x9614,
153 0x88c4,
154 0x88d4,
155 0xa008,
156 0x900c,
157 0x9100,
158 0x913c,
159 0x98f8,
160 0x98f4,
161 0x9b7c,
162 0x3f8c,
163 0x8950,
164 0x8954,
165 0x8a18,
166 0x8b28,
167 0x9144,
168 0x9148,
169 0x914c,
170 0x3f90,
171 0x3f94,
172 0x915c,
173 0x9160,
174 0x9178,
175 0x917c,
176 0x9180,
177 0x918c,
178 0x9190,
179 0x9194,
180 0x9198,
181 0x919c,
182 0x91a8,
183 0x91ac,
184 0x91b0,
185 0x91b4,
186 0x91b8,
187 0x91c4,
188 0x91c8,
189 0x91cc,
190 0x91d0,
191 0x91d4,
192 0x91e0,
193 0x91e4,
194 0x91ec,
195 0x91f0,
196 0x91f4,
197 0x9200,
198 0x9204,
199 0x929c,
200 0x9150,
201 0x802c,
202};
203
204static void evergreen_gpu_init(struct radeon_device *rdev);
205void evergreen_fini(struct radeon_device *rdev);
206void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
207void evergreen_program_aspm(struct radeon_device *rdev);
208extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
209 int ring, u32 cp_int_cntl);
210extern void cayman_vm_decode_fault(struct radeon_device *rdev,
211 u32 status, u32 addr);
212void cik_init_cp_pg_table(struct radeon_device *rdev);
213
214extern u32 si_get_csb_size(struct radeon_device *rdev);
215extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
216extern u32 cik_get_csb_size(struct radeon_device *rdev);
217extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
218extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
219
220static const u32 evergreen_golden_registers[] =
221{
222 0x3f90, 0xffff0000, 0xff000000,
223 0x9148, 0xffff0000, 0xff000000,
224 0x3f94, 0xffff0000, 0xff000000,
225 0x914c, 0xffff0000, 0xff000000,
226 0x9b7c, 0xffffffff, 0x00000000,
227 0x8a14, 0xffffffff, 0x00000007,
228 0x8b10, 0xffffffff, 0x00000000,
229 0x960c, 0xffffffff, 0x54763210,
230 0x88c4, 0xffffffff, 0x000000c2,
231 0x88d4, 0xffffffff, 0x00000010,
232 0x8974, 0xffffffff, 0x00000000,
233 0xc78, 0x00000080, 0x00000080,
234 0x5eb4, 0xffffffff, 0x00000002,
235 0x5e78, 0xffffffff, 0x001000f0,
236 0x6104, 0x01000300, 0x00000000,
237 0x5bc0, 0x00300000, 0x00000000,
238 0x7030, 0xffffffff, 0x00000011,
239 0x7c30, 0xffffffff, 0x00000011,
240 0x10830, 0xffffffff, 0x00000011,
241 0x11430, 0xffffffff, 0x00000011,
242 0x12030, 0xffffffff, 0x00000011,
243 0x12c30, 0xffffffff, 0x00000011,
244 0xd02c, 0xffffffff, 0x08421000,
245 0x240c, 0xffffffff, 0x00000380,
246 0x8b24, 0xffffffff, 0x00ff0fff,
247 0x28a4c, 0x06000000, 0x06000000,
248 0x10c, 0x00000001, 0x00000001,
249 0x8d00, 0xffffffff, 0x100e4848,
250 0x8d04, 0xffffffff, 0x00164745,
251 0x8c00, 0xffffffff, 0xe4000003,
252 0x8c04, 0xffffffff, 0x40600060,
253 0x8c08, 0xffffffff, 0x001c001c,
254 0x8cf0, 0xffffffff, 0x08e00620,
255 0x8c20, 0xffffffff, 0x00800080,
256 0x8c24, 0xffffffff, 0x00800080,
257 0x8c18, 0xffffffff, 0x20202078,
258 0x8c1c, 0xffffffff, 0x00001010,
259 0x28350, 0xffffffff, 0x00000000,
260 0xa008, 0xffffffff, 0x00010000,
261 0x5c4, 0xffffffff, 0x00000001,
262 0x9508, 0xffffffff, 0x00000002,
263 0x913c, 0x0000000f, 0x0000000a
264};
265
266static const u32 evergreen_golden_registers2[] =
267{
268 0x2f4c, 0xffffffff, 0x00000000,
269 0x54f4, 0xffffffff, 0x00000000,
270 0x54f0, 0xffffffff, 0x00000000,
271 0x5498, 0xffffffff, 0x00000000,
272 0x549c, 0xffffffff, 0x00000000,
273 0x5494, 0xffffffff, 0x00000000,
274 0x53cc, 0xffffffff, 0x00000000,
275 0x53c8, 0xffffffff, 0x00000000,
276 0x53c4, 0xffffffff, 0x00000000,
277 0x53c0, 0xffffffff, 0x00000000,
278 0x53bc, 0xffffffff, 0x00000000,
279 0x53b8, 0xffffffff, 0x00000000,
280 0x53b4, 0xffffffff, 0x00000000,
281 0x53b0, 0xffffffff, 0x00000000
282};
283
284static const u32 cypress_mgcg_init[] =
285{
286 0x802c, 0xffffffff, 0xc0000000,
287 0x5448, 0xffffffff, 0x00000100,
288 0x55e4, 0xffffffff, 0x00000100,
289 0x160c, 0xffffffff, 0x00000100,
290 0x5644, 0xffffffff, 0x00000100,
291 0xc164, 0xffffffff, 0x00000100,
292 0x8a18, 0xffffffff, 0x00000100,
293 0x897c, 0xffffffff, 0x06000100,
294 0x8b28, 0xffffffff, 0x00000100,
295 0x9144, 0xffffffff, 0x00000100,
296 0x9a60, 0xffffffff, 0x00000100,
297 0x9868, 0xffffffff, 0x00000100,
298 0x8d58, 0xffffffff, 0x00000100,
299 0x9510, 0xffffffff, 0x00000100,
300 0x949c, 0xffffffff, 0x00000100,
301 0x9654, 0xffffffff, 0x00000100,
302 0x9030, 0xffffffff, 0x00000100,
303 0x9034, 0xffffffff, 0x00000100,
304 0x9038, 0xffffffff, 0x00000100,
305 0x903c, 0xffffffff, 0x00000100,
306 0x9040, 0xffffffff, 0x00000100,
307 0xa200, 0xffffffff, 0x00000100,
308 0xa204, 0xffffffff, 0x00000100,
309 0xa208, 0xffffffff, 0x00000100,
310 0xa20c, 0xffffffff, 0x00000100,
311 0x971c, 0xffffffff, 0x00000100,
312 0x977c, 0xffffffff, 0x00000100,
313 0x3f80, 0xffffffff, 0x00000100,
314 0xa210, 0xffffffff, 0x00000100,
315 0xa214, 0xffffffff, 0x00000100,
316 0x4d8, 0xffffffff, 0x00000100,
317 0x9784, 0xffffffff, 0x00000100,
318 0x9698, 0xffffffff, 0x00000100,
319 0x4d4, 0xffffffff, 0x00000200,
320 0x30cc, 0xffffffff, 0x00000100,
321 0xd0c0, 0xffffffff, 0xff000100,
322 0x802c, 0xffffffff, 0x40000000,
323 0x915c, 0xffffffff, 0x00010000,
324 0x9160, 0xffffffff, 0x00030002,
325 0x9178, 0xffffffff, 0x00070000,
326 0x917c, 0xffffffff, 0x00030002,
327 0x9180, 0xffffffff, 0x00050004,
328 0x918c, 0xffffffff, 0x00010006,
329 0x9190, 0xffffffff, 0x00090008,
330 0x9194, 0xffffffff, 0x00070000,
331 0x9198, 0xffffffff, 0x00030002,
332 0x919c, 0xffffffff, 0x00050004,
333 0x91a8, 0xffffffff, 0x00010006,
334 0x91ac, 0xffffffff, 0x00090008,
335 0x91b0, 0xffffffff, 0x00070000,
336 0x91b4, 0xffffffff, 0x00030002,
337 0x91b8, 0xffffffff, 0x00050004,
338 0x91c4, 0xffffffff, 0x00010006,
339 0x91c8, 0xffffffff, 0x00090008,
340 0x91cc, 0xffffffff, 0x00070000,
341 0x91d0, 0xffffffff, 0x00030002,
342 0x91d4, 0xffffffff, 0x00050004,
343 0x91e0, 0xffffffff, 0x00010006,
344 0x91e4, 0xffffffff, 0x00090008,
345 0x91e8, 0xffffffff, 0x00000000,
346 0x91ec, 0xffffffff, 0x00070000,
347 0x91f0, 0xffffffff, 0x00030002,
348 0x91f4, 0xffffffff, 0x00050004,
349 0x9200, 0xffffffff, 0x00010006,
350 0x9204, 0xffffffff, 0x00090008,
351 0x9208, 0xffffffff, 0x00070000,
352 0x920c, 0xffffffff, 0x00030002,
353 0x9210, 0xffffffff, 0x00050004,
354 0x921c, 0xffffffff, 0x00010006,
355 0x9220, 0xffffffff, 0x00090008,
356 0x9224, 0xffffffff, 0x00070000,
357 0x9228, 0xffffffff, 0x00030002,
358 0x922c, 0xffffffff, 0x00050004,
359 0x9238, 0xffffffff, 0x00010006,
360 0x923c, 0xffffffff, 0x00090008,
361 0x9240, 0xffffffff, 0x00070000,
362 0x9244, 0xffffffff, 0x00030002,
363 0x9248, 0xffffffff, 0x00050004,
364 0x9254, 0xffffffff, 0x00010006,
365 0x9258, 0xffffffff, 0x00090008,
366 0x925c, 0xffffffff, 0x00070000,
367 0x9260, 0xffffffff, 0x00030002,
368 0x9264, 0xffffffff, 0x00050004,
369 0x9270, 0xffffffff, 0x00010006,
370 0x9274, 0xffffffff, 0x00090008,
371 0x9278, 0xffffffff, 0x00070000,
372 0x927c, 0xffffffff, 0x00030002,
373 0x9280, 0xffffffff, 0x00050004,
374 0x928c, 0xffffffff, 0x00010006,
375 0x9290, 0xffffffff, 0x00090008,
376 0x9294, 0xffffffff, 0x00000000,
377 0x929c, 0xffffffff, 0x00000001,
378 0x802c, 0xffffffff, 0x40010000,
379 0x915c, 0xffffffff, 0x00010000,
380 0x9160, 0xffffffff, 0x00030002,
381 0x9178, 0xffffffff, 0x00070000,
382 0x917c, 0xffffffff, 0x00030002,
383 0x9180, 0xffffffff, 0x00050004,
384 0x918c, 0xffffffff, 0x00010006,
385 0x9190, 0xffffffff, 0x00090008,
386 0x9194, 0xffffffff, 0x00070000,
387 0x9198, 0xffffffff, 0x00030002,
388 0x919c, 0xffffffff, 0x00050004,
389 0x91a8, 0xffffffff, 0x00010006,
390 0x91ac, 0xffffffff, 0x00090008,
391 0x91b0, 0xffffffff, 0x00070000,
392 0x91b4, 0xffffffff, 0x00030002,
393 0x91b8, 0xffffffff, 0x00050004,
394 0x91c4, 0xffffffff, 0x00010006,
395 0x91c8, 0xffffffff, 0x00090008,
396 0x91cc, 0xffffffff, 0x00070000,
397 0x91d0, 0xffffffff, 0x00030002,
398 0x91d4, 0xffffffff, 0x00050004,
399 0x91e0, 0xffffffff, 0x00010006,
400 0x91e4, 0xffffffff, 0x00090008,
401 0x91e8, 0xffffffff, 0x00000000,
402 0x91ec, 0xffffffff, 0x00070000,
403 0x91f0, 0xffffffff, 0x00030002,
404 0x91f4, 0xffffffff, 0x00050004,
405 0x9200, 0xffffffff, 0x00010006,
406 0x9204, 0xffffffff, 0x00090008,
407 0x9208, 0xffffffff, 0x00070000,
408 0x920c, 0xffffffff, 0x00030002,
409 0x9210, 0xffffffff, 0x00050004,
410 0x921c, 0xffffffff, 0x00010006,
411 0x9220, 0xffffffff, 0x00090008,
412 0x9224, 0xffffffff, 0x00070000,
413 0x9228, 0xffffffff, 0x00030002,
414 0x922c, 0xffffffff, 0x00050004,
415 0x9238, 0xffffffff, 0x00010006,
416 0x923c, 0xffffffff, 0x00090008,
417 0x9240, 0xffffffff, 0x00070000,
418 0x9244, 0xffffffff, 0x00030002,
419 0x9248, 0xffffffff, 0x00050004,
420 0x9254, 0xffffffff, 0x00010006,
421 0x9258, 0xffffffff, 0x00090008,
422 0x925c, 0xffffffff, 0x00070000,
423 0x9260, 0xffffffff, 0x00030002,
424 0x9264, 0xffffffff, 0x00050004,
425 0x9270, 0xffffffff, 0x00010006,
426 0x9274, 0xffffffff, 0x00090008,
427 0x9278, 0xffffffff, 0x00070000,
428 0x927c, 0xffffffff, 0x00030002,
429 0x9280, 0xffffffff, 0x00050004,
430 0x928c, 0xffffffff, 0x00010006,
431 0x9290, 0xffffffff, 0x00090008,
432 0x9294, 0xffffffff, 0x00000000,
433 0x929c, 0xffffffff, 0x00000001,
434 0x802c, 0xffffffff, 0xc0000000
435};
436
437static const u32 redwood_mgcg_init[] =
438{
439 0x802c, 0xffffffff, 0xc0000000,
440 0x5448, 0xffffffff, 0x00000100,
441 0x55e4, 0xffffffff, 0x00000100,
442 0x160c, 0xffffffff, 0x00000100,
443 0x5644, 0xffffffff, 0x00000100,
444 0xc164, 0xffffffff, 0x00000100,
445 0x8a18, 0xffffffff, 0x00000100,
446 0x897c, 0xffffffff, 0x06000100,
447 0x8b28, 0xffffffff, 0x00000100,
448 0x9144, 0xffffffff, 0x00000100,
449 0x9a60, 0xffffffff, 0x00000100,
450 0x9868, 0xffffffff, 0x00000100,
451 0x8d58, 0xffffffff, 0x00000100,
452 0x9510, 0xffffffff, 0x00000100,
453 0x949c, 0xffffffff, 0x00000100,
454 0x9654, 0xffffffff, 0x00000100,
455 0x9030, 0xffffffff, 0x00000100,
456 0x9034, 0xffffffff, 0x00000100,
457 0x9038, 0xffffffff, 0x00000100,
458 0x903c, 0xffffffff, 0x00000100,
459 0x9040, 0xffffffff, 0x00000100,
460 0xa200, 0xffffffff, 0x00000100,
461 0xa204, 0xffffffff, 0x00000100,
462 0xa208, 0xffffffff, 0x00000100,
463 0xa20c, 0xffffffff, 0x00000100,
464 0x971c, 0xffffffff, 0x00000100,
465 0x977c, 0xffffffff, 0x00000100,
466 0x3f80, 0xffffffff, 0x00000100,
467 0xa210, 0xffffffff, 0x00000100,
468 0xa214, 0xffffffff, 0x00000100,
469 0x4d8, 0xffffffff, 0x00000100,
470 0x9784, 0xffffffff, 0x00000100,
471 0x9698, 0xffffffff, 0x00000100,
472 0x4d4, 0xffffffff, 0x00000200,
473 0x30cc, 0xffffffff, 0x00000100,
474 0xd0c0, 0xffffffff, 0xff000100,
475 0x802c, 0xffffffff, 0x40000000,
476 0x915c, 0xffffffff, 0x00010000,
477 0x9160, 0xffffffff, 0x00030002,
478 0x9178, 0xffffffff, 0x00070000,
479 0x917c, 0xffffffff, 0x00030002,
480 0x9180, 0xffffffff, 0x00050004,
481 0x918c, 0xffffffff, 0x00010006,
482 0x9190, 0xffffffff, 0x00090008,
483 0x9194, 0xffffffff, 0x00070000,
484 0x9198, 0xffffffff, 0x00030002,
485 0x919c, 0xffffffff, 0x00050004,
486 0x91a8, 0xffffffff, 0x00010006,
487 0x91ac, 0xffffffff, 0x00090008,
488 0x91b0, 0xffffffff, 0x00070000,
489 0x91b4, 0xffffffff, 0x00030002,
490 0x91b8, 0xffffffff, 0x00050004,
491 0x91c4, 0xffffffff, 0x00010006,
492 0x91c8, 0xffffffff, 0x00090008,
493 0x91cc, 0xffffffff, 0x00070000,
494 0x91d0, 0xffffffff, 0x00030002,
495 0x91d4, 0xffffffff, 0x00050004,
496 0x91e0, 0xffffffff, 0x00010006,
497 0x91e4, 0xffffffff, 0x00090008,
498 0x91e8, 0xffffffff, 0x00000000,
499 0x91ec, 0xffffffff, 0x00070000,
500 0x91f0, 0xffffffff, 0x00030002,
501 0x91f4, 0xffffffff, 0x00050004,
502 0x9200, 0xffffffff, 0x00010006,
503 0x9204, 0xffffffff, 0x00090008,
504 0x9294, 0xffffffff, 0x00000000,
505 0x929c, 0xffffffff, 0x00000001,
506 0x802c, 0xffffffff, 0xc0000000
507};
508
509static const u32 cedar_golden_registers[] =
510{
511 0x3f90, 0xffff0000, 0xff000000,
512 0x9148, 0xffff0000, 0xff000000,
513 0x3f94, 0xffff0000, 0xff000000,
514 0x914c, 0xffff0000, 0xff000000,
515 0x9b7c, 0xffffffff, 0x00000000,
516 0x8a14, 0xffffffff, 0x00000007,
517 0x8b10, 0xffffffff, 0x00000000,
518 0x960c, 0xffffffff, 0x54763210,
519 0x88c4, 0xffffffff, 0x000000c2,
520 0x88d4, 0xffffffff, 0x00000000,
521 0x8974, 0xffffffff, 0x00000000,
522 0xc78, 0x00000080, 0x00000080,
523 0x5eb4, 0xffffffff, 0x00000002,
524 0x5e78, 0xffffffff, 0x001000f0,
525 0x6104, 0x01000300, 0x00000000,
526 0x5bc0, 0x00300000, 0x00000000,
527 0x7030, 0xffffffff, 0x00000011,
528 0x7c30, 0xffffffff, 0x00000011,
529 0x10830, 0xffffffff, 0x00000011,
530 0x11430, 0xffffffff, 0x00000011,
531 0xd02c, 0xffffffff, 0x08421000,
532 0x240c, 0xffffffff, 0x00000380,
533 0x8b24, 0xffffffff, 0x00ff0fff,
534 0x28a4c, 0x06000000, 0x06000000,
535 0x10c, 0x00000001, 0x00000001,
536 0x8d00, 0xffffffff, 0x100e4848,
537 0x8d04, 0xffffffff, 0x00164745,
538 0x8c00, 0xffffffff, 0xe4000003,
539 0x8c04, 0xffffffff, 0x40600060,
540 0x8c08, 0xffffffff, 0x001c001c,
541 0x8cf0, 0xffffffff, 0x08e00410,
542 0x8c20, 0xffffffff, 0x00800080,
543 0x8c24, 0xffffffff, 0x00800080,
544 0x8c18, 0xffffffff, 0x20202078,
545 0x8c1c, 0xffffffff, 0x00001010,
546 0x28350, 0xffffffff, 0x00000000,
547 0xa008, 0xffffffff, 0x00010000,
548 0x5c4, 0xffffffff, 0x00000001,
549 0x9508, 0xffffffff, 0x00000002
550};
551
552static const u32 cedar_mgcg_init[] =
553{
554 0x802c, 0xffffffff, 0xc0000000,
555 0x5448, 0xffffffff, 0x00000100,
556 0x55e4, 0xffffffff, 0x00000100,
557 0x160c, 0xffffffff, 0x00000100,
558 0x5644, 0xffffffff, 0x00000100,
559 0xc164, 0xffffffff, 0x00000100,
560 0x8a18, 0xffffffff, 0x00000100,
561 0x897c, 0xffffffff, 0x06000100,
562 0x8b28, 0xffffffff, 0x00000100,
563 0x9144, 0xffffffff, 0x00000100,
564 0x9a60, 0xffffffff, 0x00000100,
565 0x9868, 0xffffffff, 0x00000100,
566 0x8d58, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0x949c, 0xffffffff, 0x00000100,
569 0x9654, 0xffffffff, 0x00000100,
570 0x9030, 0xffffffff, 0x00000100,
571 0x9034, 0xffffffff, 0x00000100,
572 0x9038, 0xffffffff, 0x00000100,
573 0x903c, 0xffffffff, 0x00000100,
574 0x9040, 0xffffffff, 0x00000100,
575 0xa200, 0xffffffff, 0x00000100,
576 0xa204, 0xffffffff, 0x00000100,
577 0xa208, 0xffffffff, 0x00000100,
578 0xa20c, 0xffffffff, 0x00000100,
579 0x971c, 0xffffffff, 0x00000100,
580 0x977c, 0xffffffff, 0x00000100,
581 0x3f80, 0xffffffff, 0x00000100,
582 0xa210, 0xffffffff, 0x00000100,
583 0xa214, 0xffffffff, 0x00000100,
584 0x4d8, 0xffffffff, 0x00000100,
585 0x9784, 0xffffffff, 0x00000100,
586 0x9698, 0xffffffff, 0x00000100,
587 0x4d4, 0xffffffff, 0x00000200,
588 0x30cc, 0xffffffff, 0x00000100,
589 0xd0c0, 0xffffffff, 0xff000100,
590 0x802c, 0xffffffff, 0x40000000,
591 0x915c, 0xffffffff, 0x00010000,
592 0x9178, 0xffffffff, 0x00050000,
593 0x917c, 0xffffffff, 0x00030002,
594 0x918c, 0xffffffff, 0x00010004,
595 0x9190, 0xffffffff, 0x00070006,
596 0x9194, 0xffffffff, 0x00050000,
597 0x9198, 0xffffffff, 0x00030002,
598 0x91a8, 0xffffffff, 0x00010004,
599 0x91ac, 0xffffffff, 0x00070006,
600 0x91e8, 0xffffffff, 0x00000000,
601 0x9294, 0xffffffff, 0x00000000,
602 0x929c, 0xffffffff, 0x00000001,
603 0x802c, 0xffffffff, 0xc0000000
604};
605
606static const u32 juniper_mgcg_init[] =
607{
608 0x802c, 0xffffffff, 0xc0000000,
609 0x5448, 0xffffffff, 0x00000100,
610 0x55e4, 0xffffffff, 0x00000100,
611 0x160c, 0xffffffff, 0x00000100,
612 0x5644, 0xffffffff, 0x00000100,
613 0xc164, 0xffffffff, 0x00000100,
614 0x8a18, 0xffffffff, 0x00000100,
615 0x897c, 0xffffffff, 0x06000100,
616 0x8b28, 0xffffffff, 0x00000100,
617 0x9144, 0xffffffff, 0x00000100,
618 0x9a60, 0xffffffff, 0x00000100,
619 0x9868, 0xffffffff, 0x00000100,
620 0x8d58, 0xffffffff, 0x00000100,
621 0x9510, 0xffffffff, 0x00000100,
622 0x949c, 0xffffffff, 0x00000100,
623 0x9654, 0xffffffff, 0x00000100,
624 0x9030, 0xffffffff, 0x00000100,
625 0x9034, 0xffffffff, 0x00000100,
626 0x9038, 0xffffffff, 0x00000100,
627 0x903c, 0xffffffff, 0x00000100,
628 0x9040, 0xffffffff, 0x00000100,
629 0xa200, 0xffffffff, 0x00000100,
630 0xa204, 0xffffffff, 0x00000100,
631 0xa208, 0xffffffff, 0x00000100,
632 0xa20c, 0xffffffff, 0x00000100,
633 0x971c, 0xffffffff, 0x00000100,
634 0xd0c0, 0xffffffff, 0xff000100,
635 0x802c, 0xffffffff, 0x40000000,
636 0x915c, 0xffffffff, 0x00010000,
637 0x9160, 0xffffffff, 0x00030002,
638 0x9178, 0xffffffff, 0x00070000,
639 0x917c, 0xffffffff, 0x00030002,
640 0x9180, 0xffffffff, 0x00050004,
641 0x918c, 0xffffffff, 0x00010006,
642 0x9190, 0xffffffff, 0x00090008,
643 0x9194, 0xffffffff, 0x00070000,
644 0x9198, 0xffffffff, 0x00030002,
645 0x919c, 0xffffffff, 0x00050004,
646 0x91a8, 0xffffffff, 0x00010006,
647 0x91ac, 0xffffffff, 0x00090008,
648 0x91b0, 0xffffffff, 0x00070000,
649 0x91b4, 0xffffffff, 0x00030002,
650 0x91b8, 0xffffffff, 0x00050004,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91c8, 0xffffffff, 0x00090008,
653 0x91cc, 0xffffffff, 0x00070000,
654 0x91d0, 0xffffffff, 0x00030002,
655 0x91d4, 0xffffffff, 0x00050004,
656 0x91e0, 0xffffffff, 0x00010006,
657 0x91e4, 0xffffffff, 0x00090008,
658 0x91e8, 0xffffffff, 0x00000000,
659 0x91ec, 0xffffffff, 0x00070000,
660 0x91f0, 0xffffffff, 0x00030002,
661 0x91f4, 0xffffffff, 0x00050004,
662 0x9200, 0xffffffff, 0x00010006,
663 0x9204, 0xffffffff, 0x00090008,
664 0x9208, 0xffffffff, 0x00070000,
665 0x920c, 0xffffffff, 0x00030002,
666 0x9210, 0xffffffff, 0x00050004,
667 0x921c, 0xffffffff, 0x00010006,
668 0x9220, 0xffffffff, 0x00090008,
669 0x9224, 0xffffffff, 0x00070000,
670 0x9228, 0xffffffff, 0x00030002,
671 0x922c, 0xffffffff, 0x00050004,
672 0x9238, 0xffffffff, 0x00010006,
673 0x923c, 0xffffffff, 0x00090008,
674 0x9240, 0xffffffff, 0x00070000,
675 0x9244, 0xffffffff, 0x00030002,
676 0x9248, 0xffffffff, 0x00050004,
677 0x9254, 0xffffffff, 0x00010006,
678 0x9258, 0xffffffff, 0x00090008,
679 0x925c, 0xffffffff, 0x00070000,
680 0x9260, 0xffffffff, 0x00030002,
681 0x9264, 0xffffffff, 0x00050004,
682 0x9270, 0xffffffff, 0x00010006,
683 0x9274, 0xffffffff, 0x00090008,
684 0x9278, 0xffffffff, 0x00070000,
685 0x927c, 0xffffffff, 0x00030002,
686 0x9280, 0xffffffff, 0x00050004,
687 0x928c, 0xffffffff, 0x00010006,
688 0x9290, 0xffffffff, 0x00090008,
689 0x9294, 0xffffffff, 0x00000000,
690 0x929c, 0xffffffff, 0x00000001,
691 0x802c, 0xffffffff, 0xc0000000,
692 0x977c, 0xffffffff, 0x00000100,
693 0x3f80, 0xffffffff, 0x00000100,
694 0xa210, 0xffffffff, 0x00000100,
695 0xa214, 0xffffffff, 0x00000100,
696 0x4d8, 0xffffffff, 0x00000100,
697 0x9784, 0xffffffff, 0x00000100,
698 0x9698, 0xffffffff, 0x00000100,
699 0x4d4, 0xffffffff, 0x00000200,
700 0x30cc, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xc0000000
702};
703
704static const u32 supersumo_golden_registers[] =
705{
706 0x5eb4, 0xffffffff, 0x00000002,
707 0x5c4, 0xffffffff, 0x00000001,
708 0x7030, 0xffffffff, 0x00000011,
709 0x7c30, 0xffffffff, 0x00000011,
710 0x6104, 0x01000300, 0x00000000,
711 0x5bc0, 0x00300000, 0x00000000,
712 0x8c04, 0xffffffff, 0x40600060,
713 0x8c08, 0xffffffff, 0x001c001c,
714 0x8c20, 0xffffffff, 0x00800080,
715 0x8c24, 0xffffffff, 0x00800080,
716 0x8c18, 0xffffffff, 0x20202078,
717 0x8c1c, 0xffffffff, 0x00001010,
718 0x918c, 0xffffffff, 0x00010006,
719 0x91a8, 0xffffffff, 0x00010006,
720 0x91c4, 0xffffffff, 0x00010006,
721 0x91e0, 0xffffffff, 0x00010006,
722 0x9200, 0xffffffff, 0x00010006,
723 0x9150, 0xffffffff, 0x6e944040,
724 0x917c, 0xffffffff, 0x00030002,
725 0x9180, 0xffffffff, 0x00050004,
726 0x9198, 0xffffffff, 0x00030002,
727 0x919c, 0xffffffff, 0x00050004,
728 0x91b4, 0xffffffff, 0x00030002,
729 0x91b8, 0xffffffff, 0x00050004,
730 0x91d0, 0xffffffff, 0x00030002,
731 0x91d4, 0xffffffff, 0x00050004,
732 0x91f0, 0xffffffff, 0x00030002,
733 0x91f4, 0xffffffff, 0x00050004,
734 0x915c, 0xffffffff, 0x00010000,
735 0x9160, 0xffffffff, 0x00030002,
736 0x3f90, 0xffff0000, 0xff000000,
737 0x9178, 0xffffffff, 0x00070000,
738 0x9194, 0xffffffff, 0x00070000,
739 0x91b0, 0xffffffff, 0x00070000,
740 0x91cc, 0xffffffff, 0x00070000,
741 0x91ec, 0xffffffff, 0x00070000,
742 0x9148, 0xffff0000, 0xff000000,
743 0x9190, 0xffffffff, 0x00090008,
744 0x91ac, 0xffffffff, 0x00090008,
745 0x91c8, 0xffffffff, 0x00090008,
746 0x91e4, 0xffffffff, 0x00090008,
747 0x9204, 0xffffffff, 0x00090008,
748 0x3f94, 0xffff0000, 0xff000000,
749 0x914c, 0xffff0000, 0xff000000,
750 0x929c, 0xffffffff, 0x00000001,
751 0x8a18, 0xffffffff, 0x00000100,
752 0x8b28, 0xffffffff, 0x00000100,
753 0x9144, 0xffffffff, 0x00000100,
754 0x5644, 0xffffffff, 0x00000100,
755 0x9b7c, 0xffffffff, 0x00000000,
756 0x8030, 0xffffffff, 0x0000100a,
757 0x8a14, 0xffffffff, 0x00000007,
758 0x8b24, 0xffffffff, 0x00ff0fff,
759 0x8b10, 0xffffffff, 0x00000000,
760 0x28a4c, 0x06000000, 0x06000000,
761 0x4d8, 0xffffffff, 0x00000100,
762 0x913c, 0xffff000f, 0x0100000a,
763 0x960c, 0xffffffff, 0x54763210,
764 0x88c4, 0xffffffff, 0x000000c2,
765 0x88d4, 0xffffffff, 0x00000010,
766 0x8974, 0xffffffff, 0x00000000,
767 0xc78, 0x00000080, 0x00000080,
768 0x5e78, 0xffffffff, 0x001000f0,
769 0xd02c, 0xffffffff, 0x08421000,
770 0xa008, 0xffffffff, 0x00010000,
771 0x8d00, 0xffffffff, 0x100e4848,
772 0x8d04, 0xffffffff, 0x00164745,
773 0x8c00, 0xffffffff, 0xe4000003,
774 0x8cf0, 0x1fffffff, 0x08e00620,
775 0x28350, 0xffffffff, 0x00000000,
776 0x9508, 0xffffffff, 0x00000002
777};
778
779static const u32 sumo_golden_registers[] =
780{
781 0x900c, 0x00ffffff, 0x0017071f,
782 0x8c18, 0xffffffff, 0x10101060,
783 0x8c1c, 0xffffffff, 0x00001010,
784 0x8c30, 0x0000000f, 0x00000005,
785 0x9688, 0x0000000f, 0x00000007
786};
787
788static const u32 wrestler_golden_registers[] =
789{
790 0x5eb4, 0xffffffff, 0x00000002,
791 0x5c4, 0xffffffff, 0x00000001,
792 0x7030, 0xffffffff, 0x00000011,
793 0x7c30, 0xffffffff, 0x00000011,
794 0x6104, 0x01000300, 0x00000000,
795 0x5bc0, 0x00300000, 0x00000000,
796 0x918c, 0xffffffff, 0x00010006,
797 0x91a8, 0xffffffff, 0x00010006,
798 0x9150, 0xffffffff, 0x6e944040,
799 0x917c, 0xffffffff, 0x00030002,
800 0x9198, 0xffffffff, 0x00030002,
801 0x915c, 0xffffffff, 0x00010000,
802 0x3f90, 0xffff0000, 0xff000000,
803 0x9178, 0xffffffff, 0x00070000,
804 0x9194, 0xffffffff, 0x00070000,
805 0x9148, 0xffff0000, 0xff000000,
806 0x9190, 0xffffffff, 0x00090008,
807 0x91ac, 0xffffffff, 0x00090008,
808 0x3f94, 0xffff0000, 0xff000000,
809 0x914c, 0xffff0000, 0xff000000,
810 0x929c, 0xffffffff, 0x00000001,
811 0x8a18, 0xffffffff, 0x00000100,
812 0x8b28, 0xffffffff, 0x00000100,
813 0x9144, 0xffffffff, 0x00000100,
814 0x9b7c, 0xffffffff, 0x00000000,
815 0x8030, 0xffffffff, 0x0000100a,
816 0x8a14, 0xffffffff, 0x00000001,
817 0x8b24, 0xffffffff, 0x00ff0fff,
818 0x8b10, 0xffffffff, 0x00000000,
819 0x28a4c, 0x06000000, 0x06000000,
820 0x4d8, 0xffffffff, 0x00000100,
821 0x913c, 0xffff000f, 0x0100000a,
822 0x960c, 0xffffffff, 0x54763210,
823 0x88c4, 0xffffffff, 0x000000c2,
824 0x88d4, 0xffffffff, 0x00000010,
825 0x8974, 0xffffffff, 0x00000000,
826 0xc78, 0x00000080, 0x00000080,
827 0x5e78, 0xffffffff, 0x001000f0,
828 0xd02c, 0xffffffff, 0x08421000,
829 0xa008, 0xffffffff, 0x00010000,
830 0x8d00, 0xffffffff, 0x100e4848,
831 0x8d04, 0xffffffff, 0x00164745,
832 0x8c00, 0xffffffff, 0xe4000003,
833 0x8cf0, 0x1fffffff, 0x08e00410,
834 0x28350, 0xffffffff, 0x00000000,
835 0x9508, 0xffffffff, 0x00000002,
836 0x900c, 0xffffffff, 0x0017071f,
837 0x8c18, 0xffffffff, 0x10101060,
838 0x8c1c, 0xffffffff, 0x00001010
839};
840
841static const u32 barts_golden_registers[] =
842{
843 0x5eb4, 0xffffffff, 0x00000002,
844 0x5e78, 0x8f311ff1, 0x001000f0,
845 0x3f90, 0xffff0000, 0xff000000,
846 0x9148, 0xffff0000, 0xff000000,
847 0x3f94, 0xffff0000, 0xff000000,
848 0x914c, 0xffff0000, 0xff000000,
849 0xc78, 0x00000080, 0x00000080,
850 0xbd4, 0x70073777, 0x00010001,
851 0xd02c, 0xbfffff1f, 0x08421000,
852 0xd0b8, 0x03773777, 0x02011003,
853 0x5bc0, 0x00200000, 0x50100000,
854 0x98f8, 0x33773777, 0x02011003,
855 0x98fc, 0xffffffff, 0x76543210,
856 0x7030, 0x31000311, 0x00000011,
857 0x2f48, 0x00000007, 0x02011003,
858 0x6b28, 0x00000010, 0x00000012,
859 0x7728, 0x00000010, 0x00000012,
860 0x10328, 0x00000010, 0x00000012,
861 0x10f28, 0x00000010, 0x00000012,
862 0x11b28, 0x00000010, 0x00000012,
863 0x12728, 0x00000010, 0x00000012,
864 0x240c, 0x000007ff, 0x00000380,
865 0x8a14, 0xf000001f, 0x00000007,
866 0x8b24, 0x3fff3fff, 0x00ff0fff,
867 0x8b10, 0x0000ff0f, 0x00000000,
868 0x28a4c, 0x07ffffff, 0x06000000,
869 0x10c, 0x00000001, 0x00010003,
870 0xa02c, 0xffffffff, 0x0000009b,
871 0x913c, 0x0000000f, 0x0100000a,
872 0x8d00, 0xffff7f7f, 0x100e4848,
873 0x8d04, 0x00ffffff, 0x00164745,
874 0x8c00, 0xfffc0003, 0xe4000003,
875 0x8c04, 0xf8ff00ff, 0x40600060,
876 0x8c08, 0x00ff00ff, 0x001c001c,
877 0x8cf0, 0x1fff1fff, 0x08e00620,
878 0x8c20, 0x0fff0fff, 0x00800080,
879 0x8c24, 0x0fff0fff, 0x00800080,
880 0x8c18, 0xffffffff, 0x20202078,
881 0x8c1c, 0x0000ffff, 0x00001010,
882 0x28350, 0x00000f01, 0x00000000,
883 0x9508, 0x3700001f, 0x00000002,
884 0x960c, 0xffffffff, 0x54763210,
885 0x88c4, 0x001f3ae3, 0x000000c2,
886 0x88d4, 0x0000001f, 0x00000010,
887 0x8974, 0xffffffff, 0x00000000
888};
889
890static const u32 turks_golden_registers[] =
891{
892 0x5eb4, 0xffffffff, 0x00000002,
893 0x5e78, 0x8f311ff1, 0x001000f0,
894 0x8c8, 0x00003000, 0x00001070,
895 0x8cc, 0x000fffff, 0x00040035,
896 0x3f90, 0xffff0000, 0xfff00000,
897 0x9148, 0xffff0000, 0xfff00000,
898 0x3f94, 0xffff0000, 0xfff00000,
899 0x914c, 0xffff0000, 0xfff00000,
900 0xc78, 0x00000080, 0x00000080,
901 0xbd4, 0x00073007, 0x00010002,
902 0xd02c, 0xbfffff1f, 0x08421000,
903 0xd0b8, 0x03773777, 0x02010002,
904 0x5bc0, 0x00200000, 0x50100000,
905 0x98f8, 0x33773777, 0x00010002,
906 0x98fc, 0xffffffff, 0x33221100,
907 0x7030, 0x31000311, 0x00000011,
908 0x2f48, 0x33773777, 0x00010002,
909 0x6b28, 0x00000010, 0x00000012,
910 0x7728, 0x00000010, 0x00000012,
911 0x10328, 0x00000010, 0x00000012,
912 0x10f28, 0x00000010, 0x00000012,
913 0x11b28, 0x00000010, 0x00000012,
914 0x12728, 0x00000010, 0x00000012,
915 0x240c, 0x000007ff, 0x00000380,
916 0x8a14, 0xf000001f, 0x00000007,
917 0x8b24, 0x3fff3fff, 0x00ff0fff,
918 0x8b10, 0x0000ff0f, 0x00000000,
919 0x28a4c, 0x07ffffff, 0x06000000,
920 0x10c, 0x00000001, 0x00010003,
921 0xa02c, 0xffffffff, 0x0000009b,
922 0x913c, 0x0000000f, 0x0100000a,
923 0x8d00, 0xffff7f7f, 0x100e4848,
924 0x8d04, 0x00ffffff, 0x00164745,
925 0x8c00, 0xfffc0003, 0xe4000003,
926 0x8c04, 0xf8ff00ff, 0x40600060,
927 0x8c08, 0x00ff00ff, 0x001c001c,
928 0x8cf0, 0x1fff1fff, 0x08e00410,
929 0x8c20, 0x0fff0fff, 0x00800080,
930 0x8c24, 0x0fff0fff, 0x00800080,
931 0x8c18, 0xffffffff, 0x20202078,
932 0x8c1c, 0x0000ffff, 0x00001010,
933 0x28350, 0x00000f01, 0x00000000,
934 0x9508, 0x3700001f, 0x00000002,
935 0x960c, 0xffffffff, 0x54763210,
936 0x88c4, 0x001f3ae3, 0x000000c2,
937 0x88d4, 0x0000001f, 0x00000010,
938 0x8974, 0xffffffff, 0x00000000
939};
940
941static const u32 caicos_golden_registers[] =
942{
943 0x5eb4, 0xffffffff, 0x00000002,
944 0x5e78, 0x8f311ff1, 0x001000f0,
945 0x8c8, 0x00003420, 0x00001450,
946 0x8cc, 0x000fffff, 0x00040035,
947 0x3f90, 0xffff0000, 0xfffc0000,
948 0x9148, 0xffff0000, 0xfffc0000,
949 0x3f94, 0xffff0000, 0xfffc0000,
950 0x914c, 0xffff0000, 0xfffc0000,
951 0xc78, 0x00000080, 0x00000080,
952 0xbd4, 0x00073007, 0x00010001,
953 0xd02c, 0xbfffff1f, 0x08421000,
954 0xd0b8, 0x03773777, 0x02010001,
955 0x5bc0, 0x00200000, 0x50100000,
956 0x98f8, 0x33773777, 0x02010001,
957 0x98fc, 0xffffffff, 0x33221100,
958 0x7030, 0x31000311, 0x00000011,
959 0x2f48, 0x33773777, 0x02010001,
960 0x6b28, 0x00000010, 0x00000012,
961 0x7728, 0x00000010, 0x00000012,
962 0x10328, 0x00000010, 0x00000012,
963 0x10f28, 0x00000010, 0x00000012,
964 0x11b28, 0x00000010, 0x00000012,
965 0x12728, 0x00000010, 0x00000012,
966 0x240c, 0x000007ff, 0x00000380,
967 0x8a14, 0xf000001f, 0x00000001,
968 0x8b24, 0x3fff3fff, 0x00ff0fff,
969 0x8b10, 0x0000ff0f, 0x00000000,
970 0x28a4c, 0x07ffffff, 0x06000000,
971 0x10c, 0x00000001, 0x00010003,
972 0xa02c, 0xffffffff, 0x0000009b,
973 0x913c, 0x0000000f, 0x0100000a,
974 0x8d00, 0xffff7f7f, 0x100e4848,
975 0x8d04, 0x00ffffff, 0x00164745,
976 0x8c00, 0xfffc0003, 0xe4000003,
977 0x8c04, 0xf8ff00ff, 0x40600060,
978 0x8c08, 0x00ff00ff, 0x001c001c,
979 0x8cf0, 0x1fff1fff, 0x08e00410,
980 0x8c20, 0x0fff0fff, 0x00800080,
981 0x8c24, 0x0fff0fff, 0x00800080,
982 0x8c18, 0xffffffff, 0x20202078,
983 0x8c1c, 0x0000ffff, 0x00001010,
984 0x28350, 0x00000f01, 0x00000000,
985 0x9508, 0x3700001f, 0x00000002,
986 0x960c, 0xffffffff, 0x54763210,
987 0x88c4, 0x001f3ae3, 0x000000c2,
988 0x88d4, 0x0000001f, 0x00000010,
989 0x8974, 0xffffffff, 0x00000000
990};
991
992static void evergreen_init_golden_registers(struct radeon_device *rdev)
993{
994 switch (rdev->family) {
995 case CHIP_CYPRESS:
996 case CHIP_HEMLOCK:
997 radeon_program_register_sequence(rdev,
998 evergreen_golden_registers,
999 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 evergreen_golden_registers2,
1002 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1003 radeon_program_register_sequence(rdev,
1004 cypress_mgcg_init,
1005 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1006 break;
1007 case CHIP_JUNIPER:
1008 radeon_program_register_sequence(rdev,
1009 evergreen_golden_registers,
1010 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1011 radeon_program_register_sequence(rdev,
1012 evergreen_golden_registers2,
1013 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1014 radeon_program_register_sequence(rdev,
1015 juniper_mgcg_init,
1016 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1017 break;
1018 case CHIP_REDWOOD:
1019 radeon_program_register_sequence(rdev,
1020 evergreen_golden_registers,
1021 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1022 radeon_program_register_sequence(rdev,
1023 evergreen_golden_registers2,
1024 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1025 radeon_program_register_sequence(rdev,
1026 redwood_mgcg_init,
1027 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1028 break;
1029 case CHIP_CEDAR:
1030 radeon_program_register_sequence(rdev,
1031 cedar_golden_registers,
1032 (const u32)ARRAY_SIZE(cedar_golden_registers));
1033 radeon_program_register_sequence(rdev,
1034 evergreen_golden_registers2,
1035 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1036 radeon_program_register_sequence(rdev,
1037 cedar_mgcg_init,
1038 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1039 break;
1040 case CHIP_PALM:
1041 radeon_program_register_sequence(rdev,
1042 wrestler_golden_registers,
1043 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1044 break;
1045 case CHIP_SUMO:
1046 radeon_program_register_sequence(rdev,
1047 supersumo_golden_registers,
1048 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1049 break;
1050 case CHIP_SUMO2:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 radeon_program_register_sequence(rdev,
1055 sumo_golden_registers,
1056 (const u32)ARRAY_SIZE(sumo_golden_registers));
1057 break;
1058 case CHIP_BARTS:
1059 radeon_program_register_sequence(rdev,
1060 barts_golden_registers,
1061 (const u32)ARRAY_SIZE(barts_golden_registers));
1062 break;
1063 case CHIP_TURKS:
1064 radeon_program_register_sequence(rdev,
1065 turks_golden_registers,
1066 (const u32)ARRAY_SIZE(turks_golden_registers));
1067 break;
1068 case CHIP_CAICOS:
1069 radeon_program_register_sequence(rdev,
1070 caicos_golden_registers,
1071 (const u32)ARRAY_SIZE(caicos_golden_registers));
1072 break;
1073 default:
1074 break;
1075 }
1076}
1077
1078/**
1079 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1080 *
1081 * @rdev: radeon_device pointer
1082 * @reg: register offset in bytes
1083 * @val: register value
1084 *
1085 * Returns 0 for success or -EINVAL for an invalid register
1086 *
1087 */
1088int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1089 u32 reg, u32 *val)
1090{
1091 switch (reg) {
1092 case GRBM_STATUS:
1093 case GRBM_STATUS_SE0:
1094 case GRBM_STATUS_SE1:
1095 case SRBM_STATUS:
1096 case SRBM_STATUS2:
1097 case DMA_STATUS_REG:
1098 case UVD_STATUS:
1099 *val = RREG32(reg);
1100 return 0;
1101 default:
1102 return -EINVAL;
1103 }
1104}
1105
1106void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1107 unsigned *bankh, unsigned *mtaspect,
1108 unsigned *tile_split)
1109{
1110 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1111 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1112 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1113 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1114 switch (*bankw) {
1115 default:
1116 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1117 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1118 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1119 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1120 }
1121 switch (*bankh) {
1122 default:
1123 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1124 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1125 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1126 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1127 }
1128 switch (*mtaspect) {
1129 default:
1130 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1131 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1132 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1133 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1134 }
1135}
1136
1137static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1138 u32 cntl_reg, u32 status_reg)
1139{
1140 int r, i;
1141 struct atom_clock_dividers dividers;
1142
1143 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1144 clock, false, ÷rs);
1145 if (r)
1146 return r;
1147
1148 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1149
1150 for (i = 0; i < 100; i++) {
1151 if (RREG32(status_reg) & DCLK_STATUS)
1152 break;
1153 mdelay(10);
1154 }
1155 if (i == 100)
1156 return -ETIMEDOUT;
1157
1158 return 0;
1159}
1160
1161int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1162{
1163 int r = 0;
1164 u32 cg_scratch = RREG32(CG_SCRATCH1);
1165
1166 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1167 if (r)
1168 goto done;
1169 cg_scratch &= 0xffff0000;
1170 cg_scratch |= vclk / 100; /* Mhz */
1171
1172 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1173 if (r)
1174 goto done;
1175 cg_scratch &= 0x0000ffff;
1176 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1177
1178done:
1179 WREG32(CG_SCRATCH1, cg_scratch);
1180
1181 return r;
1182}
1183
1184int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1185{
1186 /* start off with something large */
1187 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1188 int r;
1189
1190 /* bypass vclk and dclk with bclk */
1191 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1192 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1193 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1194
1195 /* put PLL in bypass mode */
1196 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1197
1198 if (!vclk || !dclk) {
1199 /* keep the Bypass mode, put PLL to sleep */
1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1201 return 0;
1202 }
1203
1204 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1205 16384, 0x03FFFFFF, 0, 128, 5,
1206 &fb_div, &vclk_div, &dclk_div);
1207 if (r)
1208 return r;
1209
1210 /* set VCO_MODE to 1 */
1211 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1212
1213 /* toggle UPLL_SLEEP to 1 then back to 0 */
1214 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1215 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1216
1217 /* deassert UPLL_RESET */
1218 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1219
1220 mdelay(1);
1221
1222 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1223 if (r)
1224 return r;
1225
1226 /* assert UPLL_RESET again */
1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1228
1229 /* disable spread spectrum. */
1230 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1231
1232 /* set feedback divider */
1233 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1234
1235 /* set ref divider to 0 */
1236 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1237
1238 if (fb_div < 307200)
1239 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1240 else
1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1242
1243 /* set PDIV_A and PDIV_B */
1244 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1245 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1246 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1247
1248 /* give the PLL some time to settle */
1249 mdelay(15);
1250
1251 /* deassert PLL_RESET */
1252 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1253
1254 mdelay(15);
1255
1256 /* switch from bypass mode to normal mode */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1258
1259 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1260 if (r)
1261 return r;
1262
1263 /* switch VCLK and DCLK selection */
1264 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1265 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1266 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1267
1268 mdelay(100);
1269
1270 return 0;
1271}
1272
1273void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1274{
1275 int readrq;
1276 u16 v;
1277
1278 readrq = pcie_get_readrq(rdev->pdev);
1279 v = ffs(readrq) - 8;
1280 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1281 * to avoid hangs or perfomance issues
1282 */
1283 if ((v == 0) || (v == 6) || (v == 7))
1284 pcie_set_readrq(rdev->pdev, 512);
1285}
1286
1287void dce4_program_fmt(struct drm_encoder *encoder)
1288{
1289 struct drm_device *dev = encoder->dev;
1290 struct radeon_device *rdev = dev->dev_private;
1291 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1292 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1293 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1294 int bpc = 0;
1295 u32 tmp = 0;
1296 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1297
1298 if (connector) {
1299 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1300 bpc = radeon_get_monitor_bpc(connector);
1301 dither = radeon_connector->dither;
1302 }
1303
1304 /* LVDS/eDP FMT is set up by atom */
1305 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1306 return;
1307
1308 /* not needed for analog */
1309 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1310 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1311 return;
1312
1313 if (bpc == 0)
1314 return;
1315
1316 switch (bpc) {
1317 case 6:
1318 if (dither == RADEON_FMT_DITHER_ENABLE)
1319 /* XXX sort out optimal dither settings */
1320 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1321 FMT_SPATIAL_DITHER_EN);
1322 else
1323 tmp |= FMT_TRUNCATE_EN;
1324 break;
1325 case 8:
1326 if (dither == RADEON_FMT_DITHER_ENABLE)
1327 /* XXX sort out optimal dither settings */
1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329 FMT_RGB_RANDOM_ENABLE |
1330 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1331 else
1332 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1333 break;
1334 case 10:
1335 default:
1336 /* not needed */
1337 break;
1338 }
1339
1340 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1341}
1342
1343static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1344{
1345 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1346 return true;
1347 else
1348 return false;
1349}
1350
1351static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1352{
1353 u32 pos1, pos2;
1354
1355 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1356 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1357
1358 if (pos1 != pos2)
1359 return true;
1360 else
1361 return false;
1362}
1363
1364/**
1365 * dce4_wait_for_vblank - vblank wait asic callback.
1366 *
1367 * @rdev: radeon_device pointer
1368 * @crtc: crtc to wait for vblank on
1369 *
1370 * Wait for vblank on the requested crtc (evergreen+).
1371 */
1372void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1373{
1374 unsigned i = 0;
1375
1376 if (crtc >= rdev->num_crtc)
1377 return;
1378
1379 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1380 return;
1381
1382 /* depending on when we hit vblank, we may be close to active; if so,
1383 * wait for another frame.
1384 */
1385 while (dce4_is_in_vblank(rdev, crtc)) {
1386 if (i++ % 100 == 0) {
1387 if (!dce4_is_counter_moving(rdev, crtc))
1388 break;
1389 }
1390 }
1391
1392 while (!dce4_is_in_vblank(rdev, crtc)) {
1393 if (i++ % 100 == 0) {
1394 if (!dce4_is_counter_moving(rdev, crtc))
1395 break;
1396 }
1397 }
1398}
1399
1400/**
1401 * evergreen_page_flip - pageflip callback.
1402 *
1403 * @rdev: radeon_device pointer
1404 * @crtc_id: crtc to cleanup pageflip on
1405 * @crtc_base: new address of the crtc (GPU MC address)
1406 *
1407 * Triggers the actual pageflip by updating the primary
1408 * surface base address (evergreen+).
1409 */
1410void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1411{
1412 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1413
1414 /* update the scanout addresses */
1415 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1416 upper_32_bits(crtc_base));
1417 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1418 (u32)crtc_base);
1419 /* post the write */
1420 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1421}
1422
1423/**
1424 * evergreen_page_flip_pending - check if page flip is still pending
1425 *
1426 * @rdev: radeon_device pointer
1427 * @crtc_id: crtc to check
1428 *
1429 * Returns the current update pending status.
1430 */
1431bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1432{
1433 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1434
1435 /* Return current update_pending status: */
1436 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1437 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1438}
1439
1440/* get temperature in millidegrees */
1441int evergreen_get_temp(struct radeon_device *rdev)
1442{
1443 u32 temp, toffset;
1444 int actual_temp = 0;
1445
1446 if (rdev->family == CHIP_JUNIPER) {
1447 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1448 TOFFSET_SHIFT;
1449 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1450 TS0_ADC_DOUT_SHIFT;
1451
1452 if (toffset & 0x100)
1453 actual_temp = temp / 2 - (0x200 - toffset);
1454 else
1455 actual_temp = temp / 2 + toffset;
1456
1457 actual_temp = actual_temp * 1000;
1458
1459 } else {
1460 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1461 ASIC_T_SHIFT;
1462
1463 if (temp & 0x400)
1464 actual_temp = -256;
1465 else if (temp & 0x200)
1466 actual_temp = 255;
1467 else if (temp & 0x100) {
1468 actual_temp = temp & 0x1ff;
1469 actual_temp |= ~0x1ff;
1470 } else
1471 actual_temp = temp & 0xff;
1472
1473 actual_temp = (actual_temp * 1000) / 2;
1474 }
1475
1476 return actual_temp;
1477}
1478
1479int sumo_get_temp(struct radeon_device *rdev)
1480{
1481 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1482 int actual_temp = temp - 49;
1483
1484 return actual_temp * 1000;
1485}
1486
1487/**
1488 * sumo_pm_init_profile - Initialize power profiles callback.
1489 *
1490 * @rdev: radeon_device pointer
1491 *
1492 * Initialize the power states used in profile mode
1493 * (sumo, trinity, SI).
1494 * Used for profile mode only.
1495 */
1496void sumo_pm_init_profile(struct radeon_device *rdev)
1497{
1498 int idx;
1499
1500 /* default */
1501 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1502 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1503 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1505
1506 /* low,mid sh/mh */
1507 if (rdev->flags & RADEON_IS_MOBILITY)
1508 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1509 else
1510 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1511
1512 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1513 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1514 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1515 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1516
1517 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1519 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1520 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1521
1522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1526
1527 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1530 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1531
1532 /* high sh/mh */
1533 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1534 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1538 rdev->pm.power_state[idx].num_clock_modes - 1;
1539
1540 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1544 rdev->pm.power_state[idx].num_clock_modes - 1;
1545}
1546
1547/**
1548 * btc_pm_init_profile - Initialize power profiles callback.
1549 *
1550 * @rdev: radeon_device pointer
1551 *
1552 * Initialize the power states used in profile mode
1553 * (BTC, cayman).
1554 * Used for profile mode only.
1555 */
1556void btc_pm_init_profile(struct radeon_device *rdev)
1557{
1558 int idx;
1559
1560 /* default */
1561 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1562 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1563 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1564 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1565 /* starting with BTC, there is one state that is used for both
1566 * MH and SH. Difference is that we always use the high clock index for
1567 * mclk.
1568 */
1569 if (rdev->flags & RADEON_IS_MOBILITY)
1570 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1571 else
1572 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1573 /* low sh */
1574 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1575 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1576 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1577 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1578 /* mid sh */
1579 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1580 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1581 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1582 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1583 /* high sh */
1584 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1585 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1586 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1587 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1588 /* low mh */
1589 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1590 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1591 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1592 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1593 /* mid mh */
1594 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1595 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1596 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1597 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1598 /* high mh */
1599 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1601 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1602 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1603}
1604
1605/**
1606 * evergreen_pm_misc - set additional pm hw parameters callback.
1607 *
1608 * @rdev: radeon_device pointer
1609 *
1610 * Set non-clock parameters associated with a power state
1611 * (voltage, etc.) (evergreen+).
1612 */
1613void evergreen_pm_misc(struct radeon_device *rdev)
1614{
1615 int req_ps_idx = rdev->pm.requested_power_state_index;
1616 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1617 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1618 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1619
1620 if (voltage->type == VOLTAGE_SW) {
1621 /* 0xff0x are flags rather then an actual voltage */
1622 if ((voltage->voltage & 0xff00) == 0xff00)
1623 return;
1624 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1625 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1626 rdev->pm.current_vddc = voltage->voltage;
1627 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1628 }
1629
1630 /* starting with BTC, there is one state that is used for both
1631 * MH and SH. Difference is that we always use the high clock index for
1632 * mclk and vddci.
1633 */
1634 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1635 (rdev->family >= CHIP_BARTS) &&
1636 rdev->pm.active_crtc_count &&
1637 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1638 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1639 voltage = &rdev->pm.power_state[req_ps_idx].
1640 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1641
1642 /* 0xff0x are flags rather then an actual voltage */
1643 if ((voltage->vddci & 0xff00) == 0xff00)
1644 return;
1645 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1646 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1647 rdev->pm.current_vddci = voltage->vddci;
1648 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1649 }
1650 }
1651}
1652
1653/**
1654 * evergreen_pm_prepare - pre-power state change callback.
1655 *
1656 * @rdev: radeon_device pointer
1657 *
1658 * Prepare for a power state change (evergreen+).
1659 */
1660void evergreen_pm_prepare(struct radeon_device *rdev)
1661{
1662 struct drm_device *ddev = rdev->ddev;
1663 struct drm_crtc *crtc;
1664 struct radeon_crtc *radeon_crtc;
1665 u32 tmp;
1666
1667 /* disable any active CRTCs */
1668 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1669 radeon_crtc = to_radeon_crtc(crtc);
1670 if (radeon_crtc->enabled) {
1671 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1672 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1673 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1674 }
1675 }
1676}
1677
1678/**
1679 * evergreen_pm_finish - post-power state change callback.
1680 *
1681 * @rdev: radeon_device pointer
1682 *
1683 * Clean up after a power state change (evergreen+).
1684 */
1685void evergreen_pm_finish(struct radeon_device *rdev)
1686{
1687 struct drm_device *ddev = rdev->ddev;
1688 struct drm_crtc *crtc;
1689 struct radeon_crtc *radeon_crtc;
1690 u32 tmp;
1691
1692 /* enable any active CRTCs */
1693 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1694 radeon_crtc = to_radeon_crtc(crtc);
1695 if (radeon_crtc->enabled) {
1696 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1697 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1698 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1699 }
1700 }
1701}
1702
1703/**
1704 * evergreen_hpd_sense - hpd sense callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 * @hpd: hpd (hotplug detect) pin
1708 *
1709 * Checks if a digital monitor is connected (evergreen+).
1710 * Returns true if connected, false if not connected.
1711 */
1712bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1713{
1714 bool connected = false;
1715
1716 switch (hpd) {
1717 case RADEON_HPD_1:
1718 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1719 connected = true;
1720 break;
1721 case RADEON_HPD_2:
1722 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1723 connected = true;
1724 break;
1725 case RADEON_HPD_3:
1726 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1727 connected = true;
1728 break;
1729 case RADEON_HPD_4:
1730 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1731 connected = true;
1732 break;
1733 case RADEON_HPD_5:
1734 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1735 connected = true;
1736 break;
1737 case RADEON_HPD_6:
1738 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1739 connected = true;
1740 break;
1741 default:
1742 break;
1743 }
1744
1745 return connected;
1746}
1747
1748/**
1749 * evergreen_hpd_set_polarity - hpd set polarity callback.
1750 *
1751 * @rdev: radeon_device pointer
1752 * @hpd: hpd (hotplug detect) pin
1753 *
1754 * Set the polarity of the hpd pin (evergreen+).
1755 */
1756void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1757 enum radeon_hpd_id hpd)
1758{
1759 u32 tmp;
1760 bool connected = evergreen_hpd_sense(rdev, hpd);
1761
1762 switch (hpd) {
1763 case RADEON_HPD_1:
1764 tmp = RREG32(DC_HPD1_INT_CONTROL);
1765 if (connected)
1766 tmp &= ~DC_HPDx_INT_POLARITY;
1767 else
1768 tmp |= DC_HPDx_INT_POLARITY;
1769 WREG32(DC_HPD1_INT_CONTROL, tmp);
1770 break;
1771 case RADEON_HPD_2:
1772 tmp = RREG32(DC_HPD2_INT_CONTROL);
1773 if (connected)
1774 tmp &= ~DC_HPDx_INT_POLARITY;
1775 else
1776 tmp |= DC_HPDx_INT_POLARITY;
1777 WREG32(DC_HPD2_INT_CONTROL, tmp);
1778 break;
1779 case RADEON_HPD_3:
1780 tmp = RREG32(DC_HPD3_INT_CONTROL);
1781 if (connected)
1782 tmp &= ~DC_HPDx_INT_POLARITY;
1783 else
1784 tmp |= DC_HPDx_INT_POLARITY;
1785 WREG32(DC_HPD3_INT_CONTROL, tmp);
1786 break;
1787 case RADEON_HPD_4:
1788 tmp = RREG32(DC_HPD4_INT_CONTROL);
1789 if (connected)
1790 tmp &= ~DC_HPDx_INT_POLARITY;
1791 else
1792 tmp |= DC_HPDx_INT_POLARITY;
1793 WREG32(DC_HPD4_INT_CONTROL, tmp);
1794 break;
1795 case RADEON_HPD_5:
1796 tmp = RREG32(DC_HPD5_INT_CONTROL);
1797 if (connected)
1798 tmp &= ~DC_HPDx_INT_POLARITY;
1799 else
1800 tmp |= DC_HPDx_INT_POLARITY;
1801 WREG32(DC_HPD5_INT_CONTROL, tmp);
1802 break;
1803 case RADEON_HPD_6:
1804 tmp = RREG32(DC_HPD6_INT_CONTROL);
1805 if (connected)
1806 tmp &= ~DC_HPDx_INT_POLARITY;
1807 else
1808 tmp |= DC_HPDx_INT_POLARITY;
1809 WREG32(DC_HPD6_INT_CONTROL, tmp);
1810 break;
1811 default:
1812 break;
1813 }
1814}
1815
1816/**
1817 * evergreen_hpd_init - hpd setup callback.
1818 *
1819 * @rdev: radeon_device pointer
1820 *
1821 * Setup the hpd pins used by the card (evergreen+).
1822 * Enable the pin, set the polarity, and enable the hpd interrupts.
1823 */
1824void evergreen_hpd_init(struct radeon_device *rdev)
1825{
1826 struct drm_device *dev = rdev->ddev;
1827 struct drm_connector *connector;
1828 unsigned enabled = 0;
1829 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1830 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1831
1832 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1833 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1834
1835 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1836 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1837 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1838 * aux dp channel on imac and help (but not completely fix)
1839 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1840 * also avoid interrupt storms during dpms.
1841 */
1842 continue;
1843 }
1844 switch (radeon_connector->hpd.hpd) {
1845 case RADEON_HPD_1:
1846 WREG32(DC_HPD1_CONTROL, tmp);
1847 break;
1848 case RADEON_HPD_2:
1849 WREG32(DC_HPD2_CONTROL, tmp);
1850 break;
1851 case RADEON_HPD_3:
1852 WREG32(DC_HPD3_CONTROL, tmp);
1853 break;
1854 case RADEON_HPD_4:
1855 WREG32(DC_HPD4_CONTROL, tmp);
1856 break;
1857 case RADEON_HPD_5:
1858 WREG32(DC_HPD5_CONTROL, tmp);
1859 break;
1860 case RADEON_HPD_6:
1861 WREG32(DC_HPD6_CONTROL, tmp);
1862 break;
1863 default:
1864 break;
1865 }
1866 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1867 enabled |= 1 << radeon_connector->hpd.hpd;
1868 }
1869 radeon_irq_kms_enable_hpd(rdev, enabled);
1870}
1871
1872/**
1873 * evergreen_hpd_fini - hpd tear down callback.
1874 *
1875 * @rdev: radeon_device pointer
1876 *
1877 * Tear down the hpd pins used by the card (evergreen+).
1878 * Disable the hpd interrupts.
1879 */
1880void evergreen_hpd_fini(struct radeon_device *rdev)
1881{
1882 struct drm_device *dev = rdev->ddev;
1883 struct drm_connector *connector;
1884 unsigned disabled = 0;
1885
1886 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1887 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1888 switch (radeon_connector->hpd.hpd) {
1889 case RADEON_HPD_1:
1890 WREG32(DC_HPD1_CONTROL, 0);
1891 break;
1892 case RADEON_HPD_2:
1893 WREG32(DC_HPD2_CONTROL, 0);
1894 break;
1895 case RADEON_HPD_3:
1896 WREG32(DC_HPD3_CONTROL, 0);
1897 break;
1898 case RADEON_HPD_4:
1899 WREG32(DC_HPD4_CONTROL, 0);
1900 break;
1901 case RADEON_HPD_5:
1902 WREG32(DC_HPD5_CONTROL, 0);
1903 break;
1904 case RADEON_HPD_6:
1905 WREG32(DC_HPD6_CONTROL, 0);
1906 break;
1907 default:
1908 break;
1909 }
1910 disabled |= 1 << radeon_connector->hpd.hpd;
1911 }
1912 radeon_irq_kms_disable_hpd(rdev, disabled);
1913}
1914
1915/* watermark setup */
1916
1917static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1918 struct radeon_crtc *radeon_crtc,
1919 struct drm_display_mode *mode,
1920 struct drm_display_mode *other_mode)
1921{
1922 u32 tmp, buffer_alloc, i;
1923 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1924 /*
1925 * Line Buffer Setup
1926 * There are 3 line buffers, each one shared by 2 display controllers.
1927 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1928 * the display controllers. The paritioning is done via one of four
1929 * preset allocations specified in bits 2:0:
1930 * first display controller
1931 * 0 - first half of lb (3840 * 2)
1932 * 1 - first 3/4 of lb (5760 * 2)
1933 * 2 - whole lb (7680 * 2), other crtc must be disabled
1934 * 3 - first 1/4 of lb (1920 * 2)
1935 * second display controller
1936 * 4 - second half of lb (3840 * 2)
1937 * 5 - second 3/4 of lb (5760 * 2)
1938 * 6 - whole lb (7680 * 2), other crtc must be disabled
1939 * 7 - last 1/4 of lb (1920 * 2)
1940 */
1941 /* this can get tricky if we have two large displays on a paired group
1942 * of crtcs. Ideally for multiple large displays we'd assign them to
1943 * non-linked crtcs for maximum line buffer allocation.
1944 */
1945 if (radeon_crtc->base.enabled && mode) {
1946 if (other_mode) {
1947 tmp = 0; /* 1/2 */
1948 buffer_alloc = 1;
1949 } else {
1950 tmp = 2; /* whole */
1951 buffer_alloc = 2;
1952 }
1953 } else {
1954 tmp = 0;
1955 buffer_alloc = 0;
1956 }
1957
1958 /* second controller of the pair uses second half of the lb */
1959 if (radeon_crtc->crtc_id % 2)
1960 tmp += 4;
1961 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1962
1963 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1964 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1965 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1966 for (i = 0; i < rdev->usec_timeout; i++) {
1967 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1968 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1969 break;
1970 udelay(1);
1971 }
1972 }
1973
1974 if (radeon_crtc->base.enabled && mode) {
1975 switch (tmp) {
1976 case 0:
1977 case 4:
1978 default:
1979 if (ASIC_IS_DCE5(rdev))
1980 return 4096 * 2;
1981 else
1982 return 3840 * 2;
1983 case 1:
1984 case 5:
1985 if (ASIC_IS_DCE5(rdev))
1986 return 6144 * 2;
1987 else
1988 return 5760 * 2;
1989 case 2:
1990 case 6:
1991 if (ASIC_IS_DCE5(rdev))
1992 return 8192 * 2;
1993 else
1994 return 7680 * 2;
1995 case 3:
1996 case 7:
1997 if (ASIC_IS_DCE5(rdev))
1998 return 2048 * 2;
1999 else
2000 return 1920 * 2;
2001 }
2002 }
2003
2004 /* controller not enabled, so no lb used */
2005 return 0;
2006}
2007
2008u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
2009{
2010 u32 tmp = RREG32(MC_SHARED_CHMAP);
2011
2012 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2013 case 0:
2014 default:
2015 return 1;
2016 case 1:
2017 return 2;
2018 case 2:
2019 return 4;
2020 case 3:
2021 return 8;
2022 }
2023}
2024
2025struct evergreen_wm_params {
2026 u32 dram_channels; /* number of dram channels */
2027 u32 yclk; /* bandwidth per dram data pin in kHz */
2028 u32 sclk; /* engine clock in kHz */
2029 u32 disp_clk; /* display clock in kHz */
2030 u32 src_width; /* viewport width */
2031 u32 active_time; /* active display time in ns */
2032 u32 blank_time; /* blank time in ns */
2033 bool interlaced; /* mode is interlaced */
2034 fixed20_12 vsc; /* vertical scale ratio */
2035 u32 num_heads; /* number of active crtcs */
2036 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2037 u32 lb_size; /* line buffer allocated to pipe */
2038 u32 vtaps; /* vertical scaler taps */
2039};
2040
2041static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
2042{
2043 /* Calculate DRAM Bandwidth and the part allocated to display. */
2044 fixed20_12 dram_efficiency; /* 0.7 */
2045 fixed20_12 yclk, dram_channels, bandwidth;
2046 fixed20_12 a;
2047
2048 a.full = dfixed_const(1000);
2049 yclk.full = dfixed_const(wm->yclk);
2050 yclk.full = dfixed_div(yclk, a);
2051 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2052 a.full = dfixed_const(10);
2053 dram_efficiency.full = dfixed_const(7);
2054 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2055 bandwidth.full = dfixed_mul(dram_channels, yclk);
2056 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2057
2058 return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2062{
2063 /* Calculate DRAM Bandwidth and the part allocated to display. */
2064 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2065 fixed20_12 yclk, dram_channels, bandwidth;
2066 fixed20_12 a;
2067
2068 a.full = dfixed_const(1000);
2069 yclk.full = dfixed_const(wm->yclk);
2070 yclk.full = dfixed_div(yclk, a);
2071 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2072 a.full = dfixed_const(10);
2073 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2074 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2075 bandwidth.full = dfixed_mul(dram_channels, yclk);
2076 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2077
2078 return dfixed_trunc(bandwidth);
2079}
2080
2081static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2082{
2083 /* Calculate the display Data return Bandwidth */
2084 fixed20_12 return_efficiency; /* 0.8 */
2085 fixed20_12 sclk, bandwidth;
2086 fixed20_12 a;
2087
2088 a.full = dfixed_const(1000);
2089 sclk.full = dfixed_const(wm->sclk);
2090 sclk.full = dfixed_div(sclk, a);
2091 a.full = dfixed_const(10);
2092 return_efficiency.full = dfixed_const(8);
2093 return_efficiency.full = dfixed_div(return_efficiency, a);
2094 a.full = dfixed_const(32);
2095 bandwidth.full = dfixed_mul(a, sclk);
2096 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2097
2098 return dfixed_trunc(bandwidth);
2099}
2100
2101static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2102{
2103 /* Calculate the DMIF Request Bandwidth */
2104 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2105 fixed20_12 disp_clk, bandwidth;
2106 fixed20_12 a;
2107
2108 a.full = dfixed_const(1000);
2109 disp_clk.full = dfixed_const(wm->disp_clk);
2110 disp_clk.full = dfixed_div(disp_clk, a);
2111 a.full = dfixed_const(10);
2112 disp_clk_request_efficiency.full = dfixed_const(8);
2113 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2114 a.full = dfixed_const(32);
2115 bandwidth.full = dfixed_mul(a, disp_clk);
2116 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2117
2118 return dfixed_trunc(bandwidth);
2119}
2120
2121static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2122{
2123 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2124 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2125 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2126 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2127
2128 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2129}
2130
2131static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2132{
2133 /* Calculate the display mode Average Bandwidth
2134 * DisplayMode should contain the source and destination dimensions,
2135 * timing, etc.
2136 */
2137 fixed20_12 bpp;
2138 fixed20_12 line_time;
2139 fixed20_12 src_width;
2140 fixed20_12 bandwidth;
2141 fixed20_12 a;
2142
2143 a.full = dfixed_const(1000);
2144 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2145 line_time.full = dfixed_div(line_time, a);
2146 bpp.full = dfixed_const(wm->bytes_per_pixel);
2147 src_width.full = dfixed_const(wm->src_width);
2148 bandwidth.full = dfixed_mul(src_width, bpp);
2149 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2150 bandwidth.full = dfixed_div(bandwidth, line_time);
2151
2152 return dfixed_trunc(bandwidth);
2153}
2154
2155static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2156{
2157 /* First calcualte the latency in ns */
2158 u32 mc_latency = 2000; /* 2000 ns. */
2159 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2160 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2161 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2162 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2163 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2164 (wm->num_heads * cursor_line_pair_return_time);
2165 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2166 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2167 fixed20_12 a, b, c;
2168
2169 if (wm->num_heads == 0)
2170 return 0;
2171
2172 a.full = dfixed_const(2);
2173 b.full = dfixed_const(1);
2174 if ((wm->vsc.full > a.full) ||
2175 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2176 (wm->vtaps >= 5) ||
2177 ((wm->vsc.full >= a.full) && wm->interlaced))
2178 max_src_lines_per_dst_line = 4;
2179 else
2180 max_src_lines_per_dst_line = 2;
2181
2182 a.full = dfixed_const(available_bandwidth);
2183 b.full = dfixed_const(wm->num_heads);
2184 a.full = dfixed_div(a, b);
2185
2186 b.full = dfixed_const(1000);
2187 c.full = dfixed_const(wm->disp_clk);
2188 b.full = dfixed_div(c, b);
2189 c.full = dfixed_const(wm->bytes_per_pixel);
2190 b.full = dfixed_mul(b, c);
2191
2192 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2193
2194 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2195 b.full = dfixed_const(1000);
2196 c.full = dfixed_const(lb_fill_bw);
2197 b.full = dfixed_div(c, b);
2198 a.full = dfixed_div(a, b);
2199 line_fill_time = dfixed_trunc(a);
2200
2201 if (line_fill_time < wm->active_time)
2202 return latency;
2203 else
2204 return latency + (line_fill_time - wm->active_time);
2205
2206}
2207
2208static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2209{
2210 if (evergreen_average_bandwidth(wm) <=
2211 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2212 return true;
2213 else
2214 return false;
2215};
2216
2217static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2218{
2219 if (evergreen_average_bandwidth(wm) <=
2220 (evergreen_available_bandwidth(wm) / wm->num_heads))
2221 return true;
2222 else
2223 return false;
2224};
2225
2226static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2227{
2228 u32 lb_partitions = wm->lb_size / wm->src_width;
2229 u32 line_time = wm->active_time + wm->blank_time;
2230 u32 latency_tolerant_lines;
2231 u32 latency_hiding;
2232 fixed20_12 a;
2233
2234 a.full = dfixed_const(1);
2235 if (wm->vsc.full > a.full)
2236 latency_tolerant_lines = 1;
2237 else {
2238 if (lb_partitions <= (wm->vtaps + 1))
2239 latency_tolerant_lines = 1;
2240 else
2241 latency_tolerant_lines = 2;
2242 }
2243
2244 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2245
2246 if (evergreen_latency_watermark(wm) <= latency_hiding)
2247 return true;
2248 else
2249 return false;
2250}
2251
2252static void evergreen_program_watermarks(struct radeon_device *rdev,
2253 struct radeon_crtc *radeon_crtc,
2254 u32 lb_size, u32 num_heads)
2255{
2256 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2257 struct evergreen_wm_params wm_low, wm_high;
2258 u32 dram_channels;
2259 u32 pixel_period;
2260 u32 line_time = 0;
2261 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2262 u32 priority_a_mark = 0, priority_b_mark = 0;
2263 u32 priority_a_cnt = PRIORITY_OFF;
2264 u32 priority_b_cnt = PRIORITY_OFF;
2265 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2266 u32 tmp, arb_control3;
2267 fixed20_12 a, b, c;
2268
2269 if (radeon_crtc->base.enabled && num_heads && mode) {
2270 pixel_period = 1000000 / (u32)mode->clock;
2271 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2272 priority_a_cnt = 0;
2273 priority_b_cnt = 0;
2274 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275
2276 /* watermark for high clocks */
2277 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2278 wm_high.yclk =
2279 radeon_dpm_get_mclk(rdev, false) * 10;
2280 wm_high.sclk =
2281 radeon_dpm_get_sclk(rdev, false) * 10;
2282 } else {
2283 wm_high.yclk = rdev->pm.current_mclk * 10;
2284 wm_high.sclk = rdev->pm.current_sclk * 10;
2285 }
2286
2287 wm_high.disp_clk = mode->clock;
2288 wm_high.src_width = mode->crtc_hdisplay;
2289 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2290 wm_high.blank_time = line_time - wm_high.active_time;
2291 wm_high.interlaced = false;
2292 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2293 wm_high.interlaced = true;
2294 wm_high.vsc = radeon_crtc->vsc;
2295 wm_high.vtaps = 1;
2296 if (radeon_crtc->rmx_type != RMX_OFF)
2297 wm_high.vtaps = 2;
2298 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2299 wm_high.lb_size = lb_size;
2300 wm_high.dram_channels = dram_channels;
2301 wm_high.num_heads = num_heads;
2302
2303 /* watermark for low clocks */
2304 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2305 wm_low.yclk =
2306 radeon_dpm_get_mclk(rdev, true) * 10;
2307 wm_low.sclk =
2308 radeon_dpm_get_sclk(rdev, true) * 10;
2309 } else {
2310 wm_low.yclk = rdev->pm.current_mclk * 10;
2311 wm_low.sclk = rdev->pm.current_sclk * 10;
2312 }
2313
2314 wm_low.disp_clk = mode->clock;
2315 wm_low.src_width = mode->crtc_hdisplay;
2316 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2317 wm_low.blank_time = line_time - wm_low.active_time;
2318 wm_low.interlaced = false;
2319 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2320 wm_low.interlaced = true;
2321 wm_low.vsc = radeon_crtc->vsc;
2322 wm_low.vtaps = 1;
2323 if (radeon_crtc->rmx_type != RMX_OFF)
2324 wm_low.vtaps = 2;
2325 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2326 wm_low.lb_size = lb_size;
2327 wm_low.dram_channels = dram_channels;
2328 wm_low.num_heads = num_heads;
2329
2330 /* set for high clocks */
2331 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2332 /* set for low clocks */
2333 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2334
2335 /* possibly force display priority to high */
2336 /* should really do this at mode validation time... */
2337 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2338 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2339 !evergreen_check_latency_hiding(&wm_high) ||
2340 (rdev->disp_priority == 2)) {
2341 DRM_DEBUG_KMS("force priority a to high\n");
2342 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2343 }
2344 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2345 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2346 !evergreen_check_latency_hiding(&wm_low) ||
2347 (rdev->disp_priority == 2)) {
2348 DRM_DEBUG_KMS("force priority b to high\n");
2349 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2350 }
2351
2352 a.full = dfixed_const(1000);
2353 b.full = dfixed_const(mode->clock);
2354 b.full = dfixed_div(b, a);
2355 c.full = dfixed_const(latency_watermark_a);
2356 c.full = dfixed_mul(c, b);
2357 c.full = dfixed_mul(c, radeon_crtc->hsc);
2358 c.full = dfixed_div(c, a);
2359 a.full = dfixed_const(16);
2360 c.full = dfixed_div(c, a);
2361 priority_a_mark = dfixed_trunc(c);
2362 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2363
2364 a.full = dfixed_const(1000);
2365 b.full = dfixed_const(mode->clock);
2366 b.full = dfixed_div(b, a);
2367 c.full = dfixed_const(latency_watermark_b);
2368 c.full = dfixed_mul(c, b);
2369 c.full = dfixed_mul(c, radeon_crtc->hsc);
2370 c.full = dfixed_div(c, a);
2371 a.full = dfixed_const(16);
2372 c.full = dfixed_div(c, a);
2373 priority_b_mark = dfixed_trunc(c);
2374 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2375
2376 /* Save number of lines the linebuffer leads before the scanout */
2377 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2378 }
2379
2380 /* select wm A */
2381 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2382 tmp = arb_control3;
2383 tmp &= ~LATENCY_WATERMARK_MASK(3);
2384 tmp |= LATENCY_WATERMARK_MASK(1);
2385 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2386 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2387 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2388 LATENCY_HIGH_WATERMARK(line_time)));
2389 /* select wm B */
2390 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2391 tmp &= ~LATENCY_WATERMARK_MASK(3);
2392 tmp |= LATENCY_WATERMARK_MASK(2);
2393 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2394 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2395 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2396 LATENCY_HIGH_WATERMARK(line_time)));
2397 /* restore original selection */
2398 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2399
2400 /* write the priority marks */
2401 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2402 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2403
2404 /* save values for DPM */
2405 radeon_crtc->line_time = line_time;
2406 radeon_crtc->wm_high = latency_watermark_a;
2407 radeon_crtc->wm_low = latency_watermark_b;
2408}
2409
2410/**
2411 * evergreen_bandwidth_update - update display watermarks callback.
2412 *
2413 * @rdev: radeon_device pointer
2414 *
2415 * Update the display watermarks based on the requested mode(s)
2416 * (evergreen+).
2417 */
2418void evergreen_bandwidth_update(struct radeon_device *rdev)
2419{
2420 struct drm_display_mode *mode0 = NULL;
2421 struct drm_display_mode *mode1 = NULL;
2422 u32 num_heads = 0, lb_size;
2423 int i;
2424
2425 if (!rdev->mode_info.mode_config_initialized)
2426 return;
2427
2428 radeon_update_display_priority(rdev);
2429
2430 for (i = 0; i < rdev->num_crtc; i++) {
2431 if (rdev->mode_info.crtcs[i]->base.enabled)
2432 num_heads++;
2433 }
2434 for (i = 0; i < rdev->num_crtc; i += 2) {
2435 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2436 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2437 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2438 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2439 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2440 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2441 }
2442}
2443
2444/**
2445 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2446 *
2447 * @rdev: radeon_device pointer
2448 *
2449 * Wait for the MC (memory controller) to be idle.
2450 * (evergreen+).
2451 * Returns 0 if the MC is idle, -1 if not.
2452 */
2453int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2454{
2455 unsigned i;
2456 u32 tmp;
2457
2458 for (i = 0; i < rdev->usec_timeout; i++) {
2459 /* read MC_STATUS */
2460 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2461 if (!tmp)
2462 return 0;
2463 udelay(1);
2464 }
2465 return -1;
2466}
2467
2468/*
2469 * GART
2470 */
2471void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2472{
2473 unsigned i;
2474 u32 tmp;
2475
2476 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2477
2478 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2479 for (i = 0; i < rdev->usec_timeout; i++) {
2480 /* read MC_STATUS */
2481 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2482 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2483 if (tmp == 2) {
2484 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2485 return;
2486 }
2487 if (tmp) {
2488 return;
2489 }
2490 udelay(1);
2491 }
2492}
2493
2494static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2495{
2496 u32 tmp;
2497 int r;
2498
2499 if (rdev->gart.robj == NULL) {
2500 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2501 return -EINVAL;
2502 }
2503 r = radeon_gart_table_vram_pin(rdev);
2504 if (r)
2505 return r;
2506 /* Setup L2 cache */
2507 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2508 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2509 EFFECTIVE_L2_QUEUE_SIZE(7));
2510 WREG32(VM_L2_CNTL2, 0);
2511 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2512 /* Setup TLB control */
2513 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2514 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2515 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2516 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2517 if (rdev->flags & RADEON_IS_IGP) {
2518 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2519 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2520 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2521 } else {
2522 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2523 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2524 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2525 if ((rdev->family == CHIP_JUNIPER) ||
2526 (rdev->family == CHIP_CYPRESS) ||
2527 (rdev->family == CHIP_HEMLOCK) ||
2528 (rdev->family == CHIP_BARTS))
2529 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2530 }
2531 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2532 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2533 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2534 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2535 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2536 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2537 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2538 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2539 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2540 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2541 (u32)(rdev->dummy_page.addr >> 12));
2542 WREG32(VM_CONTEXT1_CNTL, 0);
2543
2544 evergreen_pcie_gart_tlb_flush(rdev);
2545 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2546 (unsigned)(rdev->mc.gtt_size >> 20),
2547 (unsigned long long)rdev->gart.table_addr);
2548 rdev->gart.ready = true;
2549 return 0;
2550}
2551
2552static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2553{
2554 u32 tmp;
2555
2556 /* Disable all tables */
2557 WREG32(VM_CONTEXT0_CNTL, 0);
2558 WREG32(VM_CONTEXT1_CNTL, 0);
2559
2560 /* Setup L2 cache */
2561 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2562 EFFECTIVE_L2_QUEUE_SIZE(7));
2563 WREG32(VM_L2_CNTL2, 0);
2564 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2565 /* Setup TLB control */
2566 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2567 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2568 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2569 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2570 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2571 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2572 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2573 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2574 radeon_gart_table_vram_unpin(rdev);
2575}
2576
2577static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2578{
2579 evergreen_pcie_gart_disable(rdev);
2580 radeon_gart_table_vram_free(rdev);
2581 radeon_gart_fini(rdev);
2582}
2583
2584
2585static void evergreen_agp_enable(struct radeon_device *rdev)
2586{
2587 u32 tmp;
2588
2589 /* Setup L2 cache */
2590 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2591 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2592 EFFECTIVE_L2_QUEUE_SIZE(7));
2593 WREG32(VM_L2_CNTL2, 0);
2594 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2595 /* Setup TLB control */
2596 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2597 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2598 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2599 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2600 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2601 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2602 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2603 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2604 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2605 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2606 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2607 WREG32(VM_CONTEXT0_CNTL, 0);
2608 WREG32(VM_CONTEXT1_CNTL, 0);
2609}
2610
2611static const unsigned ni_dig_offsets[] =
2612{
2613 NI_DIG0_REGISTER_OFFSET,
2614 NI_DIG1_REGISTER_OFFSET,
2615 NI_DIG2_REGISTER_OFFSET,
2616 NI_DIG3_REGISTER_OFFSET,
2617 NI_DIG4_REGISTER_OFFSET,
2618 NI_DIG5_REGISTER_OFFSET
2619};
2620
2621static const unsigned ni_tx_offsets[] =
2622{
2623 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2624 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2625 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2626 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2627 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2628 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2629};
2630
2631static const unsigned evergreen_dp_offsets[] =
2632{
2633 EVERGREEN_DP0_REGISTER_OFFSET,
2634 EVERGREEN_DP1_REGISTER_OFFSET,
2635 EVERGREEN_DP2_REGISTER_OFFSET,
2636 EVERGREEN_DP3_REGISTER_OFFSET,
2637 EVERGREEN_DP4_REGISTER_OFFSET,
2638 EVERGREEN_DP5_REGISTER_OFFSET
2639};
2640
2641
2642/*
2643 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2644 * We go from crtc to connector and it is not relible since it
2645 * should be an opposite direction .If crtc is enable then
2646 * find the dig_fe which selects this crtc and insure that it enable.
2647 * if such dig_fe is found then find dig_be which selects found dig_be and
2648 * insure that it enable and in DP_SST mode.
2649 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2650 * from dp symbols clocks .
2651 */
2652static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2653 unsigned crtc_id, unsigned *ret_dig_fe)
2654{
2655 unsigned i;
2656 unsigned dig_fe;
2657 unsigned dig_be;
2658 unsigned dig_en_be;
2659 unsigned uniphy_pll;
2660 unsigned digs_fe_selected;
2661 unsigned dig_be_mode;
2662 unsigned dig_fe_mask;
2663 bool is_enabled = false;
2664 bool found_crtc = false;
2665
2666 /* loop through all running dig_fe to find selected crtc */
2667 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2668 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2669 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2670 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2671 /* found running pipe */
2672 found_crtc = true;
2673 dig_fe_mask = 1 << i;
2674 dig_fe = i;
2675 break;
2676 }
2677 }
2678
2679 if (found_crtc) {
2680 /* loop through all running dig_be to find selected dig_fe */
2681 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2682 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2683 /* if dig_fe_selected by dig_be? */
2684 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2685 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2686 if (dig_fe_mask & digs_fe_selected &&
2687 /* if dig_be in sst mode? */
2688 dig_be_mode == NI_DIG_BE_DPSST) {
2689 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2690 ni_dig_offsets[i]);
2691 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2692 ni_tx_offsets[i]);
2693 /* dig_be enable and tx is running */
2694 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2695 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2696 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2697 is_enabled = true;
2698 *ret_dig_fe = dig_fe;
2699 break;
2700 }
2701 }
2702 }
2703 }
2704
2705 return is_enabled;
2706}
2707
2708/*
2709 * Blank dig when in dp sst mode
2710 * Dig ignores crtc timing
2711 */
2712static void evergreen_blank_dp_output(struct radeon_device *rdev,
2713 unsigned dig_fe)
2714{
2715 unsigned stream_ctrl;
2716 unsigned fifo_ctrl;
2717 unsigned counter = 0;
2718
2719 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2720 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2721 return;
2722 }
2723
2724 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2725 evergreen_dp_offsets[dig_fe]);
2726 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2727 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2728 return;
2729 }
2730
2731 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2732 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2733 evergreen_dp_offsets[dig_fe], stream_ctrl);
2734
2735 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2736 evergreen_dp_offsets[dig_fe]);
2737 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2738 msleep(1);
2739 counter++;
2740 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2741 evergreen_dp_offsets[dig_fe]);
2742 }
2743 if (counter >= 32 )
2744 DRM_ERROR("counter exceeds %d\n", counter);
2745
2746 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2747 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2748 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2749
2750}
2751
2752void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2753{
2754 u32 crtc_enabled, tmp, frame_count, blackout;
2755 int i, j;
2756 unsigned dig_fe;
2757
2758 if (!ASIC_IS_NODCE(rdev)) {
2759 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2760 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2761
2762 /* disable VGA render */
2763 WREG32(VGA_RENDER_CONTROL, 0);
2764 }
2765 /* blank the display controllers */
2766 for (i = 0; i < rdev->num_crtc; i++) {
2767 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2768 if (crtc_enabled) {
2769 save->crtc_enabled[i] = true;
2770 if (ASIC_IS_DCE6(rdev)) {
2771 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2772 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2773 radeon_wait_for_vblank(rdev, i);
2774 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2775 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2776 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2777 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2778 }
2779 } else {
2780 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2781 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2782 radeon_wait_for_vblank(rdev, i);
2783 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2784 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2785 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2786 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2787 }
2788 }
2789 /* wait for the next frame */
2790 frame_count = radeon_get_vblank_counter(rdev, i);
2791 for (j = 0; j < rdev->usec_timeout; j++) {
2792 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2793 break;
2794 udelay(1);
2795 }
2796 /*we should disable dig if it drives dp sst*/
2797 /*but we are in radeon_device_init and the topology is unknown*/
2798 /*and it is available after radeon_modeset_init*/
2799 /*the following method radeon_atom_encoder_dpms_dig*/
2800 /*does the job if we initialize it properly*/
2801 /*for now we do it this manually*/
2802 /**/
2803 if (ASIC_IS_DCE5(rdev) &&
2804 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2805 evergreen_blank_dp_output(rdev, dig_fe);
2806 /*we could remove 6 lines below*/
2807 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2808 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2809 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2810 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2811 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2812 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2813 save->crtc_enabled[i] = false;
2814 /* ***** */
2815 } else {
2816 save->crtc_enabled[i] = false;
2817 }
2818 }
2819
2820 radeon_mc_wait_for_idle(rdev);
2821
2822 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2823 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2824 /* Block CPU access */
2825 WREG32(BIF_FB_EN, 0);
2826 /* blackout the MC */
2827 blackout &= ~BLACKOUT_MODE_MASK;
2828 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2829 }
2830 /* wait for the MC to settle */
2831 udelay(100);
2832
2833 /* lock double buffered regs */
2834 for (i = 0; i < rdev->num_crtc; i++) {
2835 if (save->crtc_enabled[i]) {
2836 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2837 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2838 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2839 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2840 }
2841 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2842 if (!(tmp & 1)) {
2843 tmp |= 1;
2844 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2845 }
2846 }
2847 }
2848}
2849
2850void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2851{
2852 u32 tmp, frame_count;
2853 int i, j;
2854
2855 /* update crtc base addresses */
2856 for (i = 0; i < rdev->num_crtc; i++) {
2857 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2858 upper_32_bits(rdev->mc.vram_start));
2859 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2860 upper_32_bits(rdev->mc.vram_start));
2861 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2862 (u32)rdev->mc.vram_start);
2863 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2864 (u32)rdev->mc.vram_start);
2865 }
2866
2867 if (!ASIC_IS_NODCE(rdev)) {
2868 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2869 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2870 }
2871
2872 /* unlock regs and wait for update */
2873 for (i = 0; i < rdev->num_crtc; i++) {
2874 if (save->crtc_enabled[i]) {
2875 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2876 if ((tmp & 0x7) != 3) {
2877 tmp &= ~0x7;
2878 tmp |= 0x3;
2879 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2880 }
2881 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2882 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2883 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2884 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2885 }
2886 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2887 if (tmp & 1) {
2888 tmp &= ~1;
2889 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2890 }
2891 for (j = 0; j < rdev->usec_timeout; j++) {
2892 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2893 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2894 break;
2895 udelay(1);
2896 }
2897 }
2898 }
2899
2900 /* unblackout the MC */
2901 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2902 tmp &= ~BLACKOUT_MODE_MASK;
2903 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2904 /* allow CPU access */
2905 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2906
2907 for (i = 0; i < rdev->num_crtc; i++) {
2908 if (save->crtc_enabled[i]) {
2909 if (ASIC_IS_DCE6(rdev)) {
2910 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2911 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2912 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2913 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2914 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2915 } else {
2916 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2917 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2918 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2919 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2920 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2921 }
2922 /* wait for the next frame */
2923 frame_count = radeon_get_vblank_counter(rdev, i);
2924 for (j = 0; j < rdev->usec_timeout; j++) {
2925 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2926 break;
2927 udelay(1);
2928 }
2929 }
2930 }
2931 if (!ASIC_IS_NODCE(rdev)) {
2932 /* Unlock vga access */
2933 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2934 mdelay(1);
2935 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2936 }
2937}
2938
2939void evergreen_mc_program(struct radeon_device *rdev)
2940{
2941 struct evergreen_mc_save save;
2942 u32 tmp;
2943 int i, j;
2944
2945 /* Initialize HDP */
2946 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2947 WREG32((0x2c14 + j), 0x00000000);
2948 WREG32((0x2c18 + j), 0x00000000);
2949 WREG32((0x2c1c + j), 0x00000000);
2950 WREG32((0x2c20 + j), 0x00000000);
2951 WREG32((0x2c24 + j), 0x00000000);
2952 }
2953 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2954
2955 evergreen_mc_stop(rdev, &save);
2956 if (evergreen_mc_wait_for_idle(rdev)) {
2957 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2958 }
2959 /* Lockout access through VGA aperture*/
2960 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2961 /* Update configuration */
2962 if (rdev->flags & RADEON_IS_AGP) {
2963 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2964 /* VRAM before AGP */
2965 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2966 rdev->mc.vram_start >> 12);
2967 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2968 rdev->mc.gtt_end >> 12);
2969 } else {
2970 /* VRAM after AGP */
2971 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2972 rdev->mc.gtt_start >> 12);
2973 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2974 rdev->mc.vram_end >> 12);
2975 }
2976 } else {
2977 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2978 rdev->mc.vram_start >> 12);
2979 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2980 rdev->mc.vram_end >> 12);
2981 }
2982 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2983 /* llano/ontario only */
2984 if ((rdev->family == CHIP_PALM) ||
2985 (rdev->family == CHIP_SUMO) ||
2986 (rdev->family == CHIP_SUMO2)) {
2987 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2988 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2989 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2990 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2991 }
2992 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2993 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2994 WREG32(MC_VM_FB_LOCATION, tmp);
2995 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2996 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2997 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2998 if (rdev->flags & RADEON_IS_AGP) {
2999 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
3000 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
3001 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
3002 } else {
3003 WREG32(MC_VM_AGP_BASE, 0);
3004 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3005 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3006 }
3007 if (evergreen_mc_wait_for_idle(rdev)) {
3008 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3009 }
3010 evergreen_mc_resume(rdev, &save);
3011 /* we need to own VRAM, so turn off the VGA renderer here
3012 * to stop it overwriting our objects */
3013 rv515_vga_render_disable(rdev);
3014}
3015
3016/*
3017 * CP.
3018 */
3019void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3020{
3021 struct radeon_ring *ring = &rdev->ring[ib->ring];
3022 u32 next_rptr;
3023
3024 /* set to DX10/11 mode */
3025 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
3026 radeon_ring_write(ring, 1);
3027
3028 if (ring->rptr_save_reg) {
3029 next_rptr = ring->wptr + 3 + 4;
3030 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3031 radeon_ring_write(ring, ((ring->rptr_save_reg -
3032 PACKET3_SET_CONFIG_REG_START) >> 2));
3033 radeon_ring_write(ring, next_rptr);
3034 } else if (rdev->wb.enabled) {
3035 next_rptr = ring->wptr + 5 + 4;
3036 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3037 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3038 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3039 radeon_ring_write(ring, next_rptr);
3040 radeon_ring_write(ring, 0);
3041 }
3042
3043 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3044 radeon_ring_write(ring,
3045#ifdef __BIG_ENDIAN
3046 (2 << 0) |
3047#endif
3048 (ib->gpu_addr & 0xFFFFFFFC));
3049 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3050 radeon_ring_write(ring, ib->length_dw);
3051}
3052
3053
3054static int evergreen_cp_load_microcode(struct radeon_device *rdev)
3055{
3056 const __be32 *fw_data;
3057 int i;
3058
3059 if (!rdev->me_fw || !rdev->pfp_fw)
3060 return -EINVAL;
3061
3062 r700_cp_stop(rdev);
3063 WREG32(CP_RB_CNTL,
3064#ifdef __BIG_ENDIAN
3065 BUF_SWAP_32BIT |
3066#endif
3067 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
3068
3069 fw_data = (const __be32 *)rdev->pfp_fw->data;
3070 WREG32(CP_PFP_UCODE_ADDR, 0);
3071 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
3072 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3073 WREG32(CP_PFP_UCODE_ADDR, 0);
3074
3075 fw_data = (const __be32 *)rdev->me_fw->data;
3076 WREG32(CP_ME_RAM_WADDR, 0);
3077 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
3078 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3079
3080 WREG32(CP_PFP_UCODE_ADDR, 0);
3081 WREG32(CP_ME_RAM_WADDR, 0);
3082 WREG32(CP_ME_RAM_RADDR, 0);
3083 return 0;
3084}
3085
3086static int evergreen_cp_start(struct radeon_device *rdev)
3087{
3088 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3089 int r, i;
3090 uint32_t cp_me;
3091
3092 r = radeon_ring_lock(rdev, ring, 7);
3093 if (r) {
3094 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3095 return r;
3096 }
3097 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3098 radeon_ring_write(ring, 0x1);
3099 radeon_ring_write(ring, 0x0);
3100 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3101 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3102 radeon_ring_write(ring, 0);
3103 radeon_ring_write(ring, 0);
3104 radeon_ring_unlock_commit(rdev, ring, false);
3105
3106 cp_me = 0xff;
3107 WREG32(CP_ME_CNTL, cp_me);
3108
3109 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3110 if (r) {
3111 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3112 return r;
3113 }
3114
3115 /* setup clear context state */
3116 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3117 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3118
3119 for (i = 0; i < evergreen_default_size; i++)
3120 radeon_ring_write(ring, evergreen_default_state[i]);
3121
3122 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3123 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3124
3125 /* set clear context state */
3126 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3127 radeon_ring_write(ring, 0);
3128
3129 /* SQ_VTX_BASE_VTX_LOC */
3130 radeon_ring_write(ring, 0xc0026f00);
3131 radeon_ring_write(ring, 0x00000000);
3132 radeon_ring_write(ring, 0x00000000);
3133 radeon_ring_write(ring, 0x00000000);
3134
3135 /* Clear consts */
3136 radeon_ring_write(ring, 0xc0036f00);
3137 radeon_ring_write(ring, 0x00000bc4);
3138 radeon_ring_write(ring, 0xffffffff);
3139 radeon_ring_write(ring, 0xffffffff);
3140 radeon_ring_write(ring, 0xffffffff);
3141
3142 radeon_ring_write(ring, 0xc0026900);
3143 radeon_ring_write(ring, 0x00000316);
3144 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3145 radeon_ring_write(ring, 0x00000010); /* */
3146
3147 radeon_ring_unlock_commit(rdev, ring, false);
3148
3149 return 0;
3150}
3151
3152static int evergreen_cp_resume(struct radeon_device *rdev)
3153{
3154 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3155 u32 tmp;
3156 u32 rb_bufsz;
3157 int r;
3158
3159 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3160 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3161 SOFT_RESET_PA |
3162 SOFT_RESET_SH |
3163 SOFT_RESET_VGT |
3164 SOFT_RESET_SPI |
3165 SOFT_RESET_SX));
3166 RREG32(GRBM_SOFT_RESET);
3167 mdelay(15);
3168 WREG32(GRBM_SOFT_RESET, 0);
3169 RREG32(GRBM_SOFT_RESET);
3170
3171 /* Set ring buffer size */
3172 rb_bufsz = order_base_2(ring->ring_size / 8);
3173 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3174#ifdef __BIG_ENDIAN
3175 tmp |= BUF_SWAP_32BIT;
3176#endif
3177 WREG32(CP_RB_CNTL, tmp);
3178 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3179 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3180
3181 /* Set the write pointer delay */
3182 WREG32(CP_RB_WPTR_DELAY, 0);
3183
3184 /* Initialize the ring buffer's read and write pointers */
3185 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3186 WREG32(CP_RB_RPTR_WR, 0);
3187 ring->wptr = 0;
3188 WREG32(CP_RB_WPTR, ring->wptr);
3189
3190 /* set the wb address whether it's enabled or not */
3191 WREG32(CP_RB_RPTR_ADDR,
3192 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3193 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3194 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3195
3196 if (rdev->wb.enabled)
3197 WREG32(SCRATCH_UMSK, 0xff);
3198 else {
3199 tmp |= RB_NO_UPDATE;
3200 WREG32(SCRATCH_UMSK, 0);
3201 }
3202
3203 mdelay(1);
3204 WREG32(CP_RB_CNTL, tmp);
3205
3206 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3207 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3208
3209 evergreen_cp_start(rdev);
3210 ring->ready = true;
3211 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3212 if (r) {
3213 ring->ready = false;
3214 return r;
3215 }
3216 return 0;
3217}
3218
3219/*
3220 * Core functions
3221 */
3222static void evergreen_gpu_init(struct radeon_device *rdev)
3223{
3224 u32 gb_addr_config;
3225 u32 mc_shared_chmap, mc_arb_ramcfg;
3226 u32 sx_debug_1;
3227 u32 smx_dc_ctl0;
3228 u32 sq_config;
3229 u32 sq_lds_resource_mgmt;
3230 u32 sq_gpr_resource_mgmt_1;
3231 u32 sq_gpr_resource_mgmt_2;
3232 u32 sq_gpr_resource_mgmt_3;
3233 u32 sq_thread_resource_mgmt;
3234 u32 sq_thread_resource_mgmt_2;
3235 u32 sq_stack_resource_mgmt_1;
3236 u32 sq_stack_resource_mgmt_2;
3237 u32 sq_stack_resource_mgmt_3;
3238 u32 vgt_cache_invalidation;
3239 u32 hdp_host_path_cntl, tmp;
3240 u32 disabled_rb_mask;
3241 int i, j, ps_thread_count;
3242
3243 switch (rdev->family) {
3244 case CHIP_CYPRESS:
3245 case CHIP_HEMLOCK:
3246 rdev->config.evergreen.num_ses = 2;
3247 rdev->config.evergreen.max_pipes = 4;
3248 rdev->config.evergreen.max_tile_pipes = 8;
3249 rdev->config.evergreen.max_simds = 10;
3250 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3251 rdev->config.evergreen.max_gprs = 256;
3252 rdev->config.evergreen.max_threads = 248;
3253 rdev->config.evergreen.max_gs_threads = 32;
3254 rdev->config.evergreen.max_stack_entries = 512;
3255 rdev->config.evergreen.sx_num_of_sets = 4;
3256 rdev->config.evergreen.sx_max_export_size = 256;
3257 rdev->config.evergreen.sx_max_export_pos_size = 64;
3258 rdev->config.evergreen.sx_max_export_smx_size = 192;
3259 rdev->config.evergreen.max_hw_contexts = 8;
3260 rdev->config.evergreen.sq_num_cf_insts = 2;
3261
3262 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3263 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3264 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3265 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3266 break;
3267 case CHIP_JUNIPER:
3268 rdev->config.evergreen.num_ses = 1;
3269 rdev->config.evergreen.max_pipes = 4;
3270 rdev->config.evergreen.max_tile_pipes = 4;
3271 rdev->config.evergreen.max_simds = 10;
3272 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3273 rdev->config.evergreen.max_gprs = 256;
3274 rdev->config.evergreen.max_threads = 248;
3275 rdev->config.evergreen.max_gs_threads = 32;
3276 rdev->config.evergreen.max_stack_entries = 512;
3277 rdev->config.evergreen.sx_num_of_sets = 4;
3278 rdev->config.evergreen.sx_max_export_size = 256;
3279 rdev->config.evergreen.sx_max_export_pos_size = 64;
3280 rdev->config.evergreen.sx_max_export_smx_size = 192;
3281 rdev->config.evergreen.max_hw_contexts = 8;
3282 rdev->config.evergreen.sq_num_cf_insts = 2;
3283
3284 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3285 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3286 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3287 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3288 break;
3289 case CHIP_REDWOOD:
3290 rdev->config.evergreen.num_ses = 1;
3291 rdev->config.evergreen.max_pipes = 4;
3292 rdev->config.evergreen.max_tile_pipes = 4;
3293 rdev->config.evergreen.max_simds = 5;
3294 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3295 rdev->config.evergreen.max_gprs = 256;
3296 rdev->config.evergreen.max_threads = 248;
3297 rdev->config.evergreen.max_gs_threads = 32;
3298 rdev->config.evergreen.max_stack_entries = 256;
3299 rdev->config.evergreen.sx_num_of_sets = 4;
3300 rdev->config.evergreen.sx_max_export_size = 256;
3301 rdev->config.evergreen.sx_max_export_pos_size = 64;
3302 rdev->config.evergreen.sx_max_export_smx_size = 192;
3303 rdev->config.evergreen.max_hw_contexts = 8;
3304 rdev->config.evergreen.sq_num_cf_insts = 2;
3305
3306 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3307 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3308 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3309 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3310 break;
3311 case CHIP_CEDAR:
3312 default:
3313 rdev->config.evergreen.num_ses = 1;
3314 rdev->config.evergreen.max_pipes = 2;
3315 rdev->config.evergreen.max_tile_pipes = 2;
3316 rdev->config.evergreen.max_simds = 2;
3317 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3318 rdev->config.evergreen.max_gprs = 256;
3319 rdev->config.evergreen.max_threads = 192;
3320 rdev->config.evergreen.max_gs_threads = 16;
3321 rdev->config.evergreen.max_stack_entries = 256;
3322 rdev->config.evergreen.sx_num_of_sets = 4;
3323 rdev->config.evergreen.sx_max_export_size = 128;
3324 rdev->config.evergreen.sx_max_export_pos_size = 32;
3325 rdev->config.evergreen.sx_max_export_smx_size = 96;
3326 rdev->config.evergreen.max_hw_contexts = 4;
3327 rdev->config.evergreen.sq_num_cf_insts = 1;
3328
3329 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3330 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3331 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3332 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3333 break;
3334 case CHIP_PALM:
3335 rdev->config.evergreen.num_ses = 1;
3336 rdev->config.evergreen.max_pipes = 2;
3337 rdev->config.evergreen.max_tile_pipes = 2;
3338 rdev->config.evergreen.max_simds = 2;
3339 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3340 rdev->config.evergreen.max_gprs = 256;
3341 rdev->config.evergreen.max_threads = 192;
3342 rdev->config.evergreen.max_gs_threads = 16;
3343 rdev->config.evergreen.max_stack_entries = 256;
3344 rdev->config.evergreen.sx_num_of_sets = 4;
3345 rdev->config.evergreen.sx_max_export_size = 128;
3346 rdev->config.evergreen.sx_max_export_pos_size = 32;
3347 rdev->config.evergreen.sx_max_export_smx_size = 96;
3348 rdev->config.evergreen.max_hw_contexts = 4;
3349 rdev->config.evergreen.sq_num_cf_insts = 1;
3350
3351 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3352 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3353 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3354 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3355 break;
3356 case CHIP_SUMO:
3357 rdev->config.evergreen.num_ses = 1;
3358 rdev->config.evergreen.max_pipes = 4;
3359 rdev->config.evergreen.max_tile_pipes = 4;
3360 if (rdev->pdev->device == 0x9648)
3361 rdev->config.evergreen.max_simds = 3;
3362 else if ((rdev->pdev->device == 0x9647) ||
3363 (rdev->pdev->device == 0x964a))
3364 rdev->config.evergreen.max_simds = 4;
3365 else
3366 rdev->config.evergreen.max_simds = 5;
3367 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3368 rdev->config.evergreen.max_gprs = 256;
3369 rdev->config.evergreen.max_threads = 248;
3370 rdev->config.evergreen.max_gs_threads = 32;
3371 rdev->config.evergreen.max_stack_entries = 256;
3372 rdev->config.evergreen.sx_num_of_sets = 4;
3373 rdev->config.evergreen.sx_max_export_size = 256;
3374 rdev->config.evergreen.sx_max_export_pos_size = 64;
3375 rdev->config.evergreen.sx_max_export_smx_size = 192;
3376 rdev->config.evergreen.max_hw_contexts = 8;
3377 rdev->config.evergreen.sq_num_cf_insts = 2;
3378
3379 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3380 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3381 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3382 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3383 break;
3384 case CHIP_SUMO2:
3385 rdev->config.evergreen.num_ses = 1;
3386 rdev->config.evergreen.max_pipes = 4;
3387 rdev->config.evergreen.max_tile_pipes = 4;
3388 rdev->config.evergreen.max_simds = 2;
3389 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3390 rdev->config.evergreen.max_gprs = 256;
3391 rdev->config.evergreen.max_threads = 248;
3392 rdev->config.evergreen.max_gs_threads = 32;
3393 rdev->config.evergreen.max_stack_entries = 512;
3394 rdev->config.evergreen.sx_num_of_sets = 4;
3395 rdev->config.evergreen.sx_max_export_size = 256;
3396 rdev->config.evergreen.sx_max_export_pos_size = 64;
3397 rdev->config.evergreen.sx_max_export_smx_size = 192;
3398 rdev->config.evergreen.max_hw_contexts = 4;
3399 rdev->config.evergreen.sq_num_cf_insts = 2;
3400
3401 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3402 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3403 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3404 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3405 break;
3406 case CHIP_BARTS:
3407 rdev->config.evergreen.num_ses = 2;
3408 rdev->config.evergreen.max_pipes = 4;
3409 rdev->config.evergreen.max_tile_pipes = 8;
3410 rdev->config.evergreen.max_simds = 7;
3411 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3412 rdev->config.evergreen.max_gprs = 256;
3413 rdev->config.evergreen.max_threads = 248;
3414 rdev->config.evergreen.max_gs_threads = 32;
3415 rdev->config.evergreen.max_stack_entries = 512;
3416 rdev->config.evergreen.sx_num_of_sets = 4;
3417 rdev->config.evergreen.sx_max_export_size = 256;
3418 rdev->config.evergreen.sx_max_export_pos_size = 64;
3419 rdev->config.evergreen.sx_max_export_smx_size = 192;
3420 rdev->config.evergreen.max_hw_contexts = 8;
3421 rdev->config.evergreen.sq_num_cf_insts = 2;
3422
3423 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3424 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3425 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3426 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3427 break;
3428 case CHIP_TURKS:
3429 rdev->config.evergreen.num_ses = 1;
3430 rdev->config.evergreen.max_pipes = 4;
3431 rdev->config.evergreen.max_tile_pipes = 4;
3432 rdev->config.evergreen.max_simds = 6;
3433 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3434 rdev->config.evergreen.max_gprs = 256;
3435 rdev->config.evergreen.max_threads = 248;
3436 rdev->config.evergreen.max_gs_threads = 32;
3437 rdev->config.evergreen.max_stack_entries = 256;
3438 rdev->config.evergreen.sx_num_of_sets = 4;
3439 rdev->config.evergreen.sx_max_export_size = 256;
3440 rdev->config.evergreen.sx_max_export_pos_size = 64;
3441 rdev->config.evergreen.sx_max_export_smx_size = 192;
3442 rdev->config.evergreen.max_hw_contexts = 8;
3443 rdev->config.evergreen.sq_num_cf_insts = 2;
3444
3445 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3446 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3447 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3448 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3449 break;
3450 case CHIP_CAICOS:
3451 rdev->config.evergreen.num_ses = 1;
3452 rdev->config.evergreen.max_pipes = 2;
3453 rdev->config.evergreen.max_tile_pipes = 2;
3454 rdev->config.evergreen.max_simds = 2;
3455 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3456 rdev->config.evergreen.max_gprs = 256;
3457 rdev->config.evergreen.max_threads = 192;
3458 rdev->config.evergreen.max_gs_threads = 16;
3459 rdev->config.evergreen.max_stack_entries = 256;
3460 rdev->config.evergreen.sx_num_of_sets = 4;
3461 rdev->config.evergreen.sx_max_export_size = 128;
3462 rdev->config.evergreen.sx_max_export_pos_size = 32;
3463 rdev->config.evergreen.sx_max_export_smx_size = 96;
3464 rdev->config.evergreen.max_hw_contexts = 4;
3465 rdev->config.evergreen.sq_num_cf_insts = 1;
3466
3467 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3468 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3469 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3470 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3471 break;
3472 }
3473
3474 /* Initialize HDP */
3475 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3476 WREG32((0x2c14 + j), 0x00000000);
3477 WREG32((0x2c18 + j), 0x00000000);
3478 WREG32((0x2c1c + j), 0x00000000);
3479 WREG32((0x2c20 + j), 0x00000000);
3480 WREG32((0x2c24 + j), 0x00000000);
3481 }
3482
3483 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3484 WREG32(SRBM_INT_CNTL, 0x1);
3485 WREG32(SRBM_INT_ACK, 0x1);
3486
3487 evergreen_fix_pci_max_read_req_size(rdev);
3488
3489 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3490 if ((rdev->family == CHIP_PALM) ||
3491 (rdev->family == CHIP_SUMO) ||
3492 (rdev->family == CHIP_SUMO2))
3493 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3494 else
3495 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3496
3497 /* setup tiling info dword. gb_addr_config is not adequate since it does
3498 * not have bank info, so create a custom tiling dword.
3499 * bits 3:0 num_pipes
3500 * bits 7:4 num_banks
3501 * bits 11:8 group_size
3502 * bits 15:12 row_size
3503 */
3504 rdev->config.evergreen.tile_config = 0;
3505 switch (rdev->config.evergreen.max_tile_pipes) {
3506 case 1:
3507 default:
3508 rdev->config.evergreen.tile_config |= (0 << 0);
3509 break;
3510 case 2:
3511 rdev->config.evergreen.tile_config |= (1 << 0);
3512 break;
3513 case 4:
3514 rdev->config.evergreen.tile_config |= (2 << 0);
3515 break;
3516 case 8:
3517 rdev->config.evergreen.tile_config |= (3 << 0);
3518 break;
3519 }
3520 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3521 if (rdev->flags & RADEON_IS_IGP)
3522 rdev->config.evergreen.tile_config |= 1 << 4;
3523 else {
3524 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3525 case 0: /* four banks */
3526 rdev->config.evergreen.tile_config |= 0 << 4;
3527 break;
3528 case 1: /* eight banks */
3529 rdev->config.evergreen.tile_config |= 1 << 4;
3530 break;
3531 case 2: /* sixteen banks */
3532 default:
3533 rdev->config.evergreen.tile_config |= 2 << 4;
3534 break;
3535 }
3536 }
3537 rdev->config.evergreen.tile_config |= 0 << 8;
3538 rdev->config.evergreen.tile_config |=
3539 ((gb_addr_config & 0x30000000) >> 28) << 12;
3540
3541 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3542 u32 efuse_straps_4;
3543 u32 efuse_straps_3;
3544
3545 efuse_straps_4 = RREG32_RCU(0x204);
3546 efuse_straps_3 = RREG32_RCU(0x203);
3547 tmp = (((efuse_straps_4 & 0xf) << 4) |
3548 ((efuse_straps_3 & 0xf0000000) >> 28));
3549 } else {
3550 tmp = 0;
3551 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3552 u32 rb_disable_bitmap;
3553
3554 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3555 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3556 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3557 tmp <<= 4;
3558 tmp |= rb_disable_bitmap;
3559 }
3560 }
3561 /* enabled rb are just the one not disabled :) */
3562 disabled_rb_mask = tmp;
3563 tmp = 0;
3564 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3565 tmp |= (1 << i);
3566 /* if all the backends are disabled, fix it up here */
3567 if ((disabled_rb_mask & tmp) == tmp) {
3568 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3569 disabled_rb_mask &= ~(1 << i);
3570 }
3571
3572 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3573 u32 simd_disable_bitmap;
3574
3575 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3576 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3577 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3578 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3579 tmp <<= 16;
3580 tmp |= simd_disable_bitmap;
3581 }
3582 rdev->config.evergreen.active_simds = hweight32(~tmp);
3583
3584 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3585 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3586
3587 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3588 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3589 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3590 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3591 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3592 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3593 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3594
3595 if ((rdev->config.evergreen.max_backends == 1) &&
3596 (rdev->flags & RADEON_IS_IGP)) {
3597 if ((disabled_rb_mask & 3) == 1) {
3598 /* RB0 disabled, RB1 enabled */
3599 tmp = 0x11111111;
3600 } else {
3601 /* RB1 disabled, RB0 enabled */
3602 tmp = 0x00000000;
3603 }
3604 } else {
3605 tmp = gb_addr_config & NUM_PIPES_MASK;
3606 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3607 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3608 }
3609 WREG32(GB_BACKEND_MAP, tmp);
3610
3611 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3612 WREG32(CGTS_TCC_DISABLE, 0);
3613 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3614 WREG32(CGTS_USER_TCC_DISABLE, 0);
3615
3616 /* set HW defaults for 3D engine */
3617 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3618 ROQ_IB2_START(0x2b)));
3619
3620 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3621
3622 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3623 SYNC_GRADIENT |
3624 SYNC_WALKER |
3625 SYNC_ALIGNER));
3626
3627 sx_debug_1 = RREG32(SX_DEBUG_1);
3628 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3629 WREG32(SX_DEBUG_1, sx_debug_1);
3630
3631
3632 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3633 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3634 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3635 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3636
3637 if (rdev->family <= CHIP_SUMO2)
3638 WREG32(SMX_SAR_CTL0, 0x00010000);
3639
3640 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3641 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3642 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3643
3644 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3645 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3646 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3647
3648 WREG32(VGT_NUM_INSTANCES, 1);
3649 WREG32(SPI_CONFIG_CNTL, 0);
3650 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3651 WREG32(CP_PERFMON_CNTL, 0);
3652
3653 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3654 FETCH_FIFO_HIWATER(0x4) |
3655 DONE_FIFO_HIWATER(0xe0) |
3656 ALU_UPDATE_FIFO_HIWATER(0x8)));
3657
3658 sq_config = RREG32(SQ_CONFIG);
3659 sq_config &= ~(PS_PRIO(3) |
3660 VS_PRIO(3) |
3661 GS_PRIO(3) |
3662 ES_PRIO(3));
3663 sq_config |= (VC_ENABLE |
3664 EXPORT_SRC_C |
3665 PS_PRIO(0) |
3666 VS_PRIO(1) |
3667 GS_PRIO(2) |
3668 ES_PRIO(3));
3669
3670 switch (rdev->family) {
3671 case CHIP_CEDAR:
3672 case CHIP_PALM:
3673 case CHIP_SUMO:
3674 case CHIP_SUMO2:
3675 case CHIP_CAICOS:
3676 /* no vertex cache */
3677 sq_config &= ~VC_ENABLE;
3678 break;
3679 default:
3680 break;
3681 }
3682
3683 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3684
3685 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3686 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3687 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3688 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3689 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3690 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3691 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3692
3693 switch (rdev->family) {
3694 case CHIP_CEDAR:
3695 case CHIP_PALM:
3696 case CHIP_SUMO:
3697 case CHIP_SUMO2:
3698 ps_thread_count = 96;
3699 break;
3700 default:
3701 ps_thread_count = 128;
3702 break;
3703 }
3704
3705 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3706 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3707 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3708 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3709 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3710 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3711
3712 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3713 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3714 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3715 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3716 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3717 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3718
3719 WREG32(SQ_CONFIG, sq_config);
3720 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3721 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3722 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3723 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3724 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3725 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3726 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3727 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3728 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3729 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3730
3731 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3732 FORCE_EOV_MAX_REZ_CNT(255)));
3733
3734 switch (rdev->family) {
3735 case CHIP_CEDAR:
3736 case CHIP_PALM:
3737 case CHIP_SUMO:
3738 case CHIP_SUMO2:
3739 case CHIP_CAICOS:
3740 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3741 break;
3742 default:
3743 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3744 break;
3745 }
3746 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3747 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3748
3749 WREG32(VGT_GS_VERTEX_REUSE, 16);
3750 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3751 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3752
3753 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3754 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3755
3756 WREG32(CB_PERF_CTR0_SEL_0, 0);
3757 WREG32(CB_PERF_CTR0_SEL_1, 0);
3758 WREG32(CB_PERF_CTR1_SEL_0, 0);
3759 WREG32(CB_PERF_CTR1_SEL_1, 0);
3760 WREG32(CB_PERF_CTR2_SEL_0, 0);
3761 WREG32(CB_PERF_CTR2_SEL_1, 0);
3762 WREG32(CB_PERF_CTR3_SEL_0, 0);
3763 WREG32(CB_PERF_CTR3_SEL_1, 0);
3764
3765 /* clear render buffer base addresses */
3766 WREG32(CB_COLOR0_BASE, 0);
3767 WREG32(CB_COLOR1_BASE, 0);
3768 WREG32(CB_COLOR2_BASE, 0);
3769 WREG32(CB_COLOR3_BASE, 0);
3770 WREG32(CB_COLOR4_BASE, 0);
3771 WREG32(CB_COLOR5_BASE, 0);
3772 WREG32(CB_COLOR6_BASE, 0);
3773 WREG32(CB_COLOR7_BASE, 0);
3774 WREG32(CB_COLOR8_BASE, 0);
3775 WREG32(CB_COLOR9_BASE, 0);
3776 WREG32(CB_COLOR10_BASE, 0);
3777 WREG32(CB_COLOR11_BASE, 0);
3778
3779 /* set the shader const cache sizes to 0 */
3780 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3781 WREG32(i, 0);
3782 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3783 WREG32(i, 0);
3784
3785 tmp = RREG32(HDP_MISC_CNTL);
3786 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3787 WREG32(HDP_MISC_CNTL, tmp);
3788
3789 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3790 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3791
3792 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3793
3794 udelay(50);
3795
3796}
3797
3798int evergreen_mc_init(struct radeon_device *rdev)
3799{
3800 u32 tmp;
3801 int chansize, numchan;
3802
3803 /* Get VRAM informations */
3804 rdev->mc.vram_is_ddr = true;
3805 if ((rdev->family == CHIP_PALM) ||
3806 (rdev->family == CHIP_SUMO) ||
3807 (rdev->family == CHIP_SUMO2))
3808 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3809 else
3810 tmp = RREG32(MC_ARB_RAMCFG);
3811 if (tmp & CHANSIZE_OVERRIDE) {
3812 chansize = 16;
3813 } else if (tmp & CHANSIZE_MASK) {
3814 chansize = 64;
3815 } else {
3816 chansize = 32;
3817 }
3818 tmp = RREG32(MC_SHARED_CHMAP);
3819 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3820 case 0:
3821 default:
3822 numchan = 1;
3823 break;
3824 case 1:
3825 numchan = 2;
3826 break;
3827 case 2:
3828 numchan = 4;
3829 break;
3830 case 3:
3831 numchan = 8;
3832 break;
3833 }
3834 rdev->mc.vram_width = numchan * chansize;
3835 /* Could aper size report 0 ? */
3836 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3837 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3838 /* Setup GPU memory space */
3839 if ((rdev->family == CHIP_PALM) ||
3840 (rdev->family == CHIP_SUMO) ||
3841 (rdev->family == CHIP_SUMO2)) {
3842 /* size in bytes on fusion */
3843 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3844 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3845 } else {
3846 /* size in MB on evergreen/cayman/tn */
3847 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3848 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3849 }
3850 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3851 r700_vram_gtt_location(rdev, &rdev->mc);
3852 radeon_update_bandwidth_info(rdev);
3853
3854 return 0;
3855}
3856
3857void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3858{
3859 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3860 RREG32(GRBM_STATUS));
3861 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3862 RREG32(GRBM_STATUS_SE0));
3863 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3864 RREG32(GRBM_STATUS_SE1));
3865 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3866 RREG32(SRBM_STATUS));
3867 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3868 RREG32(SRBM_STATUS2));
3869 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3870 RREG32(CP_STALLED_STAT1));
3871 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3872 RREG32(CP_STALLED_STAT2));
3873 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3874 RREG32(CP_BUSY_STAT));
3875 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3876 RREG32(CP_STAT));
3877 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3878 RREG32(DMA_STATUS_REG));
3879 if (rdev->family >= CHIP_CAYMAN) {
3880 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3881 RREG32(DMA_STATUS_REG + 0x800));
3882 }
3883}
3884
3885bool evergreen_is_display_hung(struct radeon_device *rdev)
3886{
3887 u32 crtc_hung = 0;
3888 u32 crtc_status[6];
3889 u32 i, j, tmp;
3890
3891 for (i = 0; i < rdev->num_crtc; i++) {
3892 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3893 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3894 crtc_hung |= (1 << i);
3895 }
3896 }
3897
3898 for (j = 0; j < 10; j++) {
3899 for (i = 0; i < rdev->num_crtc; i++) {
3900 if (crtc_hung & (1 << i)) {
3901 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3902 if (tmp != crtc_status[i])
3903 crtc_hung &= ~(1 << i);
3904 }
3905 }
3906 if (crtc_hung == 0)
3907 return false;
3908 udelay(100);
3909 }
3910
3911 return true;
3912}
3913
3914u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3915{
3916 u32 reset_mask = 0;
3917 u32 tmp;
3918
3919 /* GRBM_STATUS */
3920 tmp = RREG32(GRBM_STATUS);
3921 if (tmp & (PA_BUSY | SC_BUSY |
3922 SH_BUSY | SX_BUSY |
3923 TA_BUSY | VGT_BUSY |
3924 DB_BUSY | CB_BUSY |
3925 SPI_BUSY | VGT_BUSY_NO_DMA))
3926 reset_mask |= RADEON_RESET_GFX;
3927
3928 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3929 CP_BUSY | CP_COHERENCY_BUSY))
3930 reset_mask |= RADEON_RESET_CP;
3931
3932 if (tmp & GRBM_EE_BUSY)
3933 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3934
3935 /* DMA_STATUS_REG */
3936 tmp = RREG32(DMA_STATUS_REG);
3937 if (!(tmp & DMA_IDLE))
3938 reset_mask |= RADEON_RESET_DMA;
3939
3940 /* SRBM_STATUS2 */
3941 tmp = RREG32(SRBM_STATUS2);
3942 if (tmp & DMA_BUSY)
3943 reset_mask |= RADEON_RESET_DMA;
3944
3945 /* SRBM_STATUS */
3946 tmp = RREG32(SRBM_STATUS);
3947 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3948 reset_mask |= RADEON_RESET_RLC;
3949
3950 if (tmp & IH_BUSY)
3951 reset_mask |= RADEON_RESET_IH;
3952
3953 if (tmp & SEM_BUSY)
3954 reset_mask |= RADEON_RESET_SEM;
3955
3956 if (tmp & GRBM_RQ_PENDING)
3957 reset_mask |= RADEON_RESET_GRBM;
3958
3959 if (tmp & VMC_BUSY)
3960 reset_mask |= RADEON_RESET_VMC;
3961
3962 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3963 MCC_BUSY | MCD_BUSY))
3964 reset_mask |= RADEON_RESET_MC;
3965
3966 if (evergreen_is_display_hung(rdev))
3967 reset_mask |= RADEON_RESET_DISPLAY;
3968
3969 /* VM_L2_STATUS */
3970 tmp = RREG32(VM_L2_STATUS);
3971 if (tmp & L2_BUSY)
3972 reset_mask |= RADEON_RESET_VMC;
3973
3974 /* Skip MC reset as it's mostly likely not hung, just busy */
3975 if (reset_mask & RADEON_RESET_MC) {
3976 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3977 reset_mask &= ~RADEON_RESET_MC;
3978 }
3979
3980 return reset_mask;
3981}
3982
3983static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3984{
3985 struct evergreen_mc_save save;
3986 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3987 u32 tmp;
3988
3989 if (reset_mask == 0)
3990 return;
3991
3992 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3993
3994 evergreen_print_gpu_status_regs(rdev);
3995
3996 /* Disable CP parsing/prefetching */
3997 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3998
3999 if (reset_mask & RADEON_RESET_DMA) {
4000 /* Disable DMA */
4001 tmp = RREG32(DMA_RB_CNTL);
4002 tmp &= ~DMA_RB_ENABLE;
4003 WREG32(DMA_RB_CNTL, tmp);
4004 }
4005
4006 udelay(50);
4007
4008 evergreen_mc_stop(rdev, &save);
4009 if (evergreen_mc_wait_for_idle(rdev)) {
4010 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4011 }
4012
4013 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
4014 grbm_soft_reset |= SOFT_RESET_DB |
4015 SOFT_RESET_CB |
4016 SOFT_RESET_PA |
4017 SOFT_RESET_SC |
4018 SOFT_RESET_SPI |
4019 SOFT_RESET_SX |
4020 SOFT_RESET_SH |
4021 SOFT_RESET_TC |
4022 SOFT_RESET_TA |
4023 SOFT_RESET_VC |
4024 SOFT_RESET_VGT;
4025 }
4026
4027 if (reset_mask & RADEON_RESET_CP) {
4028 grbm_soft_reset |= SOFT_RESET_CP |
4029 SOFT_RESET_VGT;
4030
4031 srbm_soft_reset |= SOFT_RESET_GRBM;
4032 }
4033
4034 if (reset_mask & RADEON_RESET_DMA)
4035 srbm_soft_reset |= SOFT_RESET_DMA;
4036
4037 if (reset_mask & RADEON_RESET_DISPLAY)
4038 srbm_soft_reset |= SOFT_RESET_DC;
4039
4040 if (reset_mask & RADEON_RESET_RLC)
4041 srbm_soft_reset |= SOFT_RESET_RLC;
4042
4043 if (reset_mask & RADEON_RESET_SEM)
4044 srbm_soft_reset |= SOFT_RESET_SEM;
4045
4046 if (reset_mask & RADEON_RESET_IH)
4047 srbm_soft_reset |= SOFT_RESET_IH;
4048
4049 if (reset_mask & RADEON_RESET_GRBM)
4050 srbm_soft_reset |= SOFT_RESET_GRBM;
4051
4052 if (reset_mask & RADEON_RESET_VMC)
4053 srbm_soft_reset |= SOFT_RESET_VMC;
4054
4055 if (!(rdev->flags & RADEON_IS_IGP)) {
4056 if (reset_mask & RADEON_RESET_MC)
4057 srbm_soft_reset |= SOFT_RESET_MC;
4058 }
4059
4060 if (grbm_soft_reset) {
4061 tmp = RREG32(GRBM_SOFT_RESET);
4062 tmp |= grbm_soft_reset;
4063 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4064 WREG32(GRBM_SOFT_RESET, tmp);
4065 tmp = RREG32(GRBM_SOFT_RESET);
4066
4067 udelay(50);
4068
4069 tmp &= ~grbm_soft_reset;
4070 WREG32(GRBM_SOFT_RESET, tmp);
4071 tmp = RREG32(GRBM_SOFT_RESET);
4072 }
4073
4074 if (srbm_soft_reset) {
4075 tmp = RREG32(SRBM_SOFT_RESET);
4076 tmp |= srbm_soft_reset;
4077 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4078 WREG32(SRBM_SOFT_RESET, tmp);
4079 tmp = RREG32(SRBM_SOFT_RESET);
4080
4081 udelay(50);
4082
4083 tmp &= ~srbm_soft_reset;
4084 WREG32(SRBM_SOFT_RESET, tmp);
4085 tmp = RREG32(SRBM_SOFT_RESET);
4086 }
4087
4088 /* Wait a little for things to settle down */
4089 udelay(50);
4090
4091 evergreen_mc_resume(rdev, &save);
4092 udelay(50);
4093
4094 evergreen_print_gpu_status_regs(rdev);
4095}
4096
4097void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4098{
4099 struct evergreen_mc_save save;
4100 u32 tmp, i;
4101
4102 dev_info(rdev->dev, "GPU pci config reset\n");
4103
4104 /* disable dpm? */
4105
4106 /* Disable CP parsing/prefetching */
4107 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4108 udelay(50);
4109 /* Disable DMA */
4110 tmp = RREG32(DMA_RB_CNTL);
4111 tmp &= ~DMA_RB_ENABLE;
4112 WREG32(DMA_RB_CNTL, tmp);
4113 /* XXX other engines? */
4114
4115 /* halt the rlc */
4116 r600_rlc_stop(rdev);
4117
4118 udelay(50);
4119
4120 /* set mclk/sclk to bypass */
4121 rv770_set_clk_bypass_mode(rdev);
4122 /* disable BM */
4123 pci_clear_master(rdev->pdev);
4124 /* disable mem access */
4125 evergreen_mc_stop(rdev, &save);
4126 if (evergreen_mc_wait_for_idle(rdev)) {
4127 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4128 }
4129 /* reset */
4130 radeon_pci_config_reset(rdev);
4131 /* wait for asic to come out of reset */
4132 for (i = 0; i < rdev->usec_timeout; i++) {
4133 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4134 break;
4135 udelay(1);
4136 }
4137}
4138
4139int evergreen_asic_reset(struct radeon_device *rdev)
4140{
4141 u32 reset_mask;
4142
4143 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4144
4145 if (reset_mask)
4146 r600_set_bios_scratch_engine_hung(rdev, true);
4147
4148 /* try soft reset */
4149 evergreen_gpu_soft_reset(rdev, reset_mask);
4150
4151 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4152
4153 /* try pci config reset */
4154 if (reset_mask && radeon_hard_reset)
4155 evergreen_gpu_pci_config_reset(rdev);
4156
4157 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4158
4159 if (!reset_mask)
4160 r600_set_bios_scratch_engine_hung(rdev, false);
4161
4162 return 0;
4163}
4164
4165/**
4166 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4167 *
4168 * @rdev: radeon_device pointer
4169 * @ring: radeon_ring structure holding ring information
4170 *
4171 * Check if the GFX engine is locked up.
4172 * Returns true if the engine appears to be locked up, false if not.
4173 */
4174bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4175{
4176 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4177
4178 if (!(reset_mask & (RADEON_RESET_GFX |
4179 RADEON_RESET_COMPUTE |
4180 RADEON_RESET_CP))) {
4181 radeon_ring_lockup_update(rdev, ring);
4182 return false;
4183 }
4184 return radeon_ring_test_lockup(rdev, ring);
4185}
4186
4187/*
4188 * RLC
4189 */
4190#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4191#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4192
4193void sumo_rlc_fini(struct radeon_device *rdev)
4194{
4195 int r;
4196
4197 /* save restore block */
4198 if (rdev->rlc.save_restore_obj) {
4199 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4200 if (unlikely(r != 0))
4201 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4202 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4203 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4204
4205 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4206 rdev->rlc.save_restore_obj = NULL;
4207 }
4208
4209 /* clear state block */
4210 if (rdev->rlc.clear_state_obj) {
4211 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4212 if (unlikely(r != 0))
4213 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4214 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4215 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4216
4217 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4218 rdev->rlc.clear_state_obj = NULL;
4219 }
4220
4221 /* clear state block */
4222 if (rdev->rlc.cp_table_obj) {
4223 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4224 if (unlikely(r != 0))
4225 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4226 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4227 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4228
4229 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4230 rdev->rlc.cp_table_obj = NULL;
4231 }
4232}
4233
4234#define CP_ME_TABLE_SIZE 96
4235
4236int sumo_rlc_init(struct radeon_device *rdev)
4237{
4238 const u32 *src_ptr;
4239 volatile u32 *dst_ptr;
4240 u32 dws, data, i, j, k, reg_num;
4241 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4242 u64 reg_list_mc_addr;
4243 const struct cs_section_def *cs_data;
4244 int r;
4245
4246 src_ptr = rdev->rlc.reg_list;
4247 dws = rdev->rlc.reg_list_size;
4248 if (rdev->family >= CHIP_BONAIRE) {
4249 dws += (5 * 16) + 48 + 48 + 64;
4250 }
4251 cs_data = rdev->rlc.cs_data;
4252
4253 if (src_ptr) {
4254 /* save restore block */
4255 if (rdev->rlc.save_restore_obj == NULL) {
4256 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4257 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4258 NULL, &rdev->rlc.save_restore_obj);
4259 if (r) {
4260 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4261 return r;
4262 }
4263 }
4264
4265 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4266 if (unlikely(r != 0)) {
4267 sumo_rlc_fini(rdev);
4268 return r;
4269 }
4270 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4271 &rdev->rlc.save_restore_gpu_addr);
4272 if (r) {
4273 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4274 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4275 sumo_rlc_fini(rdev);
4276 return r;
4277 }
4278
4279 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4280 if (r) {
4281 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4282 sumo_rlc_fini(rdev);
4283 return r;
4284 }
4285 /* write the sr buffer */
4286 dst_ptr = rdev->rlc.sr_ptr;
4287 if (rdev->family >= CHIP_TAHITI) {
4288 /* SI */
4289 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4290 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4291 } else {
4292 /* ON/LN/TN */
4293 /* format:
4294 * dw0: (reg2 << 16) | reg1
4295 * dw1: reg1 save space
4296 * dw2: reg2 save space
4297 */
4298 for (i = 0; i < dws; i++) {
4299 data = src_ptr[i] >> 2;
4300 i++;
4301 if (i < dws)
4302 data |= (src_ptr[i] >> 2) << 16;
4303 j = (((i - 1) * 3) / 2);
4304 dst_ptr[j] = cpu_to_le32(data);
4305 }
4306 j = ((i * 3) / 2);
4307 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4308 }
4309 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4310 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4311 }
4312
4313 if (cs_data) {
4314 /* clear state block */
4315 if (rdev->family >= CHIP_BONAIRE) {
4316 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4317 } else if (rdev->family >= CHIP_TAHITI) {
4318 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4319 dws = rdev->rlc.clear_state_size + (256 / 4);
4320 } else {
4321 reg_list_num = 0;
4322 dws = 0;
4323 for (i = 0; cs_data[i].section != NULL; i++) {
4324 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4325 reg_list_num++;
4326 dws += cs_data[i].section[j].reg_count;
4327 }
4328 }
4329 reg_list_blk_index = (3 * reg_list_num + 2);
4330 dws += reg_list_blk_index;
4331 rdev->rlc.clear_state_size = dws;
4332 }
4333
4334 if (rdev->rlc.clear_state_obj == NULL) {
4335 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4336 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4337 NULL, &rdev->rlc.clear_state_obj);
4338 if (r) {
4339 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4340 sumo_rlc_fini(rdev);
4341 return r;
4342 }
4343 }
4344 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4345 if (unlikely(r != 0)) {
4346 sumo_rlc_fini(rdev);
4347 return r;
4348 }
4349 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4350 &rdev->rlc.clear_state_gpu_addr);
4351 if (r) {
4352 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4353 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4354 sumo_rlc_fini(rdev);
4355 return r;
4356 }
4357
4358 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4359 if (r) {
4360 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4361 sumo_rlc_fini(rdev);
4362 return r;
4363 }
4364 /* set up the cs buffer */
4365 dst_ptr = rdev->rlc.cs_ptr;
4366 if (rdev->family >= CHIP_BONAIRE) {
4367 cik_get_csb_buffer(rdev, dst_ptr);
4368 } else if (rdev->family >= CHIP_TAHITI) {
4369 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4370 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4371 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4372 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4373 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4374 } else {
4375 reg_list_hdr_blk_index = 0;
4376 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4377 data = upper_32_bits(reg_list_mc_addr);
4378 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4379 reg_list_hdr_blk_index++;
4380 for (i = 0; cs_data[i].section != NULL; i++) {
4381 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4382 reg_num = cs_data[i].section[j].reg_count;
4383 data = reg_list_mc_addr & 0xffffffff;
4384 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4385 reg_list_hdr_blk_index++;
4386
4387 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4388 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4389 reg_list_hdr_blk_index++;
4390
4391 data = 0x08000000 | (reg_num * 4);
4392 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4393 reg_list_hdr_blk_index++;
4394
4395 for (k = 0; k < reg_num; k++) {
4396 data = cs_data[i].section[j].extent[k];
4397 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4398 }
4399 reg_list_mc_addr += reg_num * 4;
4400 reg_list_blk_index += reg_num;
4401 }
4402 }
4403 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4404 }
4405 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4406 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4407 }
4408
4409 if (rdev->rlc.cp_table_size) {
4410 if (rdev->rlc.cp_table_obj == NULL) {
4411 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4412 PAGE_SIZE, true,
4413 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4414 NULL, &rdev->rlc.cp_table_obj);
4415 if (r) {
4416 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4417 sumo_rlc_fini(rdev);
4418 return r;
4419 }
4420 }
4421
4422 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4423 if (unlikely(r != 0)) {
4424 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4425 sumo_rlc_fini(rdev);
4426 return r;
4427 }
4428 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4429 &rdev->rlc.cp_table_gpu_addr);
4430 if (r) {
4431 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4432 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4433 sumo_rlc_fini(rdev);
4434 return r;
4435 }
4436 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4437 if (r) {
4438 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4439 sumo_rlc_fini(rdev);
4440 return r;
4441 }
4442
4443 cik_init_cp_pg_table(rdev);
4444
4445 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4446 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4447
4448 }
4449
4450 return 0;
4451}
4452
4453static void evergreen_rlc_start(struct radeon_device *rdev)
4454{
4455 u32 mask = RLC_ENABLE;
4456
4457 if (rdev->flags & RADEON_IS_IGP) {
4458 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4459 }
4460
4461 WREG32(RLC_CNTL, mask);
4462}
4463
4464int evergreen_rlc_resume(struct radeon_device *rdev)
4465{
4466 u32 i;
4467 const __be32 *fw_data;
4468
4469 if (!rdev->rlc_fw)
4470 return -EINVAL;
4471
4472 r600_rlc_stop(rdev);
4473
4474 WREG32(RLC_HB_CNTL, 0);
4475
4476 if (rdev->flags & RADEON_IS_IGP) {
4477 if (rdev->family == CHIP_ARUBA) {
4478 u32 always_on_bitmap =
4479 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4480 /* find out the number of active simds */
4481 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4482 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4483 tmp = hweight32(~tmp);
4484 if (tmp == rdev->config.cayman.max_simds_per_se) {
4485 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4486 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4487 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4488 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4489 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4490 }
4491 } else {
4492 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4493 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4494 }
4495 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4496 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4497 } else {
4498 WREG32(RLC_HB_BASE, 0);
4499 WREG32(RLC_HB_RPTR, 0);
4500 WREG32(RLC_HB_WPTR, 0);
4501 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4502 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4503 }
4504 WREG32(RLC_MC_CNTL, 0);
4505 WREG32(RLC_UCODE_CNTL, 0);
4506
4507 fw_data = (const __be32 *)rdev->rlc_fw->data;
4508 if (rdev->family >= CHIP_ARUBA) {
4509 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4510 WREG32(RLC_UCODE_ADDR, i);
4511 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4512 }
4513 } else if (rdev->family >= CHIP_CAYMAN) {
4514 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4515 WREG32(RLC_UCODE_ADDR, i);
4516 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4517 }
4518 } else {
4519 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4520 WREG32(RLC_UCODE_ADDR, i);
4521 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4522 }
4523 }
4524 WREG32(RLC_UCODE_ADDR, 0);
4525
4526 evergreen_rlc_start(rdev);
4527
4528 return 0;
4529}
4530
4531/* Interrupts */
4532
4533u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4534{
4535 if (crtc >= rdev->num_crtc)
4536 return 0;
4537 else
4538 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4539}
4540
4541void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4542{
4543 u32 tmp;
4544
4545 if (rdev->family >= CHIP_CAYMAN) {
4546 cayman_cp_int_cntl_setup(rdev, 0,
4547 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4548 cayman_cp_int_cntl_setup(rdev, 1, 0);
4549 cayman_cp_int_cntl_setup(rdev, 2, 0);
4550 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4551 WREG32(CAYMAN_DMA1_CNTL, tmp);
4552 } else
4553 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4554 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4555 WREG32(DMA_CNTL, tmp);
4556 WREG32(GRBM_INT_CNTL, 0);
4557 WREG32(SRBM_INT_CNTL, 0);
4558 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4559 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4560 if (rdev->num_crtc >= 4) {
4561 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4562 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4563 }
4564 if (rdev->num_crtc >= 6) {
4565 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4566 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4567 }
4568
4569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4570 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4571 if (rdev->num_crtc >= 4) {
4572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4573 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4574 }
4575 if (rdev->num_crtc >= 6) {
4576 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4577 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4578 }
4579
4580 /* only one DAC on DCE5 */
4581 if (!ASIC_IS_DCE5(rdev))
4582 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4583 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4584
4585 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4586 WREG32(DC_HPD1_INT_CONTROL, tmp);
4587 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4588 WREG32(DC_HPD2_INT_CONTROL, tmp);
4589 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4590 WREG32(DC_HPD3_INT_CONTROL, tmp);
4591 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4592 WREG32(DC_HPD4_INT_CONTROL, tmp);
4593 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4594 WREG32(DC_HPD5_INT_CONTROL, tmp);
4595 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4596 WREG32(DC_HPD6_INT_CONTROL, tmp);
4597
4598}
4599
4600int evergreen_irq_set(struct radeon_device *rdev)
4601{
4602 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4603 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4604 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4605 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4606 u32 grbm_int_cntl = 0;
4607 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4608 u32 dma_cntl, dma_cntl1 = 0;
4609 u32 thermal_int = 0;
4610
4611 if (!rdev->irq.installed) {
4612 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4613 return -EINVAL;
4614 }
4615 /* don't enable anything if the ih is disabled */
4616 if (!rdev->ih.enabled) {
4617 r600_disable_interrupts(rdev);
4618 /* force the active interrupt state to all disabled */
4619 evergreen_disable_interrupt_state(rdev);
4620 return 0;
4621 }
4622
4623 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4624 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4625 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4626 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4627 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4628 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4629 if (rdev->family == CHIP_ARUBA)
4630 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4631 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4632 else
4633 thermal_int = RREG32(CG_THERMAL_INT) &
4634 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4635
4636 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4637 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4638 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4639 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4640 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4641 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4642
4643 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4644
4645 if (rdev->family >= CHIP_CAYMAN) {
4646 /* enable CP interrupts on all rings */
4647 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4648 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4649 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4650 }
4651 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4652 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4653 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4654 }
4655 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4656 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4657 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4658 }
4659 } else {
4660 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4661 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4662 cp_int_cntl |= RB_INT_ENABLE;
4663 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4664 }
4665 }
4666
4667 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4668 DRM_DEBUG("r600_irq_set: sw int dma\n");
4669 dma_cntl |= TRAP_ENABLE;
4670 }
4671
4672 if (rdev->family >= CHIP_CAYMAN) {
4673 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4674 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4675 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4676 dma_cntl1 |= TRAP_ENABLE;
4677 }
4678 }
4679
4680 if (rdev->irq.dpm_thermal) {
4681 DRM_DEBUG("dpm thermal\n");
4682 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4683 }
4684
4685 if (rdev->irq.crtc_vblank_int[0] ||
4686 atomic_read(&rdev->irq.pflip[0])) {
4687 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4688 crtc1 |= VBLANK_INT_MASK;
4689 }
4690 if (rdev->irq.crtc_vblank_int[1] ||
4691 atomic_read(&rdev->irq.pflip[1])) {
4692 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4693 crtc2 |= VBLANK_INT_MASK;
4694 }
4695 if (rdev->irq.crtc_vblank_int[2] ||
4696 atomic_read(&rdev->irq.pflip[2])) {
4697 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4698 crtc3 |= VBLANK_INT_MASK;
4699 }
4700 if (rdev->irq.crtc_vblank_int[3] ||
4701 atomic_read(&rdev->irq.pflip[3])) {
4702 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4703 crtc4 |= VBLANK_INT_MASK;
4704 }
4705 if (rdev->irq.crtc_vblank_int[4] ||
4706 atomic_read(&rdev->irq.pflip[4])) {
4707 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4708 crtc5 |= VBLANK_INT_MASK;
4709 }
4710 if (rdev->irq.crtc_vblank_int[5] ||
4711 atomic_read(&rdev->irq.pflip[5])) {
4712 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4713 crtc6 |= VBLANK_INT_MASK;
4714 }
4715 if (rdev->irq.hpd[0]) {
4716 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4717 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4718 }
4719 if (rdev->irq.hpd[1]) {
4720 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4721 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4722 }
4723 if (rdev->irq.hpd[2]) {
4724 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4725 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4726 }
4727 if (rdev->irq.hpd[3]) {
4728 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4729 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4730 }
4731 if (rdev->irq.hpd[4]) {
4732 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4733 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4734 }
4735 if (rdev->irq.hpd[5]) {
4736 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4737 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4738 }
4739 if (rdev->irq.afmt[0]) {
4740 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4741 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4742 }
4743 if (rdev->irq.afmt[1]) {
4744 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4745 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4746 }
4747 if (rdev->irq.afmt[2]) {
4748 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4749 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4750 }
4751 if (rdev->irq.afmt[3]) {
4752 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4753 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4754 }
4755 if (rdev->irq.afmt[4]) {
4756 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4757 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4758 }
4759 if (rdev->irq.afmt[5]) {
4760 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4761 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4762 }
4763
4764 if (rdev->family >= CHIP_CAYMAN) {
4765 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4766 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4767 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4768 } else
4769 WREG32(CP_INT_CNTL, cp_int_cntl);
4770
4771 WREG32(DMA_CNTL, dma_cntl);
4772
4773 if (rdev->family >= CHIP_CAYMAN)
4774 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4775
4776 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4777
4778 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4779 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4780 if (rdev->num_crtc >= 4) {
4781 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4782 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4783 }
4784 if (rdev->num_crtc >= 6) {
4785 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4786 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4787 }
4788
4789 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4790 GRPH_PFLIP_INT_MASK);
4791 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4792 GRPH_PFLIP_INT_MASK);
4793 if (rdev->num_crtc >= 4) {
4794 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4795 GRPH_PFLIP_INT_MASK);
4796 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4797 GRPH_PFLIP_INT_MASK);
4798 }
4799 if (rdev->num_crtc >= 6) {
4800 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4801 GRPH_PFLIP_INT_MASK);
4802 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4803 GRPH_PFLIP_INT_MASK);
4804 }
4805
4806 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4807 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4808 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4809 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4810 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4811 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4812 if (rdev->family == CHIP_ARUBA)
4813 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4814 else
4815 WREG32(CG_THERMAL_INT, thermal_int);
4816
4817 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4818 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4819 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4820 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4821 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4822 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4823
4824 /* posting read */
4825 RREG32(SRBM_STATUS);
4826
4827 return 0;
4828}
4829
4830static void evergreen_irq_ack(struct radeon_device *rdev)
4831{
4832 u32 tmp;
4833
4834 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4835 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4836 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4837 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4838 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4839 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4840 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4841 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4842 if (rdev->num_crtc >= 4) {
4843 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4844 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4845 }
4846 if (rdev->num_crtc >= 6) {
4847 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4848 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4849 }
4850
4851 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4852 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4853 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4854 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4855 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4856 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4857
4858 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4859 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4860 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4861 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4862 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4863 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4864 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4865 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4866 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4867 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4868 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4869 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4870
4871 if (rdev->num_crtc >= 4) {
4872 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4873 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4874 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4875 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4876 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4877 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4878 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4879 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4880 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4881 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4882 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4883 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4884 }
4885
4886 if (rdev->num_crtc >= 6) {
4887 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4888 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4889 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4890 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4891 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4892 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4893 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4894 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4895 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4896 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4897 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4898 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4899 }
4900
4901 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4902 tmp = RREG32(DC_HPD1_INT_CONTROL);
4903 tmp |= DC_HPDx_INT_ACK;
4904 WREG32(DC_HPD1_INT_CONTROL, tmp);
4905 }
4906 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4907 tmp = RREG32(DC_HPD2_INT_CONTROL);
4908 tmp |= DC_HPDx_INT_ACK;
4909 WREG32(DC_HPD2_INT_CONTROL, tmp);
4910 }
4911 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4912 tmp = RREG32(DC_HPD3_INT_CONTROL);
4913 tmp |= DC_HPDx_INT_ACK;
4914 WREG32(DC_HPD3_INT_CONTROL, tmp);
4915 }
4916 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4917 tmp = RREG32(DC_HPD4_INT_CONTROL);
4918 tmp |= DC_HPDx_INT_ACK;
4919 WREG32(DC_HPD4_INT_CONTROL, tmp);
4920 }
4921 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4922 tmp = RREG32(DC_HPD5_INT_CONTROL);
4923 tmp |= DC_HPDx_INT_ACK;
4924 WREG32(DC_HPD5_INT_CONTROL, tmp);
4925 }
4926 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4927 tmp = RREG32(DC_HPD5_INT_CONTROL);
4928 tmp |= DC_HPDx_INT_ACK;
4929 WREG32(DC_HPD6_INT_CONTROL, tmp);
4930 }
4931
4932 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4933 tmp = RREG32(DC_HPD1_INT_CONTROL);
4934 tmp |= DC_HPDx_RX_INT_ACK;
4935 WREG32(DC_HPD1_INT_CONTROL, tmp);
4936 }
4937 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4938 tmp = RREG32(DC_HPD2_INT_CONTROL);
4939 tmp |= DC_HPDx_RX_INT_ACK;
4940 WREG32(DC_HPD2_INT_CONTROL, tmp);
4941 }
4942 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4943 tmp = RREG32(DC_HPD3_INT_CONTROL);
4944 tmp |= DC_HPDx_RX_INT_ACK;
4945 WREG32(DC_HPD3_INT_CONTROL, tmp);
4946 }
4947 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4948 tmp = RREG32(DC_HPD4_INT_CONTROL);
4949 tmp |= DC_HPDx_RX_INT_ACK;
4950 WREG32(DC_HPD4_INT_CONTROL, tmp);
4951 }
4952 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4953 tmp = RREG32(DC_HPD5_INT_CONTROL);
4954 tmp |= DC_HPDx_RX_INT_ACK;
4955 WREG32(DC_HPD5_INT_CONTROL, tmp);
4956 }
4957 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4958 tmp = RREG32(DC_HPD5_INT_CONTROL);
4959 tmp |= DC_HPDx_RX_INT_ACK;
4960 WREG32(DC_HPD6_INT_CONTROL, tmp);
4961 }
4962
4963 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4964 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4965 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4966 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4967 }
4968 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4969 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4970 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4971 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4972 }
4973 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4974 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4975 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4976 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4977 }
4978 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4979 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4980 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4981 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4982 }
4983 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4984 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4985 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4986 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4987 }
4988 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4989 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4990 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4991 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4992 }
4993}
4994
4995static void evergreen_irq_disable(struct radeon_device *rdev)
4996{
4997 r600_disable_interrupts(rdev);
4998 /* Wait and acknowledge irq */
4999 mdelay(1);
5000 evergreen_irq_ack(rdev);
5001 evergreen_disable_interrupt_state(rdev);
5002}
5003
5004void evergreen_irq_suspend(struct radeon_device *rdev)
5005{
5006 evergreen_irq_disable(rdev);
5007 r600_rlc_stop(rdev);
5008}
5009
5010static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
5011{
5012 u32 wptr, tmp;
5013
5014 if (rdev->wb.enabled)
5015 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5016 else
5017 wptr = RREG32(IH_RB_WPTR);
5018
5019 if (wptr & RB_OVERFLOW) {
5020 wptr &= ~RB_OVERFLOW;
5021 /* When a ring buffer overflow happen start parsing interrupt
5022 * from the last not overwritten vector (wptr + 16). Hopefully
5023 * this should allow us to catchup.
5024 */
5025 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
5026 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
5027 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5028 tmp = RREG32(IH_RB_CNTL);
5029 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5030 WREG32(IH_RB_CNTL, tmp);
5031 }
5032 return (wptr & rdev->ih.ptr_mask);
5033}
5034
5035int evergreen_irq_process(struct radeon_device *rdev)
5036{
5037 u32 wptr;
5038 u32 rptr;
5039 u32 src_id, src_data;
5040 u32 ring_index;
5041 bool queue_hotplug = false;
5042 bool queue_hdmi = false;
5043 bool queue_dp = false;
5044 bool queue_thermal = false;
5045 u32 status, addr;
5046
5047 if (!rdev->ih.enabled || rdev->shutdown)
5048 return IRQ_NONE;
5049
5050 wptr = evergreen_get_ih_wptr(rdev);
5051
5052restart_ih:
5053 /* is somebody else already processing irqs? */
5054 if (atomic_xchg(&rdev->ih.lock, 1))
5055 return IRQ_NONE;
5056
5057 rptr = rdev->ih.rptr;
5058 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5059
5060 /* Order reading of wptr vs. reading of IH ring data */
5061 rmb();
5062
5063 /* display interrupts */
5064 evergreen_irq_ack(rdev);
5065
5066 while (rptr != wptr) {
5067 /* wptr/rptr are in bytes! */
5068 ring_index = rptr / 4;
5069 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5070 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5071
5072 switch (src_id) {
5073 case 1: /* D1 vblank/vline */
5074 switch (src_data) {
5075 case 0: /* D1 vblank */
5076 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
5077 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
5078
5079 if (rdev->irq.crtc_vblank_int[0]) {
5080 drm_handle_vblank(rdev->ddev, 0);
5081 rdev->pm.vblank_sync = true;
5082 wake_up(&rdev->irq.vblank_queue);
5083 }
5084 if (atomic_read(&rdev->irq.pflip[0]))
5085 radeon_crtc_handle_vblank(rdev, 0);
5086 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5087 DRM_DEBUG("IH: D1 vblank\n");
5088
5089 break;
5090 case 1: /* D1 vline */
5091 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
5092 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
5093
5094 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5095 DRM_DEBUG("IH: D1 vline\n");
5096
5097 break;
5098 default:
5099 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5100 break;
5101 }
5102 break;
5103 case 2: /* D2 vblank/vline */
5104 switch (src_data) {
5105 case 0: /* D2 vblank */
5106 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
5107 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
5108
5109 if (rdev->irq.crtc_vblank_int[1]) {
5110 drm_handle_vblank(rdev->ddev, 1);
5111 rdev->pm.vblank_sync = true;
5112 wake_up(&rdev->irq.vblank_queue);
5113 }
5114 if (atomic_read(&rdev->irq.pflip[1]))
5115 radeon_crtc_handle_vblank(rdev, 1);
5116 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5117 DRM_DEBUG("IH: D2 vblank\n");
5118
5119 break;
5120 case 1: /* D2 vline */
5121 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
5122 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
5123
5124 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5125 DRM_DEBUG("IH: D2 vline\n");
5126
5127 break;
5128 default:
5129 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5130 break;
5131 }
5132 break;
5133 case 3: /* D3 vblank/vline */
5134 switch (src_data) {
5135 case 0: /* D3 vblank */
5136 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
5137 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
5138
5139 if (rdev->irq.crtc_vblank_int[2]) {
5140 drm_handle_vblank(rdev->ddev, 2);
5141 rdev->pm.vblank_sync = true;
5142 wake_up(&rdev->irq.vblank_queue);
5143 }
5144 if (atomic_read(&rdev->irq.pflip[2]))
5145 radeon_crtc_handle_vblank(rdev, 2);
5146 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5147 DRM_DEBUG("IH: D3 vblank\n");
5148
5149 break;
5150 case 1: /* D3 vline */
5151 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5152 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5153
5154 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5155 DRM_DEBUG("IH: D3 vline\n");
5156
5157 break;
5158 default:
5159 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5160 break;
5161 }
5162 break;
5163 case 4: /* D4 vblank/vline */
5164 switch (src_data) {
5165 case 0: /* D4 vblank */
5166 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5167 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5168
5169 if (rdev->irq.crtc_vblank_int[3]) {
5170 drm_handle_vblank(rdev->ddev, 3);
5171 rdev->pm.vblank_sync = true;
5172 wake_up(&rdev->irq.vblank_queue);
5173 }
5174 if (atomic_read(&rdev->irq.pflip[3]))
5175 radeon_crtc_handle_vblank(rdev, 3);
5176 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5177 DRM_DEBUG("IH: D4 vblank\n");
5178
5179 break;
5180 case 1: /* D4 vline */
5181 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5182 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5183
5184 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5185 DRM_DEBUG("IH: D4 vline\n");
5186
5187 break;
5188 default:
5189 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5190 break;
5191 }
5192 break;
5193 case 5: /* D5 vblank/vline */
5194 switch (src_data) {
5195 case 0: /* D5 vblank */
5196 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5197 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5198
5199 if (rdev->irq.crtc_vblank_int[4]) {
5200 drm_handle_vblank(rdev->ddev, 4);
5201 rdev->pm.vblank_sync = true;
5202 wake_up(&rdev->irq.vblank_queue);
5203 }
5204 if (atomic_read(&rdev->irq.pflip[4]))
5205 radeon_crtc_handle_vblank(rdev, 4);
5206 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5207 DRM_DEBUG("IH: D5 vblank\n");
5208
5209 break;
5210 case 1: /* D5 vline */
5211 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5212 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5213
5214 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5215 DRM_DEBUG("IH: D5 vline\n");
5216
5217 break;
5218 default:
5219 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5220 break;
5221 }
5222 break;
5223 case 6: /* D6 vblank/vline */
5224 switch (src_data) {
5225 case 0: /* D6 vblank */
5226 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5227 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5228
5229 if (rdev->irq.crtc_vblank_int[5]) {
5230 drm_handle_vblank(rdev->ddev, 5);
5231 rdev->pm.vblank_sync = true;
5232 wake_up(&rdev->irq.vblank_queue);
5233 }
5234 if (atomic_read(&rdev->irq.pflip[5]))
5235 radeon_crtc_handle_vblank(rdev, 5);
5236 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5237 DRM_DEBUG("IH: D6 vblank\n");
5238
5239 break;
5240 case 1: /* D6 vline */
5241 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5242 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5243
5244 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5245 DRM_DEBUG("IH: D6 vline\n");
5246
5247 break;
5248 default:
5249 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5250 break;
5251 }
5252 break;
5253 case 8: /* D1 page flip */
5254 case 10: /* D2 page flip */
5255 case 12: /* D3 page flip */
5256 case 14: /* D4 page flip */
5257 case 16: /* D5 page flip */
5258 case 18: /* D6 page flip */
5259 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5260 if (radeon_use_pflipirq > 0)
5261 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5262 break;
5263 case 42: /* HPD hotplug */
5264 switch (src_data) {
5265 case 0:
5266 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5267 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5268
5269 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5270 queue_hotplug = true;
5271 DRM_DEBUG("IH: HPD1\n");
5272 break;
5273 case 1:
5274 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5275 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5276
5277 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5278 queue_hotplug = true;
5279 DRM_DEBUG("IH: HPD2\n");
5280 break;
5281 case 2:
5282 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5283 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5284
5285 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5286 queue_hotplug = true;
5287 DRM_DEBUG("IH: HPD3\n");
5288 break;
5289 case 3:
5290 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5291 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5292
5293 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5294 queue_hotplug = true;
5295 DRM_DEBUG("IH: HPD4\n");
5296 break;
5297 case 4:
5298 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5299 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5300
5301 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5302 queue_hotplug = true;
5303 DRM_DEBUG("IH: HPD5\n");
5304 break;
5305 case 5:
5306 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5307 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5308
5309 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5310 queue_hotplug = true;
5311 DRM_DEBUG("IH: HPD6\n");
5312 break;
5313 case 6:
5314 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5315 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5316
5317 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5318 queue_dp = true;
5319 DRM_DEBUG("IH: HPD_RX 1\n");
5320 break;
5321 case 7:
5322 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5323 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5324
5325 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5326 queue_dp = true;
5327 DRM_DEBUG("IH: HPD_RX 2\n");
5328 break;
5329 case 8:
5330 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5331 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5332
5333 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5334 queue_dp = true;
5335 DRM_DEBUG("IH: HPD_RX 3\n");
5336 break;
5337 case 9:
5338 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5339 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5340
5341 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5342 queue_dp = true;
5343 DRM_DEBUG("IH: HPD_RX 4\n");
5344 break;
5345 case 10:
5346 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5347 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5348
5349 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5350 queue_dp = true;
5351 DRM_DEBUG("IH: HPD_RX 5\n");
5352 break;
5353 case 11:
5354 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5355 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5356
5357 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5358 queue_dp = true;
5359 DRM_DEBUG("IH: HPD_RX 6\n");
5360 break;
5361 default:
5362 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5363 break;
5364 }
5365 break;
5366 case 44: /* hdmi */
5367 switch (src_data) {
5368 case 0:
5369 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5370 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5371
5372 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5373 queue_hdmi = true;
5374 DRM_DEBUG("IH: HDMI0\n");
5375 break;
5376 case 1:
5377 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5378 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5379
5380 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5381 queue_hdmi = true;
5382 DRM_DEBUG("IH: HDMI1\n");
5383 break;
5384 case 2:
5385 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5386 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5387
5388 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5389 queue_hdmi = true;
5390 DRM_DEBUG("IH: HDMI2\n");
5391 break;
5392 case 3:
5393 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5394 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5395
5396 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5397 queue_hdmi = true;
5398 DRM_DEBUG("IH: HDMI3\n");
5399 break;
5400 case 4:
5401 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5402 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5403
5404 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5405 queue_hdmi = true;
5406 DRM_DEBUG("IH: HDMI4\n");
5407 break;
5408 case 5:
5409 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5410 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5411
5412 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5413 queue_hdmi = true;
5414 DRM_DEBUG("IH: HDMI5\n");
5415 break;
5416 default:
5417 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5418 break;
5419 }
5420 case 96:
5421 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5422 WREG32(SRBM_INT_ACK, 0x1);
5423 break;
5424 case 124: /* UVD */
5425 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5426 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5427 break;
5428 case 146:
5429 case 147:
5430 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5431 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5432 /* reset addr and status */
5433 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5434 if (addr == 0x0 && status == 0x0)
5435 break;
5436 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5437 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5438 addr);
5439 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5440 status);
5441 cayman_vm_decode_fault(rdev, status, addr);
5442 break;
5443 case 176: /* CP_INT in ring buffer */
5444 case 177: /* CP_INT in IB1 */
5445 case 178: /* CP_INT in IB2 */
5446 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5447 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5448 break;
5449 case 181: /* CP EOP event */
5450 DRM_DEBUG("IH: CP EOP\n");
5451 if (rdev->family >= CHIP_CAYMAN) {
5452 switch (src_data) {
5453 case 0:
5454 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5455 break;
5456 case 1:
5457 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5458 break;
5459 case 2:
5460 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5461 break;
5462 }
5463 } else
5464 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5465 break;
5466 case 224: /* DMA trap event */
5467 DRM_DEBUG("IH: DMA trap\n");
5468 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5469 break;
5470 case 230: /* thermal low to high */
5471 DRM_DEBUG("IH: thermal low to high\n");
5472 rdev->pm.dpm.thermal.high_to_low = false;
5473 queue_thermal = true;
5474 break;
5475 case 231: /* thermal high to low */
5476 DRM_DEBUG("IH: thermal high to low\n");
5477 rdev->pm.dpm.thermal.high_to_low = true;
5478 queue_thermal = true;
5479 break;
5480 case 233: /* GUI IDLE */
5481 DRM_DEBUG("IH: GUI idle\n");
5482 break;
5483 case 244: /* DMA trap event */
5484 if (rdev->family >= CHIP_CAYMAN) {
5485 DRM_DEBUG("IH: DMA1 trap\n");
5486 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5487 }
5488 break;
5489 default:
5490 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5491 break;
5492 }
5493
5494 /* wptr/rptr are in bytes! */
5495 rptr += 16;
5496 rptr &= rdev->ih.ptr_mask;
5497 WREG32(IH_RB_RPTR, rptr);
5498 }
5499 if (queue_dp)
5500 schedule_work(&rdev->dp_work);
5501 if (queue_hotplug)
5502 schedule_delayed_work(&rdev->hotplug_work, 0);
5503 if (queue_hdmi)
5504 schedule_work(&rdev->audio_work);
5505 if (queue_thermal && rdev->pm.dpm_enabled)
5506 schedule_work(&rdev->pm.dpm.thermal.work);
5507 rdev->ih.rptr = rptr;
5508 atomic_set(&rdev->ih.lock, 0);
5509
5510 /* make sure wptr hasn't changed while processing */
5511 wptr = evergreen_get_ih_wptr(rdev);
5512 if (wptr != rptr)
5513 goto restart_ih;
5514
5515 return IRQ_HANDLED;
5516}
5517
5518static int evergreen_startup(struct radeon_device *rdev)
5519{
5520 struct radeon_ring *ring;
5521 int r;
5522
5523 /* enable pcie gen2 link */
5524 evergreen_pcie_gen2_enable(rdev);
5525 /* enable aspm */
5526 evergreen_program_aspm(rdev);
5527
5528 /* scratch needs to be initialized before MC */
5529 r = r600_vram_scratch_init(rdev);
5530 if (r)
5531 return r;
5532
5533 evergreen_mc_program(rdev);
5534
5535 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5536 r = ni_mc_load_microcode(rdev);
5537 if (r) {
5538 DRM_ERROR("Failed to load MC firmware!\n");
5539 return r;
5540 }
5541 }
5542
5543 if (rdev->flags & RADEON_IS_AGP) {
5544 evergreen_agp_enable(rdev);
5545 } else {
5546 r = evergreen_pcie_gart_enable(rdev);
5547 if (r)
5548 return r;
5549 }
5550 evergreen_gpu_init(rdev);
5551
5552 /* allocate rlc buffers */
5553 if (rdev->flags & RADEON_IS_IGP) {
5554 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5555 rdev->rlc.reg_list_size =
5556 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5557 rdev->rlc.cs_data = evergreen_cs_data;
5558 r = sumo_rlc_init(rdev);
5559 if (r) {
5560 DRM_ERROR("Failed to init rlc BOs!\n");
5561 return r;
5562 }
5563 }
5564
5565 /* allocate wb buffer */
5566 r = radeon_wb_init(rdev);
5567 if (r)
5568 return r;
5569
5570 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5571 if (r) {
5572 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5573 return r;
5574 }
5575
5576 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5577 if (r) {
5578 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5579 return r;
5580 }
5581
5582 r = uvd_v2_2_resume(rdev);
5583 if (!r) {
5584 r = radeon_fence_driver_start_ring(rdev,
5585 R600_RING_TYPE_UVD_INDEX);
5586 if (r)
5587 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5588 }
5589
5590 if (r)
5591 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5592
5593 /* Enable IRQ */
5594 if (!rdev->irq.installed) {
5595 r = radeon_irq_kms_init(rdev);
5596 if (r)
5597 return r;
5598 }
5599
5600 r = r600_irq_init(rdev);
5601 if (r) {
5602 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5603 radeon_irq_kms_fini(rdev);
5604 return r;
5605 }
5606 evergreen_irq_set(rdev);
5607
5608 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5609 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5610 RADEON_CP_PACKET2);
5611 if (r)
5612 return r;
5613
5614 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5615 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5616 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5617 if (r)
5618 return r;
5619
5620 r = evergreen_cp_load_microcode(rdev);
5621 if (r)
5622 return r;
5623 r = evergreen_cp_resume(rdev);
5624 if (r)
5625 return r;
5626 r = r600_dma_resume(rdev);
5627 if (r)
5628 return r;
5629
5630 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5631 if (ring->ring_size) {
5632 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5633 RADEON_CP_PACKET2);
5634 if (!r)
5635 r = uvd_v1_0_init(rdev);
5636
5637 if (r)
5638 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5639 }
5640
5641 r = radeon_ib_pool_init(rdev);
5642 if (r) {
5643 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5644 return r;
5645 }
5646
5647 r = radeon_audio_init(rdev);
5648 if (r) {
5649 DRM_ERROR("radeon: audio init failed\n");
5650 return r;
5651 }
5652
5653 return 0;
5654}
5655
5656int evergreen_resume(struct radeon_device *rdev)
5657{
5658 int r;
5659
5660 /* reset the asic, the gfx blocks are often in a bad state
5661 * after the driver is unloaded or after a resume
5662 */
5663 if (radeon_asic_reset(rdev))
5664 dev_warn(rdev->dev, "GPU reset failed !\n");
5665 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5666 * posting will perform necessary task to bring back GPU into good
5667 * shape.
5668 */
5669 /* post card */
5670 atom_asic_init(rdev->mode_info.atom_context);
5671
5672 /* init golden registers */
5673 evergreen_init_golden_registers(rdev);
5674
5675 if (rdev->pm.pm_method == PM_METHOD_DPM)
5676 radeon_pm_resume(rdev);
5677
5678 rdev->accel_working = true;
5679 r = evergreen_startup(rdev);
5680 if (r) {
5681 DRM_ERROR("evergreen startup failed on resume\n");
5682 rdev->accel_working = false;
5683 return r;
5684 }
5685
5686 return r;
5687
5688}
5689
5690int evergreen_suspend(struct radeon_device *rdev)
5691{
5692 radeon_pm_suspend(rdev);
5693 radeon_audio_fini(rdev);
5694 uvd_v1_0_fini(rdev);
5695 radeon_uvd_suspend(rdev);
5696 r700_cp_stop(rdev);
5697 r600_dma_stop(rdev);
5698 evergreen_irq_suspend(rdev);
5699 radeon_wb_disable(rdev);
5700 evergreen_pcie_gart_disable(rdev);
5701
5702 return 0;
5703}
5704
5705/* Plan is to move initialization in that function and use
5706 * helper function so that radeon_device_init pretty much
5707 * do nothing more than calling asic specific function. This
5708 * should also allow to remove a bunch of callback function
5709 * like vram_info.
5710 */
5711int evergreen_init(struct radeon_device *rdev)
5712{
5713 int r;
5714
5715 /* Read BIOS */
5716 if (!radeon_get_bios(rdev)) {
5717 if (ASIC_IS_AVIVO(rdev))
5718 return -EINVAL;
5719 }
5720 /* Must be an ATOMBIOS */
5721 if (!rdev->is_atom_bios) {
5722 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5723 return -EINVAL;
5724 }
5725 r = radeon_atombios_init(rdev);
5726 if (r)
5727 return r;
5728 /* reset the asic, the gfx blocks are often in a bad state
5729 * after the driver is unloaded or after a resume
5730 */
5731 if (radeon_asic_reset(rdev))
5732 dev_warn(rdev->dev, "GPU reset failed !\n");
5733 /* Post card if necessary */
5734 if (!radeon_card_posted(rdev)) {
5735 if (!rdev->bios) {
5736 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5737 return -EINVAL;
5738 }
5739 DRM_INFO("GPU not posted. posting now...\n");
5740 atom_asic_init(rdev->mode_info.atom_context);
5741 }
5742 /* init golden registers */
5743 evergreen_init_golden_registers(rdev);
5744 /* Initialize scratch registers */
5745 r600_scratch_init(rdev);
5746 /* Initialize surface registers */
5747 radeon_surface_init(rdev);
5748 /* Initialize clocks */
5749 radeon_get_clock_info(rdev->ddev);
5750 /* Fence driver */
5751 r = radeon_fence_driver_init(rdev);
5752 if (r)
5753 return r;
5754 /* initialize AGP */
5755 if (rdev->flags & RADEON_IS_AGP) {
5756 r = radeon_agp_init(rdev);
5757 if (r)
5758 radeon_agp_disable(rdev);
5759 }
5760 /* initialize memory controller */
5761 r = evergreen_mc_init(rdev);
5762 if (r)
5763 return r;
5764 /* Memory manager */
5765 r = radeon_bo_init(rdev);
5766 if (r)
5767 return r;
5768
5769 if (ASIC_IS_DCE5(rdev)) {
5770 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5771 r = ni_init_microcode(rdev);
5772 if (r) {
5773 DRM_ERROR("Failed to load firmware!\n");
5774 return r;
5775 }
5776 }
5777 } else {
5778 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5779 r = r600_init_microcode(rdev);
5780 if (r) {
5781 DRM_ERROR("Failed to load firmware!\n");
5782 return r;
5783 }
5784 }
5785 }
5786
5787 /* Initialize power management */
5788 radeon_pm_init(rdev);
5789
5790 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5791 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5792
5793 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5794 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5795
5796 r = radeon_uvd_init(rdev);
5797 if (!r) {
5798 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5799 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5800 4096);
5801 }
5802
5803 rdev->ih.ring_obj = NULL;
5804 r600_ih_ring_init(rdev, 64 * 1024);
5805
5806 r = r600_pcie_gart_init(rdev);
5807 if (r)
5808 return r;
5809
5810 rdev->accel_working = true;
5811 r = evergreen_startup(rdev);
5812 if (r) {
5813 dev_err(rdev->dev, "disabling GPU acceleration\n");
5814 r700_cp_fini(rdev);
5815 r600_dma_fini(rdev);
5816 r600_irq_fini(rdev);
5817 if (rdev->flags & RADEON_IS_IGP)
5818 sumo_rlc_fini(rdev);
5819 radeon_wb_fini(rdev);
5820 radeon_ib_pool_fini(rdev);
5821 radeon_irq_kms_fini(rdev);
5822 evergreen_pcie_gart_fini(rdev);
5823 rdev->accel_working = false;
5824 }
5825
5826 /* Don't start up if the MC ucode is missing on BTC parts.
5827 * The default clocks and voltages before the MC ucode
5828 * is loaded are not suffient for advanced operations.
5829 */
5830 if (ASIC_IS_DCE5(rdev)) {
5831 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5832 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5833 return -EINVAL;
5834 }
5835 }
5836
5837 return 0;
5838}
5839
5840void evergreen_fini(struct radeon_device *rdev)
5841{
5842 radeon_pm_fini(rdev);
5843 radeon_audio_fini(rdev);
5844 r700_cp_fini(rdev);
5845 r600_dma_fini(rdev);
5846 r600_irq_fini(rdev);
5847 if (rdev->flags & RADEON_IS_IGP)
5848 sumo_rlc_fini(rdev);
5849 radeon_wb_fini(rdev);
5850 radeon_ib_pool_fini(rdev);
5851 radeon_irq_kms_fini(rdev);
5852 uvd_v1_0_fini(rdev);
5853 radeon_uvd_fini(rdev);
5854 evergreen_pcie_gart_fini(rdev);
5855 r600_vram_scratch_fini(rdev);
5856 radeon_gem_fini(rdev);
5857 radeon_fence_driver_fini(rdev);
5858 radeon_agp_fini(rdev);
5859 radeon_bo_fini(rdev);
5860 radeon_atombios_fini(rdev);
5861 kfree(rdev->bios);
5862 rdev->bios = NULL;
5863}
5864
5865void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5866{
5867 u32 link_width_cntl, speed_cntl;
5868
5869 if (radeon_pcie_gen2 == 0)
5870 return;
5871
5872 if (rdev->flags & RADEON_IS_IGP)
5873 return;
5874
5875 if (!(rdev->flags & RADEON_IS_PCIE))
5876 return;
5877
5878 /* x2 cards have a special sequence */
5879 if (ASIC_IS_X2(rdev))
5880 return;
5881
5882 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5883 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5884 return;
5885
5886 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5887 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5888 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5889 return;
5890 }
5891
5892 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5893
5894 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5895 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5896
5897 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5898 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5899 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5900
5901 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5902 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5903 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5904
5905 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5906 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5907 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5908
5909 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5910 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5911 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5912
5913 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5914 speed_cntl |= LC_GEN2_EN_STRAP;
5915 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5916
5917 } else {
5918 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5919 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5920 if (1)
5921 link_width_cntl |= LC_UPCONFIGURE_DIS;
5922 else
5923 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5924 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5925 }
5926}
5927
5928void evergreen_program_aspm(struct radeon_device *rdev)
5929{
5930 u32 data, orig;
5931 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5932 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5933 /* fusion_platform = true
5934 * if the system is a fusion system
5935 * (APU or DGPU in a fusion system).
5936 * todo: check if the system is a fusion platform.
5937 */
5938 bool fusion_platform = false;
5939
5940 if (radeon_aspm == 0)
5941 return;
5942
5943 if (!(rdev->flags & RADEON_IS_PCIE))
5944 return;
5945
5946 switch (rdev->family) {
5947 case CHIP_CYPRESS:
5948 case CHIP_HEMLOCK:
5949 case CHIP_JUNIPER:
5950 case CHIP_REDWOOD:
5951 case CHIP_CEDAR:
5952 case CHIP_SUMO:
5953 case CHIP_SUMO2:
5954 case CHIP_PALM:
5955 case CHIP_ARUBA:
5956 disable_l0s = true;
5957 break;
5958 default:
5959 disable_l0s = false;
5960 break;
5961 }
5962
5963 if (rdev->flags & RADEON_IS_IGP)
5964 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5965
5966 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5967 if (fusion_platform)
5968 data &= ~MULTI_PIF;
5969 else
5970 data |= MULTI_PIF;
5971 if (data != orig)
5972 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5973
5974 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5975 if (fusion_platform)
5976 data &= ~MULTI_PIF;
5977 else
5978 data |= MULTI_PIF;
5979 if (data != orig)
5980 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5981
5982 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5983 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5984 if (!disable_l0s) {
5985 if (rdev->family >= CHIP_BARTS)
5986 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5987 else
5988 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5989 }
5990
5991 if (!disable_l1) {
5992 if (rdev->family >= CHIP_BARTS)
5993 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5994 else
5995 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5996
5997 if (!disable_plloff_in_l1) {
5998 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5999 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6000 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6001 if (data != orig)
6002 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6003
6004 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6005 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6006 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6007 if (data != orig)
6008 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6009
6010 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6011 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6012 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6013 if (data != orig)
6014 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6015
6016 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6017 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6018 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6019 if (data != orig)
6020 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6021
6022 if (rdev->family >= CHIP_BARTS) {
6023 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6024 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6025 data |= PLL_RAMP_UP_TIME_0(4);
6026 if (data != orig)
6027 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6028
6029 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6030 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6031 data |= PLL_RAMP_UP_TIME_1(4);
6032 if (data != orig)
6033 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6034
6035 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6036 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6037 data |= PLL_RAMP_UP_TIME_0(4);
6038 if (data != orig)
6039 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6040
6041 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6042 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6043 data |= PLL_RAMP_UP_TIME_1(4);
6044 if (data != orig)
6045 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6046 }
6047
6048 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6049 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6050 data |= LC_DYN_LANES_PWR_STATE(3);
6051 if (data != orig)
6052 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6053
6054 if (rdev->family >= CHIP_BARTS) {
6055 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6056 data &= ~LS2_EXIT_TIME_MASK;
6057 data |= LS2_EXIT_TIME(1);
6058 if (data != orig)
6059 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6060
6061 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6062 data &= ~LS2_EXIT_TIME_MASK;
6063 data |= LS2_EXIT_TIME(1);
6064 if (data != orig)
6065 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6066 }
6067 }
6068 }
6069
6070 /* evergreen parts only */
6071 if (rdev->family < CHIP_BARTS)
6072 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
6073
6074 if (pcie_lc_cntl != pcie_lc_cntl_old)
6075 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
6076}
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24
25#include <linux/firmware.h>
26#include <linux/pci.h>
27#include <linux/slab.h>
28
29#include <drm/drm_vblank.h>
30#include <drm/radeon_drm.h>
31
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_blit_shaders.h"
35#include "evergreen_reg.h"
36#include "evergreend.h"
37#include "radeon.h"
38#include "radeon_asic.h"
39#include "radeon_audio.h"
40#include "radeon_ucode.h"
41
42#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
43#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
44#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
45
46/*
47 * Indirect registers accessor
48 */
49u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
50{
51 unsigned long flags;
52 u32 r;
53
54 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
55 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
56 r = RREG32(EVERGREEN_CG_IND_DATA);
57 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
58 return r;
59}
60
61void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
62{
63 unsigned long flags;
64
65 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
66 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
67 WREG32(EVERGREEN_CG_IND_DATA, (v));
68 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
69}
70
71u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
72{
73 unsigned long flags;
74 u32 r;
75
76 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
77 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
78 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
79 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
80 return r;
81}
82
83void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
84{
85 unsigned long flags;
86
87 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
88 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
89 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
90 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
91}
92
93u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
94{
95 unsigned long flags;
96 u32 r;
97
98 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
99 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
100 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
101 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
102 return r;
103}
104
105void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
106{
107 unsigned long flags;
108
109 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
110 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
111 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
112 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
113}
114
115static const u32 crtc_offsets[6] =
116{
117 EVERGREEN_CRTC0_REGISTER_OFFSET,
118 EVERGREEN_CRTC1_REGISTER_OFFSET,
119 EVERGREEN_CRTC2_REGISTER_OFFSET,
120 EVERGREEN_CRTC3_REGISTER_OFFSET,
121 EVERGREEN_CRTC4_REGISTER_OFFSET,
122 EVERGREEN_CRTC5_REGISTER_OFFSET
123};
124
125#include "clearstate_evergreen.h"
126
127static const u32 sumo_rlc_save_restore_register_list[] =
128{
129 0x98fc,
130 0x9830,
131 0x9834,
132 0x9838,
133 0x9870,
134 0x9874,
135 0x8a14,
136 0x8b24,
137 0x8bcc,
138 0x8b10,
139 0x8d00,
140 0x8d04,
141 0x8c00,
142 0x8c04,
143 0x8c08,
144 0x8c0c,
145 0x8d8c,
146 0x8c20,
147 0x8c24,
148 0x8c28,
149 0x8c18,
150 0x8c1c,
151 0x8cf0,
152 0x8e2c,
153 0x8e38,
154 0x8c30,
155 0x9508,
156 0x9688,
157 0x9608,
158 0x960c,
159 0x9610,
160 0x9614,
161 0x88c4,
162 0x88d4,
163 0xa008,
164 0x900c,
165 0x9100,
166 0x913c,
167 0x98f8,
168 0x98f4,
169 0x9b7c,
170 0x3f8c,
171 0x8950,
172 0x8954,
173 0x8a18,
174 0x8b28,
175 0x9144,
176 0x9148,
177 0x914c,
178 0x3f90,
179 0x3f94,
180 0x915c,
181 0x9160,
182 0x9178,
183 0x917c,
184 0x9180,
185 0x918c,
186 0x9190,
187 0x9194,
188 0x9198,
189 0x919c,
190 0x91a8,
191 0x91ac,
192 0x91b0,
193 0x91b4,
194 0x91b8,
195 0x91c4,
196 0x91c8,
197 0x91cc,
198 0x91d0,
199 0x91d4,
200 0x91e0,
201 0x91e4,
202 0x91ec,
203 0x91f0,
204 0x91f4,
205 0x9200,
206 0x9204,
207 0x929c,
208 0x9150,
209 0x802c,
210};
211
212static void evergreen_gpu_init(struct radeon_device *rdev);
213void evergreen_fini(struct radeon_device *rdev);
214void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
215void evergreen_program_aspm(struct radeon_device *rdev);
216extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
217 int ring, u32 cp_int_cntl);
218extern void cayman_vm_decode_fault(struct radeon_device *rdev,
219 u32 status, u32 addr);
220void cik_init_cp_pg_table(struct radeon_device *rdev);
221
222extern u32 si_get_csb_size(struct radeon_device *rdev);
223extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
224extern u32 cik_get_csb_size(struct radeon_device *rdev);
225extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
226extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
227
228static const u32 evergreen_golden_registers[] =
229{
230 0x3f90, 0xffff0000, 0xff000000,
231 0x9148, 0xffff0000, 0xff000000,
232 0x3f94, 0xffff0000, 0xff000000,
233 0x914c, 0xffff0000, 0xff000000,
234 0x9b7c, 0xffffffff, 0x00000000,
235 0x8a14, 0xffffffff, 0x00000007,
236 0x8b10, 0xffffffff, 0x00000000,
237 0x960c, 0xffffffff, 0x54763210,
238 0x88c4, 0xffffffff, 0x000000c2,
239 0x88d4, 0xffffffff, 0x00000010,
240 0x8974, 0xffffffff, 0x00000000,
241 0xc78, 0x00000080, 0x00000080,
242 0x5eb4, 0xffffffff, 0x00000002,
243 0x5e78, 0xffffffff, 0x001000f0,
244 0x6104, 0x01000300, 0x00000000,
245 0x5bc0, 0x00300000, 0x00000000,
246 0x7030, 0xffffffff, 0x00000011,
247 0x7c30, 0xffffffff, 0x00000011,
248 0x10830, 0xffffffff, 0x00000011,
249 0x11430, 0xffffffff, 0x00000011,
250 0x12030, 0xffffffff, 0x00000011,
251 0x12c30, 0xffffffff, 0x00000011,
252 0xd02c, 0xffffffff, 0x08421000,
253 0x240c, 0xffffffff, 0x00000380,
254 0x8b24, 0xffffffff, 0x00ff0fff,
255 0x28a4c, 0x06000000, 0x06000000,
256 0x10c, 0x00000001, 0x00000001,
257 0x8d00, 0xffffffff, 0x100e4848,
258 0x8d04, 0xffffffff, 0x00164745,
259 0x8c00, 0xffffffff, 0xe4000003,
260 0x8c04, 0xffffffff, 0x40600060,
261 0x8c08, 0xffffffff, 0x001c001c,
262 0x8cf0, 0xffffffff, 0x08e00620,
263 0x8c20, 0xffffffff, 0x00800080,
264 0x8c24, 0xffffffff, 0x00800080,
265 0x8c18, 0xffffffff, 0x20202078,
266 0x8c1c, 0xffffffff, 0x00001010,
267 0x28350, 0xffffffff, 0x00000000,
268 0xa008, 0xffffffff, 0x00010000,
269 0x5c4, 0xffffffff, 0x00000001,
270 0x9508, 0xffffffff, 0x00000002,
271 0x913c, 0x0000000f, 0x0000000a
272};
273
274static const u32 evergreen_golden_registers2[] =
275{
276 0x2f4c, 0xffffffff, 0x00000000,
277 0x54f4, 0xffffffff, 0x00000000,
278 0x54f0, 0xffffffff, 0x00000000,
279 0x5498, 0xffffffff, 0x00000000,
280 0x549c, 0xffffffff, 0x00000000,
281 0x5494, 0xffffffff, 0x00000000,
282 0x53cc, 0xffffffff, 0x00000000,
283 0x53c8, 0xffffffff, 0x00000000,
284 0x53c4, 0xffffffff, 0x00000000,
285 0x53c0, 0xffffffff, 0x00000000,
286 0x53bc, 0xffffffff, 0x00000000,
287 0x53b8, 0xffffffff, 0x00000000,
288 0x53b4, 0xffffffff, 0x00000000,
289 0x53b0, 0xffffffff, 0x00000000
290};
291
292static const u32 cypress_mgcg_init[] =
293{
294 0x802c, 0xffffffff, 0xc0000000,
295 0x5448, 0xffffffff, 0x00000100,
296 0x55e4, 0xffffffff, 0x00000100,
297 0x160c, 0xffffffff, 0x00000100,
298 0x5644, 0xffffffff, 0x00000100,
299 0xc164, 0xffffffff, 0x00000100,
300 0x8a18, 0xffffffff, 0x00000100,
301 0x897c, 0xffffffff, 0x06000100,
302 0x8b28, 0xffffffff, 0x00000100,
303 0x9144, 0xffffffff, 0x00000100,
304 0x9a60, 0xffffffff, 0x00000100,
305 0x9868, 0xffffffff, 0x00000100,
306 0x8d58, 0xffffffff, 0x00000100,
307 0x9510, 0xffffffff, 0x00000100,
308 0x949c, 0xffffffff, 0x00000100,
309 0x9654, 0xffffffff, 0x00000100,
310 0x9030, 0xffffffff, 0x00000100,
311 0x9034, 0xffffffff, 0x00000100,
312 0x9038, 0xffffffff, 0x00000100,
313 0x903c, 0xffffffff, 0x00000100,
314 0x9040, 0xffffffff, 0x00000100,
315 0xa200, 0xffffffff, 0x00000100,
316 0xa204, 0xffffffff, 0x00000100,
317 0xa208, 0xffffffff, 0x00000100,
318 0xa20c, 0xffffffff, 0x00000100,
319 0x971c, 0xffffffff, 0x00000100,
320 0x977c, 0xffffffff, 0x00000100,
321 0x3f80, 0xffffffff, 0x00000100,
322 0xa210, 0xffffffff, 0x00000100,
323 0xa214, 0xffffffff, 0x00000100,
324 0x4d8, 0xffffffff, 0x00000100,
325 0x9784, 0xffffffff, 0x00000100,
326 0x9698, 0xffffffff, 0x00000100,
327 0x4d4, 0xffffffff, 0x00000200,
328 0x30cc, 0xffffffff, 0x00000100,
329 0xd0c0, 0xffffffff, 0xff000100,
330 0x802c, 0xffffffff, 0x40000000,
331 0x915c, 0xffffffff, 0x00010000,
332 0x9160, 0xffffffff, 0x00030002,
333 0x9178, 0xffffffff, 0x00070000,
334 0x917c, 0xffffffff, 0x00030002,
335 0x9180, 0xffffffff, 0x00050004,
336 0x918c, 0xffffffff, 0x00010006,
337 0x9190, 0xffffffff, 0x00090008,
338 0x9194, 0xffffffff, 0x00070000,
339 0x9198, 0xffffffff, 0x00030002,
340 0x919c, 0xffffffff, 0x00050004,
341 0x91a8, 0xffffffff, 0x00010006,
342 0x91ac, 0xffffffff, 0x00090008,
343 0x91b0, 0xffffffff, 0x00070000,
344 0x91b4, 0xffffffff, 0x00030002,
345 0x91b8, 0xffffffff, 0x00050004,
346 0x91c4, 0xffffffff, 0x00010006,
347 0x91c8, 0xffffffff, 0x00090008,
348 0x91cc, 0xffffffff, 0x00070000,
349 0x91d0, 0xffffffff, 0x00030002,
350 0x91d4, 0xffffffff, 0x00050004,
351 0x91e0, 0xffffffff, 0x00010006,
352 0x91e4, 0xffffffff, 0x00090008,
353 0x91e8, 0xffffffff, 0x00000000,
354 0x91ec, 0xffffffff, 0x00070000,
355 0x91f0, 0xffffffff, 0x00030002,
356 0x91f4, 0xffffffff, 0x00050004,
357 0x9200, 0xffffffff, 0x00010006,
358 0x9204, 0xffffffff, 0x00090008,
359 0x9208, 0xffffffff, 0x00070000,
360 0x920c, 0xffffffff, 0x00030002,
361 0x9210, 0xffffffff, 0x00050004,
362 0x921c, 0xffffffff, 0x00010006,
363 0x9220, 0xffffffff, 0x00090008,
364 0x9224, 0xffffffff, 0x00070000,
365 0x9228, 0xffffffff, 0x00030002,
366 0x922c, 0xffffffff, 0x00050004,
367 0x9238, 0xffffffff, 0x00010006,
368 0x923c, 0xffffffff, 0x00090008,
369 0x9240, 0xffffffff, 0x00070000,
370 0x9244, 0xffffffff, 0x00030002,
371 0x9248, 0xffffffff, 0x00050004,
372 0x9254, 0xffffffff, 0x00010006,
373 0x9258, 0xffffffff, 0x00090008,
374 0x925c, 0xffffffff, 0x00070000,
375 0x9260, 0xffffffff, 0x00030002,
376 0x9264, 0xffffffff, 0x00050004,
377 0x9270, 0xffffffff, 0x00010006,
378 0x9274, 0xffffffff, 0x00090008,
379 0x9278, 0xffffffff, 0x00070000,
380 0x927c, 0xffffffff, 0x00030002,
381 0x9280, 0xffffffff, 0x00050004,
382 0x928c, 0xffffffff, 0x00010006,
383 0x9290, 0xffffffff, 0x00090008,
384 0x9294, 0xffffffff, 0x00000000,
385 0x929c, 0xffffffff, 0x00000001,
386 0x802c, 0xffffffff, 0x40010000,
387 0x915c, 0xffffffff, 0x00010000,
388 0x9160, 0xffffffff, 0x00030002,
389 0x9178, 0xffffffff, 0x00070000,
390 0x917c, 0xffffffff, 0x00030002,
391 0x9180, 0xffffffff, 0x00050004,
392 0x918c, 0xffffffff, 0x00010006,
393 0x9190, 0xffffffff, 0x00090008,
394 0x9194, 0xffffffff, 0x00070000,
395 0x9198, 0xffffffff, 0x00030002,
396 0x919c, 0xffffffff, 0x00050004,
397 0x91a8, 0xffffffff, 0x00010006,
398 0x91ac, 0xffffffff, 0x00090008,
399 0x91b0, 0xffffffff, 0x00070000,
400 0x91b4, 0xffffffff, 0x00030002,
401 0x91b8, 0xffffffff, 0x00050004,
402 0x91c4, 0xffffffff, 0x00010006,
403 0x91c8, 0xffffffff, 0x00090008,
404 0x91cc, 0xffffffff, 0x00070000,
405 0x91d0, 0xffffffff, 0x00030002,
406 0x91d4, 0xffffffff, 0x00050004,
407 0x91e0, 0xffffffff, 0x00010006,
408 0x91e4, 0xffffffff, 0x00090008,
409 0x91e8, 0xffffffff, 0x00000000,
410 0x91ec, 0xffffffff, 0x00070000,
411 0x91f0, 0xffffffff, 0x00030002,
412 0x91f4, 0xffffffff, 0x00050004,
413 0x9200, 0xffffffff, 0x00010006,
414 0x9204, 0xffffffff, 0x00090008,
415 0x9208, 0xffffffff, 0x00070000,
416 0x920c, 0xffffffff, 0x00030002,
417 0x9210, 0xffffffff, 0x00050004,
418 0x921c, 0xffffffff, 0x00010006,
419 0x9220, 0xffffffff, 0x00090008,
420 0x9224, 0xffffffff, 0x00070000,
421 0x9228, 0xffffffff, 0x00030002,
422 0x922c, 0xffffffff, 0x00050004,
423 0x9238, 0xffffffff, 0x00010006,
424 0x923c, 0xffffffff, 0x00090008,
425 0x9240, 0xffffffff, 0x00070000,
426 0x9244, 0xffffffff, 0x00030002,
427 0x9248, 0xffffffff, 0x00050004,
428 0x9254, 0xffffffff, 0x00010006,
429 0x9258, 0xffffffff, 0x00090008,
430 0x925c, 0xffffffff, 0x00070000,
431 0x9260, 0xffffffff, 0x00030002,
432 0x9264, 0xffffffff, 0x00050004,
433 0x9270, 0xffffffff, 0x00010006,
434 0x9274, 0xffffffff, 0x00090008,
435 0x9278, 0xffffffff, 0x00070000,
436 0x927c, 0xffffffff, 0x00030002,
437 0x9280, 0xffffffff, 0x00050004,
438 0x928c, 0xffffffff, 0x00010006,
439 0x9290, 0xffffffff, 0x00090008,
440 0x9294, 0xffffffff, 0x00000000,
441 0x929c, 0xffffffff, 0x00000001,
442 0x802c, 0xffffffff, 0xc0000000
443};
444
445static const u32 redwood_mgcg_init[] =
446{
447 0x802c, 0xffffffff, 0xc0000000,
448 0x5448, 0xffffffff, 0x00000100,
449 0x55e4, 0xffffffff, 0x00000100,
450 0x160c, 0xffffffff, 0x00000100,
451 0x5644, 0xffffffff, 0x00000100,
452 0xc164, 0xffffffff, 0x00000100,
453 0x8a18, 0xffffffff, 0x00000100,
454 0x897c, 0xffffffff, 0x06000100,
455 0x8b28, 0xffffffff, 0x00000100,
456 0x9144, 0xffffffff, 0x00000100,
457 0x9a60, 0xffffffff, 0x00000100,
458 0x9868, 0xffffffff, 0x00000100,
459 0x8d58, 0xffffffff, 0x00000100,
460 0x9510, 0xffffffff, 0x00000100,
461 0x949c, 0xffffffff, 0x00000100,
462 0x9654, 0xffffffff, 0x00000100,
463 0x9030, 0xffffffff, 0x00000100,
464 0x9034, 0xffffffff, 0x00000100,
465 0x9038, 0xffffffff, 0x00000100,
466 0x903c, 0xffffffff, 0x00000100,
467 0x9040, 0xffffffff, 0x00000100,
468 0xa200, 0xffffffff, 0x00000100,
469 0xa204, 0xffffffff, 0x00000100,
470 0xa208, 0xffffffff, 0x00000100,
471 0xa20c, 0xffffffff, 0x00000100,
472 0x971c, 0xffffffff, 0x00000100,
473 0x977c, 0xffffffff, 0x00000100,
474 0x3f80, 0xffffffff, 0x00000100,
475 0xa210, 0xffffffff, 0x00000100,
476 0xa214, 0xffffffff, 0x00000100,
477 0x4d8, 0xffffffff, 0x00000100,
478 0x9784, 0xffffffff, 0x00000100,
479 0x9698, 0xffffffff, 0x00000100,
480 0x4d4, 0xffffffff, 0x00000200,
481 0x30cc, 0xffffffff, 0x00000100,
482 0xd0c0, 0xffffffff, 0xff000100,
483 0x802c, 0xffffffff, 0x40000000,
484 0x915c, 0xffffffff, 0x00010000,
485 0x9160, 0xffffffff, 0x00030002,
486 0x9178, 0xffffffff, 0x00070000,
487 0x917c, 0xffffffff, 0x00030002,
488 0x9180, 0xffffffff, 0x00050004,
489 0x918c, 0xffffffff, 0x00010006,
490 0x9190, 0xffffffff, 0x00090008,
491 0x9194, 0xffffffff, 0x00070000,
492 0x9198, 0xffffffff, 0x00030002,
493 0x919c, 0xffffffff, 0x00050004,
494 0x91a8, 0xffffffff, 0x00010006,
495 0x91ac, 0xffffffff, 0x00090008,
496 0x91b0, 0xffffffff, 0x00070000,
497 0x91b4, 0xffffffff, 0x00030002,
498 0x91b8, 0xffffffff, 0x00050004,
499 0x91c4, 0xffffffff, 0x00010006,
500 0x91c8, 0xffffffff, 0x00090008,
501 0x91cc, 0xffffffff, 0x00070000,
502 0x91d0, 0xffffffff, 0x00030002,
503 0x91d4, 0xffffffff, 0x00050004,
504 0x91e0, 0xffffffff, 0x00010006,
505 0x91e4, 0xffffffff, 0x00090008,
506 0x91e8, 0xffffffff, 0x00000000,
507 0x91ec, 0xffffffff, 0x00070000,
508 0x91f0, 0xffffffff, 0x00030002,
509 0x91f4, 0xffffffff, 0x00050004,
510 0x9200, 0xffffffff, 0x00010006,
511 0x9204, 0xffffffff, 0x00090008,
512 0x9294, 0xffffffff, 0x00000000,
513 0x929c, 0xffffffff, 0x00000001,
514 0x802c, 0xffffffff, 0xc0000000
515};
516
517static const u32 cedar_golden_registers[] =
518{
519 0x3f90, 0xffff0000, 0xff000000,
520 0x9148, 0xffff0000, 0xff000000,
521 0x3f94, 0xffff0000, 0xff000000,
522 0x914c, 0xffff0000, 0xff000000,
523 0x9b7c, 0xffffffff, 0x00000000,
524 0x8a14, 0xffffffff, 0x00000007,
525 0x8b10, 0xffffffff, 0x00000000,
526 0x960c, 0xffffffff, 0x54763210,
527 0x88c4, 0xffffffff, 0x000000c2,
528 0x88d4, 0xffffffff, 0x00000000,
529 0x8974, 0xffffffff, 0x00000000,
530 0xc78, 0x00000080, 0x00000080,
531 0x5eb4, 0xffffffff, 0x00000002,
532 0x5e78, 0xffffffff, 0x001000f0,
533 0x6104, 0x01000300, 0x00000000,
534 0x5bc0, 0x00300000, 0x00000000,
535 0x7030, 0xffffffff, 0x00000011,
536 0x7c30, 0xffffffff, 0x00000011,
537 0x10830, 0xffffffff, 0x00000011,
538 0x11430, 0xffffffff, 0x00000011,
539 0xd02c, 0xffffffff, 0x08421000,
540 0x240c, 0xffffffff, 0x00000380,
541 0x8b24, 0xffffffff, 0x00ff0fff,
542 0x28a4c, 0x06000000, 0x06000000,
543 0x10c, 0x00000001, 0x00000001,
544 0x8d00, 0xffffffff, 0x100e4848,
545 0x8d04, 0xffffffff, 0x00164745,
546 0x8c00, 0xffffffff, 0xe4000003,
547 0x8c04, 0xffffffff, 0x40600060,
548 0x8c08, 0xffffffff, 0x001c001c,
549 0x8cf0, 0xffffffff, 0x08e00410,
550 0x8c20, 0xffffffff, 0x00800080,
551 0x8c24, 0xffffffff, 0x00800080,
552 0x8c18, 0xffffffff, 0x20202078,
553 0x8c1c, 0xffffffff, 0x00001010,
554 0x28350, 0xffffffff, 0x00000000,
555 0xa008, 0xffffffff, 0x00010000,
556 0x5c4, 0xffffffff, 0x00000001,
557 0x9508, 0xffffffff, 0x00000002
558};
559
560static const u32 cedar_mgcg_init[] =
561{
562 0x802c, 0xffffffff, 0xc0000000,
563 0x5448, 0xffffffff, 0x00000100,
564 0x55e4, 0xffffffff, 0x00000100,
565 0x160c, 0xffffffff, 0x00000100,
566 0x5644, 0xffffffff, 0x00000100,
567 0xc164, 0xffffffff, 0x00000100,
568 0x8a18, 0xffffffff, 0x00000100,
569 0x897c, 0xffffffff, 0x06000100,
570 0x8b28, 0xffffffff, 0x00000100,
571 0x9144, 0xffffffff, 0x00000100,
572 0x9a60, 0xffffffff, 0x00000100,
573 0x9868, 0xffffffff, 0x00000100,
574 0x8d58, 0xffffffff, 0x00000100,
575 0x9510, 0xffffffff, 0x00000100,
576 0x949c, 0xffffffff, 0x00000100,
577 0x9654, 0xffffffff, 0x00000100,
578 0x9030, 0xffffffff, 0x00000100,
579 0x9034, 0xffffffff, 0x00000100,
580 0x9038, 0xffffffff, 0x00000100,
581 0x903c, 0xffffffff, 0x00000100,
582 0x9040, 0xffffffff, 0x00000100,
583 0xa200, 0xffffffff, 0x00000100,
584 0xa204, 0xffffffff, 0x00000100,
585 0xa208, 0xffffffff, 0x00000100,
586 0xa20c, 0xffffffff, 0x00000100,
587 0x971c, 0xffffffff, 0x00000100,
588 0x977c, 0xffffffff, 0x00000100,
589 0x3f80, 0xffffffff, 0x00000100,
590 0xa210, 0xffffffff, 0x00000100,
591 0xa214, 0xffffffff, 0x00000100,
592 0x4d8, 0xffffffff, 0x00000100,
593 0x9784, 0xffffffff, 0x00000100,
594 0x9698, 0xffffffff, 0x00000100,
595 0x4d4, 0xffffffff, 0x00000200,
596 0x30cc, 0xffffffff, 0x00000100,
597 0xd0c0, 0xffffffff, 0xff000100,
598 0x802c, 0xffffffff, 0x40000000,
599 0x915c, 0xffffffff, 0x00010000,
600 0x9178, 0xffffffff, 0x00050000,
601 0x917c, 0xffffffff, 0x00030002,
602 0x918c, 0xffffffff, 0x00010004,
603 0x9190, 0xffffffff, 0x00070006,
604 0x9194, 0xffffffff, 0x00050000,
605 0x9198, 0xffffffff, 0x00030002,
606 0x91a8, 0xffffffff, 0x00010004,
607 0x91ac, 0xffffffff, 0x00070006,
608 0x91e8, 0xffffffff, 0x00000000,
609 0x9294, 0xffffffff, 0x00000000,
610 0x929c, 0xffffffff, 0x00000001,
611 0x802c, 0xffffffff, 0xc0000000
612};
613
614static const u32 juniper_mgcg_init[] =
615{
616 0x802c, 0xffffffff, 0xc0000000,
617 0x5448, 0xffffffff, 0x00000100,
618 0x55e4, 0xffffffff, 0x00000100,
619 0x160c, 0xffffffff, 0x00000100,
620 0x5644, 0xffffffff, 0x00000100,
621 0xc164, 0xffffffff, 0x00000100,
622 0x8a18, 0xffffffff, 0x00000100,
623 0x897c, 0xffffffff, 0x06000100,
624 0x8b28, 0xffffffff, 0x00000100,
625 0x9144, 0xffffffff, 0x00000100,
626 0x9a60, 0xffffffff, 0x00000100,
627 0x9868, 0xffffffff, 0x00000100,
628 0x8d58, 0xffffffff, 0x00000100,
629 0x9510, 0xffffffff, 0x00000100,
630 0x949c, 0xffffffff, 0x00000100,
631 0x9654, 0xffffffff, 0x00000100,
632 0x9030, 0xffffffff, 0x00000100,
633 0x9034, 0xffffffff, 0x00000100,
634 0x9038, 0xffffffff, 0x00000100,
635 0x903c, 0xffffffff, 0x00000100,
636 0x9040, 0xffffffff, 0x00000100,
637 0xa200, 0xffffffff, 0x00000100,
638 0xa204, 0xffffffff, 0x00000100,
639 0xa208, 0xffffffff, 0x00000100,
640 0xa20c, 0xffffffff, 0x00000100,
641 0x971c, 0xffffffff, 0x00000100,
642 0xd0c0, 0xffffffff, 0xff000100,
643 0x802c, 0xffffffff, 0x40000000,
644 0x915c, 0xffffffff, 0x00010000,
645 0x9160, 0xffffffff, 0x00030002,
646 0x9178, 0xffffffff, 0x00070000,
647 0x917c, 0xffffffff, 0x00030002,
648 0x9180, 0xffffffff, 0x00050004,
649 0x918c, 0xffffffff, 0x00010006,
650 0x9190, 0xffffffff, 0x00090008,
651 0x9194, 0xffffffff, 0x00070000,
652 0x9198, 0xffffffff, 0x00030002,
653 0x919c, 0xffffffff, 0x00050004,
654 0x91a8, 0xffffffff, 0x00010006,
655 0x91ac, 0xffffffff, 0x00090008,
656 0x91b0, 0xffffffff, 0x00070000,
657 0x91b4, 0xffffffff, 0x00030002,
658 0x91b8, 0xffffffff, 0x00050004,
659 0x91c4, 0xffffffff, 0x00010006,
660 0x91c8, 0xffffffff, 0x00090008,
661 0x91cc, 0xffffffff, 0x00070000,
662 0x91d0, 0xffffffff, 0x00030002,
663 0x91d4, 0xffffffff, 0x00050004,
664 0x91e0, 0xffffffff, 0x00010006,
665 0x91e4, 0xffffffff, 0x00090008,
666 0x91e8, 0xffffffff, 0x00000000,
667 0x91ec, 0xffffffff, 0x00070000,
668 0x91f0, 0xffffffff, 0x00030002,
669 0x91f4, 0xffffffff, 0x00050004,
670 0x9200, 0xffffffff, 0x00010006,
671 0x9204, 0xffffffff, 0x00090008,
672 0x9208, 0xffffffff, 0x00070000,
673 0x920c, 0xffffffff, 0x00030002,
674 0x9210, 0xffffffff, 0x00050004,
675 0x921c, 0xffffffff, 0x00010006,
676 0x9220, 0xffffffff, 0x00090008,
677 0x9224, 0xffffffff, 0x00070000,
678 0x9228, 0xffffffff, 0x00030002,
679 0x922c, 0xffffffff, 0x00050004,
680 0x9238, 0xffffffff, 0x00010006,
681 0x923c, 0xffffffff, 0x00090008,
682 0x9240, 0xffffffff, 0x00070000,
683 0x9244, 0xffffffff, 0x00030002,
684 0x9248, 0xffffffff, 0x00050004,
685 0x9254, 0xffffffff, 0x00010006,
686 0x9258, 0xffffffff, 0x00090008,
687 0x925c, 0xffffffff, 0x00070000,
688 0x9260, 0xffffffff, 0x00030002,
689 0x9264, 0xffffffff, 0x00050004,
690 0x9270, 0xffffffff, 0x00010006,
691 0x9274, 0xffffffff, 0x00090008,
692 0x9278, 0xffffffff, 0x00070000,
693 0x927c, 0xffffffff, 0x00030002,
694 0x9280, 0xffffffff, 0x00050004,
695 0x928c, 0xffffffff, 0x00010006,
696 0x9290, 0xffffffff, 0x00090008,
697 0x9294, 0xffffffff, 0x00000000,
698 0x929c, 0xffffffff, 0x00000001,
699 0x802c, 0xffffffff, 0xc0000000,
700 0x977c, 0xffffffff, 0x00000100,
701 0x3f80, 0xffffffff, 0x00000100,
702 0xa210, 0xffffffff, 0x00000100,
703 0xa214, 0xffffffff, 0x00000100,
704 0x4d8, 0xffffffff, 0x00000100,
705 0x9784, 0xffffffff, 0x00000100,
706 0x9698, 0xffffffff, 0x00000100,
707 0x4d4, 0xffffffff, 0x00000200,
708 0x30cc, 0xffffffff, 0x00000100,
709 0x802c, 0xffffffff, 0xc0000000
710};
711
712static const u32 supersumo_golden_registers[] =
713{
714 0x5eb4, 0xffffffff, 0x00000002,
715 0x5c4, 0xffffffff, 0x00000001,
716 0x7030, 0xffffffff, 0x00000011,
717 0x7c30, 0xffffffff, 0x00000011,
718 0x6104, 0x01000300, 0x00000000,
719 0x5bc0, 0x00300000, 0x00000000,
720 0x8c04, 0xffffffff, 0x40600060,
721 0x8c08, 0xffffffff, 0x001c001c,
722 0x8c20, 0xffffffff, 0x00800080,
723 0x8c24, 0xffffffff, 0x00800080,
724 0x8c18, 0xffffffff, 0x20202078,
725 0x8c1c, 0xffffffff, 0x00001010,
726 0x918c, 0xffffffff, 0x00010006,
727 0x91a8, 0xffffffff, 0x00010006,
728 0x91c4, 0xffffffff, 0x00010006,
729 0x91e0, 0xffffffff, 0x00010006,
730 0x9200, 0xffffffff, 0x00010006,
731 0x9150, 0xffffffff, 0x6e944040,
732 0x917c, 0xffffffff, 0x00030002,
733 0x9180, 0xffffffff, 0x00050004,
734 0x9198, 0xffffffff, 0x00030002,
735 0x919c, 0xffffffff, 0x00050004,
736 0x91b4, 0xffffffff, 0x00030002,
737 0x91b8, 0xffffffff, 0x00050004,
738 0x91d0, 0xffffffff, 0x00030002,
739 0x91d4, 0xffffffff, 0x00050004,
740 0x91f0, 0xffffffff, 0x00030002,
741 0x91f4, 0xffffffff, 0x00050004,
742 0x915c, 0xffffffff, 0x00010000,
743 0x9160, 0xffffffff, 0x00030002,
744 0x3f90, 0xffff0000, 0xff000000,
745 0x9178, 0xffffffff, 0x00070000,
746 0x9194, 0xffffffff, 0x00070000,
747 0x91b0, 0xffffffff, 0x00070000,
748 0x91cc, 0xffffffff, 0x00070000,
749 0x91ec, 0xffffffff, 0x00070000,
750 0x9148, 0xffff0000, 0xff000000,
751 0x9190, 0xffffffff, 0x00090008,
752 0x91ac, 0xffffffff, 0x00090008,
753 0x91c8, 0xffffffff, 0x00090008,
754 0x91e4, 0xffffffff, 0x00090008,
755 0x9204, 0xffffffff, 0x00090008,
756 0x3f94, 0xffff0000, 0xff000000,
757 0x914c, 0xffff0000, 0xff000000,
758 0x929c, 0xffffffff, 0x00000001,
759 0x8a18, 0xffffffff, 0x00000100,
760 0x8b28, 0xffffffff, 0x00000100,
761 0x9144, 0xffffffff, 0x00000100,
762 0x5644, 0xffffffff, 0x00000100,
763 0x9b7c, 0xffffffff, 0x00000000,
764 0x8030, 0xffffffff, 0x0000100a,
765 0x8a14, 0xffffffff, 0x00000007,
766 0x8b24, 0xffffffff, 0x00ff0fff,
767 0x8b10, 0xffffffff, 0x00000000,
768 0x28a4c, 0x06000000, 0x06000000,
769 0x4d8, 0xffffffff, 0x00000100,
770 0x913c, 0xffff000f, 0x0100000a,
771 0x960c, 0xffffffff, 0x54763210,
772 0x88c4, 0xffffffff, 0x000000c2,
773 0x88d4, 0xffffffff, 0x00000010,
774 0x8974, 0xffffffff, 0x00000000,
775 0xc78, 0x00000080, 0x00000080,
776 0x5e78, 0xffffffff, 0x001000f0,
777 0xd02c, 0xffffffff, 0x08421000,
778 0xa008, 0xffffffff, 0x00010000,
779 0x8d00, 0xffffffff, 0x100e4848,
780 0x8d04, 0xffffffff, 0x00164745,
781 0x8c00, 0xffffffff, 0xe4000003,
782 0x8cf0, 0x1fffffff, 0x08e00620,
783 0x28350, 0xffffffff, 0x00000000,
784 0x9508, 0xffffffff, 0x00000002
785};
786
787static const u32 sumo_golden_registers[] =
788{
789 0x900c, 0x00ffffff, 0x0017071f,
790 0x8c18, 0xffffffff, 0x10101060,
791 0x8c1c, 0xffffffff, 0x00001010,
792 0x8c30, 0x0000000f, 0x00000005,
793 0x9688, 0x0000000f, 0x00000007
794};
795
796static const u32 wrestler_golden_registers[] =
797{
798 0x5eb4, 0xffffffff, 0x00000002,
799 0x5c4, 0xffffffff, 0x00000001,
800 0x7030, 0xffffffff, 0x00000011,
801 0x7c30, 0xffffffff, 0x00000011,
802 0x6104, 0x01000300, 0x00000000,
803 0x5bc0, 0x00300000, 0x00000000,
804 0x918c, 0xffffffff, 0x00010006,
805 0x91a8, 0xffffffff, 0x00010006,
806 0x9150, 0xffffffff, 0x6e944040,
807 0x917c, 0xffffffff, 0x00030002,
808 0x9198, 0xffffffff, 0x00030002,
809 0x915c, 0xffffffff, 0x00010000,
810 0x3f90, 0xffff0000, 0xff000000,
811 0x9178, 0xffffffff, 0x00070000,
812 0x9194, 0xffffffff, 0x00070000,
813 0x9148, 0xffff0000, 0xff000000,
814 0x9190, 0xffffffff, 0x00090008,
815 0x91ac, 0xffffffff, 0x00090008,
816 0x3f94, 0xffff0000, 0xff000000,
817 0x914c, 0xffff0000, 0xff000000,
818 0x929c, 0xffffffff, 0x00000001,
819 0x8a18, 0xffffffff, 0x00000100,
820 0x8b28, 0xffffffff, 0x00000100,
821 0x9144, 0xffffffff, 0x00000100,
822 0x9b7c, 0xffffffff, 0x00000000,
823 0x8030, 0xffffffff, 0x0000100a,
824 0x8a14, 0xffffffff, 0x00000001,
825 0x8b24, 0xffffffff, 0x00ff0fff,
826 0x8b10, 0xffffffff, 0x00000000,
827 0x28a4c, 0x06000000, 0x06000000,
828 0x4d8, 0xffffffff, 0x00000100,
829 0x913c, 0xffff000f, 0x0100000a,
830 0x960c, 0xffffffff, 0x54763210,
831 0x88c4, 0xffffffff, 0x000000c2,
832 0x88d4, 0xffffffff, 0x00000010,
833 0x8974, 0xffffffff, 0x00000000,
834 0xc78, 0x00000080, 0x00000080,
835 0x5e78, 0xffffffff, 0x001000f0,
836 0xd02c, 0xffffffff, 0x08421000,
837 0xa008, 0xffffffff, 0x00010000,
838 0x8d00, 0xffffffff, 0x100e4848,
839 0x8d04, 0xffffffff, 0x00164745,
840 0x8c00, 0xffffffff, 0xe4000003,
841 0x8cf0, 0x1fffffff, 0x08e00410,
842 0x28350, 0xffffffff, 0x00000000,
843 0x9508, 0xffffffff, 0x00000002,
844 0x900c, 0xffffffff, 0x0017071f,
845 0x8c18, 0xffffffff, 0x10101060,
846 0x8c1c, 0xffffffff, 0x00001010
847};
848
849static const u32 barts_golden_registers[] =
850{
851 0x5eb4, 0xffffffff, 0x00000002,
852 0x5e78, 0x8f311ff1, 0x001000f0,
853 0x3f90, 0xffff0000, 0xff000000,
854 0x9148, 0xffff0000, 0xff000000,
855 0x3f94, 0xffff0000, 0xff000000,
856 0x914c, 0xffff0000, 0xff000000,
857 0xc78, 0x00000080, 0x00000080,
858 0xbd4, 0x70073777, 0x00010001,
859 0xd02c, 0xbfffff1f, 0x08421000,
860 0xd0b8, 0x03773777, 0x02011003,
861 0x5bc0, 0x00200000, 0x50100000,
862 0x98f8, 0x33773777, 0x02011003,
863 0x98fc, 0xffffffff, 0x76543210,
864 0x7030, 0x31000311, 0x00000011,
865 0x2f48, 0x00000007, 0x02011003,
866 0x6b28, 0x00000010, 0x00000012,
867 0x7728, 0x00000010, 0x00000012,
868 0x10328, 0x00000010, 0x00000012,
869 0x10f28, 0x00000010, 0x00000012,
870 0x11b28, 0x00000010, 0x00000012,
871 0x12728, 0x00000010, 0x00000012,
872 0x240c, 0x000007ff, 0x00000380,
873 0x8a14, 0xf000001f, 0x00000007,
874 0x8b24, 0x3fff3fff, 0x00ff0fff,
875 0x8b10, 0x0000ff0f, 0x00000000,
876 0x28a4c, 0x07ffffff, 0x06000000,
877 0x10c, 0x00000001, 0x00010003,
878 0xa02c, 0xffffffff, 0x0000009b,
879 0x913c, 0x0000000f, 0x0100000a,
880 0x8d00, 0xffff7f7f, 0x100e4848,
881 0x8d04, 0x00ffffff, 0x00164745,
882 0x8c00, 0xfffc0003, 0xe4000003,
883 0x8c04, 0xf8ff00ff, 0x40600060,
884 0x8c08, 0x00ff00ff, 0x001c001c,
885 0x8cf0, 0x1fff1fff, 0x08e00620,
886 0x8c20, 0x0fff0fff, 0x00800080,
887 0x8c24, 0x0fff0fff, 0x00800080,
888 0x8c18, 0xffffffff, 0x20202078,
889 0x8c1c, 0x0000ffff, 0x00001010,
890 0x28350, 0x00000f01, 0x00000000,
891 0x9508, 0x3700001f, 0x00000002,
892 0x960c, 0xffffffff, 0x54763210,
893 0x88c4, 0x001f3ae3, 0x000000c2,
894 0x88d4, 0x0000001f, 0x00000010,
895 0x8974, 0xffffffff, 0x00000000
896};
897
898static const u32 turks_golden_registers[] =
899{
900 0x5eb4, 0xffffffff, 0x00000002,
901 0x5e78, 0x8f311ff1, 0x001000f0,
902 0x8c8, 0x00003000, 0x00001070,
903 0x8cc, 0x000fffff, 0x00040035,
904 0x3f90, 0xffff0000, 0xfff00000,
905 0x9148, 0xffff0000, 0xfff00000,
906 0x3f94, 0xffff0000, 0xfff00000,
907 0x914c, 0xffff0000, 0xfff00000,
908 0xc78, 0x00000080, 0x00000080,
909 0xbd4, 0x00073007, 0x00010002,
910 0xd02c, 0xbfffff1f, 0x08421000,
911 0xd0b8, 0x03773777, 0x02010002,
912 0x5bc0, 0x00200000, 0x50100000,
913 0x98f8, 0x33773777, 0x00010002,
914 0x98fc, 0xffffffff, 0x33221100,
915 0x7030, 0x31000311, 0x00000011,
916 0x2f48, 0x33773777, 0x00010002,
917 0x6b28, 0x00000010, 0x00000012,
918 0x7728, 0x00000010, 0x00000012,
919 0x10328, 0x00000010, 0x00000012,
920 0x10f28, 0x00000010, 0x00000012,
921 0x11b28, 0x00000010, 0x00000012,
922 0x12728, 0x00000010, 0x00000012,
923 0x240c, 0x000007ff, 0x00000380,
924 0x8a14, 0xf000001f, 0x00000007,
925 0x8b24, 0x3fff3fff, 0x00ff0fff,
926 0x8b10, 0x0000ff0f, 0x00000000,
927 0x28a4c, 0x07ffffff, 0x06000000,
928 0x10c, 0x00000001, 0x00010003,
929 0xa02c, 0xffffffff, 0x0000009b,
930 0x913c, 0x0000000f, 0x0100000a,
931 0x8d00, 0xffff7f7f, 0x100e4848,
932 0x8d04, 0x00ffffff, 0x00164745,
933 0x8c00, 0xfffc0003, 0xe4000003,
934 0x8c04, 0xf8ff00ff, 0x40600060,
935 0x8c08, 0x00ff00ff, 0x001c001c,
936 0x8cf0, 0x1fff1fff, 0x08e00410,
937 0x8c20, 0x0fff0fff, 0x00800080,
938 0x8c24, 0x0fff0fff, 0x00800080,
939 0x8c18, 0xffffffff, 0x20202078,
940 0x8c1c, 0x0000ffff, 0x00001010,
941 0x28350, 0x00000f01, 0x00000000,
942 0x9508, 0x3700001f, 0x00000002,
943 0x960c, 0xffffffff, 0x54763210,
944 0x88c4, 0x001f3ae3, 0x000000c2,
945 0x88d4, 0x0000001f, 0x00000010,
946 0x8974, 0xffffffff, 0x00000000
947};
948
949static const u32 caicos_golden_registers[] =
950{
951 0x5eb4, 0xffffffff, 0x00000002,
952 0x5e78, 0x8f311ff1, 0x001000f0,
953 0x8c8, 0x00003420, 0x00001450,
954 0x8cc, 0x000fffff, 0x00040035,
955 0x3f90, 0xffff0000, 0xfffc0000,
956 0x9148, 0xffff0000, 0xfffc0000,
957 0x3f94, 0xffff0000, 0xfffc0000,
958 0x914c, 0xffff0000, 0xfffc0000,
959 0xc78, 0x00000080, 0x00000080,
960 0xbd4, 0x00073007, 0x00010001,
961 0xd02c, 0xbfffff1f, 0x08421000,
962 0xd0b8, 0x03773777, 0x02010001,
963 0x5bc0, 0x00200000, 0x50100000,
964 0x98f8, 0x33773777, 0x02010001,
965 0x98fc, 0xffffffff, 0x33221100,
966 0x7030, 0x31000311, 0x00000011,
967 0x2f48, 0x33773777, 0x02010001,
968 0x6b28, 0x00000010, 0x00000012,
969 0x7728, 0x00000010, 0x00000012,
970 0x10328, 0x00000010, 0x00000012,
971 0x10f28, 0x00000010, 0x00000012,
972 0x11b28, 0x00000010, 0x00000012,
973 0x12728, 0x00000010, 0x00000012,
974 0x240c, 0x000007ff, 0x00000380,
975 0x8a14, 0xf000001f, 0x00000001,
976 0x8b24, 0x3fff3fff, 0x00ff0fff,
977 0x8b10, 0x0000ff0f, 0x00000000,
978 0x28a4c, 0x07ffffff, 0x06000000,
979 0x10c, 0x00000001, 0x00010003,
980 0xa02c, 0xffffffff, 0x0000009b,
981 0x913c, 0x0000000f, 0x0100000a,
982 0x8d00, 0xffff7f7f, 0x100e4848,
983 0x8d04, 0x00ffffff, 0x00164745,
984 0x8c00, 0xfffc0003, 0xe4000003,
985 0x8c04, 0xf8ff00ff, 0x40600060,
986 0x8c08, 0x00ff00ff, 0x001c001c,
987 0x8cf0, 0x1fff1fff, 0x08e00410,
988 0x8c20, 0x0fff0fff, 0x00800080,
989 0x8c24, 0x0fff0fff, 0x00800080,
990 0x8c18, 0xffffffff, 0x20202078,
991 0x8c1c, 0x0000ffff, 0x00001010,
992 0x28350, 0x00000f01, 0x00000000,
993 0x9508, 0x3700001f, 0x00000002,
994 0x960c, 0xffffffff, 0x54763210,
995 0x88c4, 0x001f3ae3, 0x000000c2,
996 0x88d4, 0x0000001f, 0x00000010,
997 0x8974, 0xffffffff, 0x00000000
998};
999
1000static void evergreen_init_golden_registers(struct radeon_device *rdev)
1001{
1002 switch (rdev->family) {
1003 case CHIP_CYPRESS:
1004 case CHIP_HEMLOCK:
1005 radeon_program_register_sequence(rdev,
1006 evergreen_golden_registers,
1007 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1008 radeon_program_register_sequence(rdev,
1009 evergreen_golden_registers2,
1010 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1011 radeon_program_register_sequence(rdev,
1012 cypress_mgcg_init,
1013 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1014 break;
1015 case CHIP_JUNIPER:
1016 radeon_program_register_sequence(rdev,
1017 evergreen_golden_registers,
1018 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1019 radeon_program_register_sequence(rdev,
1020 evergreen_golden_registers2,
1021 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1022 radeon_program_register_sequence(rdev,
1023 juniper_mgcg_init,
1024 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1025 break;
1026 case CHIP_REDWOOD:
1027 radeon_program_register_sequence(rdev,
1028 evergreen_golden_registers,
1029 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1030 radeon_program_register_sequence(rdev,
1031 evergreen_golden_registers2,
1032 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1033 radeon_program_register_sequence(rdev,
1034 redwood_mgcg_init,
1035 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1036 break;
1037 case CHIP_CEDAR:
1038 radeon_program_register_sequence(rdev,
1039 cedar_golden_registers,
1040 (const u32)ARRAY_SIZE(cedar_golden_registers));
1041 radeon_program_register_sequence(rdev,
1042 evergreen_golden_registers2,
1043 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1044 radeon_program_register_sequence(rdev,
1045 cedar_mgcg_init,
1046 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1047 break;
1048 case CHIP_PALM:
1049 radeon_program_register_sequence(rdev,
1050 wrestler_golden_registers,
1051 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1052 break;
1053 case CHIP_SUMO:
1054 radeon_program_register_sequence(rdev,
1055 supersumo_golden_registers,
1056 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1057 break;
1058 case CHIP_SUMO2:
1059 radeon_program_register_sequence(rdev,
1060 supersumo_golden_registers,
1061 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1062 radeon_program_register_sequence(rdev,
1063 sumo_golden_registers,
1064 (const u32)ARRAY_SIZE(sumo_golden_registers));
1065 break;
1066 case CHIP_BARTS:
1067 radeon_program_register_sequence(rdev,
1068 barts_golden_registers,
1069 (const u32)ARRAY_SIZE(barts_golden_registers));
1070 break;
1071 case CHIP_TURKS:
1072 radeon_program_register_sequence(rdev,
1073 turks_golden_registers,
1074 (const u32)ARRAY_SIZE(turks_golden_registers));
1075 break;
1076 case CHIP_CAICOS:
1077 radeon_program_register_sequence(rdev,
1078 caicos_golden_registers,
1079 (const u32)ARRAY_SIZE(caicos_golden_registers));
1080 break;
1081 default:
1082 break;
1083 }
1084}
1085
1086/**
1087 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1088 *
1089 * @rdev: radeon_device pointer
1090 * @reg: register offset in bytes
1091 * @val: register value
1092 *
1093 * Returns 0 for success or -EINVAL for an invalid register
1094 *
1095 */
1096int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1097 u32 reg, u32 *val)
1098{
1099 switch (reg) {
1100 case GRBM_STATUS:
1101 case GRBM_STATUS_SE0:
1102 case GRBM_STATUS_SE1:
1103 case SRBM_STATUS:
1104 case SRBM_STATUS2:
1105 case DMA_STATUS_REG:
1106 case UVD_STATUS:
1107 *val = RREG32(reg);
1108 return 0;
1109 default:
1110 return -EINVAL;
1111 }
1112}
1113
1114void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1115 unsigned *bankh, unsigned *mtaspect,
1116 unsigned *tile_split)
1117{
1118 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1119 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1120 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1121 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1122 switch (*bankw) {
1123 default:
1124 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1125 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1126 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1127 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1128 }
1129 switch (*bankh) {
1130 default:
1131 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1132 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1133 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1134 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1135 }
1136 switch (*mtaspect) {
1137 default:
1138 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1139 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1140 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1141 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1142 }
1143}
1144
1145static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1146 u32 cntl_reg, u32 status_reg)
1147{
1148 int r, i;
1149 struct atom_clock_dividers dividers;
1150
1151 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1152 clock, false, ÷rs);
1153 if (r)
1154 return r;
1155
1156 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1157
1158 for (i = 0; i < 100; i++) {
1159 if (RREG32(status_reg) & DCLK_STATUS)
1160 break;
1161 mdelay(10);
1162 }
1163 if (i == 100)
1164 return -ETIMEDOUT;
1165
1166 return 0;
1167}
1168
1169int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1170{
1171 int r = 0;
1172 u32 cg_scratch = RREG32(CG_SCRATCH1);
1173
1174 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1175 if (r)
1176 goto done;
1177 cg_scratch &= 0xffff0000;
1178 cg_scratch |= vclk / 100; /* Mhz */
1179
1180 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1181 if (r)
1182 goto done;
1183 cg_scratch &= 0x0000ffff;
1184 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1185
1186done:
1187 WREG32(CG_SCRATCH1, cg_scratch);
1188
1189 return r;
1190}
1191
1192int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1193{
1194 /* start off with something large */
1195 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1196 int r;
1197
1198 /* bypass vclk and dclk with bclk */
1199 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1200 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1201 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1202
1203 /* put PLL in bypass mode */
1204 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1205
1206 if (!vclk || !dclk) {
1207 /* keep the Bypass mode, put PLL to sleep */
1208 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1209 return 0;
1210 }
1211
1212 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1213 16384, 0x03FFFFFF, 0, 128, 5,
1214 &fb_div, &vclk_div, &dclk_div);
1215 if (r)
1216 return r;
1217
1218 /* set VCO_MODE to 1 */
1219 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1220
1221 /* toggle UPLL_SLEEP to 1 then back to 0 */
1222 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1223 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1224
1225 /* deassert UPLL_RESET */
1226 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1227
1228 mdelay(1);
1229
1230 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1231 if (r)
1232 return r;
1233
1234 /* assert UPLL_RESET again */
1235 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1236
1237 /* disable spread spectrum. */
1238 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1239
1240 /* set feedback divider */
1241 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1242
1243 /* set ref divider to 0 */
1244 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1245
1246 if (fb_div < 307200)
1247 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1248 else
1249 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1250
1251 /* set PDIV_A and PDIV_B */
1252 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1253 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1254 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1255
1256 /* give the PLL some time to settle */
1257 mdelay(15);
1258
1259 /* deassert PLL_RESET */
1260 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1261
1262 mdelay(15);
1263
1264 /* switch from bypass mode to normal mode */
1265 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1266
1267 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1268 if (r)
1269 return r;
1270
1271 /* switch VCLK and DCLK selection */
1272 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1273 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1274 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1275
1276 mdelay(100);
1277
1278 return 0;
1279}
1280
1281void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1282{
1283 int readrq;
1284 u16 v;
1285
1286 readrq = pcie_get_readrq(rdev->pdev);
1287 v = ffs(readrq) - 8;
1288 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1289 * to avoid hangs or perfomance issues
1290 */
1291 if ((v == 0) || (v == 6) || (v == 7))
1292 pcie_set_readrq(rdev->pdev, 512);
1293}
1294
1295void dce4_program_fmt(struct drm_encoder *encoder)
1296{
1297 struct drm_device *dev = encoder->dev;
1298 struct radeon_device *rdev = dev->dev_private;
1299 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1300 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1301 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1302 int bpc = 0;
1303 u32 tmp = 0;
1304 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1305
1306 if (connector) {
1307 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1308 bpc = radeon_get_monitor_bpc(connector);
1309 dither = radeon_connector->dither;
1310 }
1311
1312 /* LVDS/eDP FMT is set up by atom */
1313 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1314 return;
1315
1316 /* not needed for analog */
1317 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1318 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1319 return;
1320
1321 if (bpc == 0)
1322 return;
1323
1324 switch (bpc) {
1325 case 6:
1326 if (dither == RADEON_FMT_DITHER_ENABLE)
1327 /* XXX sort out optimal dither settings */
1328 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1329 FMT_SPATIAL_DITHER_EN);
1330 else
1331 tmp |= FMT_TRUNCATE_EN;
1332 break;
1333 case 8:
1334 if (dither == RADEON_FMT_DITHER_ENABLE)
1335 /* XXX sort out optimal dither settings */
1336 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1337 FMT_RGB_RANDOM_ENABLE |
1338 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1339 else
1340 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1341 break;
1342 case 10:
1343 default:
1344 /* not needed */
1345 break;
1346 }
1347
1348 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1349}
1350
1351static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1352{
1353 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1354 return true;
1355 else
1356 return false;
1357}
1358
1359static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1360{
1361 u32 pos1, pos2;
1362
1363 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1364 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1365
1366 if (pos1 != pos2)
1367 return true;
1368 else
1369 return false;
1370}
1371
1372/**
1373 * dce4_wait_for_vblank - vblank wait asic callback.
1374 *
1375 * @rdev: radeon_device pointer
1376 * @crtc: crtc to wait for vblank on
1377 *
1378 * Wait for vblank on the requested crtc (evergreen+).
1379 */
1380void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1381{
1382 unsigned i = 0;
1383
1384 if (crtc >= rdev->num_crtc)
1385 return;
1386
1387 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1388 return;
1389
1390 /* depending on when we hit vblank, we may be close to active; if so,
1391 * wait for another frame.
1392 */
1393 while (dce4_is_in_vblank(rdev, crtc)) {
1394 if (i++ % 100 == 0) {
1395 if (!dce4_is_counter_moving(rdev, crtc))
1396 break;
1397 }
1398 }
1399
1400 while (!dce4_is_in_vblank(rdev, crtc)) {
1401 if (i++ % 100 == 0) {
1402 if (!dce4_is_counter_moving(rdev, crtc))
1403 break;
1404 }
1405 }
1406}
1407
1408/**
1409 * evergreen_page_flip - pageflip callback.
1410 *
1411 * @rdev: radeon_device pointer
1412 * @crtc_id: crtc to cleanup pageflip on
1413 * @crtc_base: new address of the crtc (GPU MC address)
1414 *
1415 * Triggers the actual pageflip by updating the primary
1416 * surface base address (evergreen+).
1417 */
1418void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1419 bool async)
1420{
1421 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1422
1423 /* update the scanout addresses */
1424 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1425 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1426 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1427 upper_32_bits(crtc_base));
1428 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1429 (u32)crtc_base);
1430 /* post the write */
1431 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1432}
1433
1434/**
1435 * evergreen_page_flip_pending - check if page flip is still pending
1436 *
1437 * @rdev: radeon_device pointer
1438 * @crtc_id: crtc to check
1439 *
1440 * Returns the current update pending status.
1441 */
1442bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1443{
1444 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1445
1446 /* Return current update_pending status: */
1447 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1448 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1449}
1450
1451/* get temperature in millidegrees */
1452int evergreen_get_temp(struct radeon_device *rdev)
1453{
1454 u32 temp, toffset;
1455 int actual_temp = 0;
1456
1457 if (rdev->family == CHIP_JUNIPER) {
1458 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1459 TOFFSET_SHIFT;
1460 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1461 TS0_ADC_DOUT_SHIFT;
1462
1463 if (toffset & 0x100)
1464 actual_temp = temp / 2 - (0x200 - toffset);
1465 else
1466 actual_temp = temp / 2 + toffset;
1467
1468 actual_temp = actual_temp * 1000;
1469
1470 } else {
1471 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1472 ASIC_T_SHIFT;
1473
1474 if (temp & 0x400)
1475 actual_temp = -256;
1476 else if (temp & 0x200)
1477 actual_temp = 255;
1478 else if (temp & 0x100) {
1479 actual_temp = temp & 0x1ff;
1480 actual_temp |= ~0x1ff;
1481 } else
1482 actual_temp = temp & 0xff;
1483
1484 actual_temp = (actual_temp * 1000) / 2;
1485 }
1486
1487 return actual_temp;
1488}
1489
1490int sumo_get_temp(struct radeon_device *rdev)
1491{
1492 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1493 int actual_temp = temp - 49;
1494
1495 return actual_temp * 1000;
1496}
1497
1498/**
1499 * sumo_pm_init_profile - Initialize power profiles callback.
1500 *
1501 * @rdev: radeon_device pointer
1502 *
1503 * Initialize the power states used in profile mode
1504 * (sumo, trinity, SI).
1505 * Used for profile mode only.
1506 */
1507void sumo_pm_init_profile(struct radeon_device *rdev)
1508{
1509 int idx;
1510
1511 /* default */
1512 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1513 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1514 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1515 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1516
1517 /* low,mid sh/mh */
1518 if (rdev->flags & RADEON_IS_MOBILITY)
1519 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1520 else
1521 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1522
1523 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1525 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1527
1528 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1532
1533 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1534 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1537
1538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1542
1543 /* high sh/mh */
1544 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1545 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1546 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1548 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1549 rdev->pm.power_state[idx].num_clock_modes - 1;
1550
1551 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1552 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1553 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1555 rdev->pm.power_state[idx].num_clock_modes - 1;
1556}
1557
1558/**
1559 * btc_pm_init_profile - Initialize power profiles callback.
1560 *
1561 * @rdev: radeon_device pointer
1562 *
1563 * Initialize the power states used in profile mode
1564 * (BTC, cayman).
1565 * Used for profile mode only.
1566 */
1567void btc_pm_init_profile(struct radeon_device *rdev)
1568{
1569 int idx;
1570
1571 /* default */
1572 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1573 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1574 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1575 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1576 /* starting with BTC, there is one state that is used for both
1577 * MH and SH. Difference is that we always use the high clock index for
1578 * mclk.
1579 */
1580 if (rdev->flags & RADEON_IS_MOBILITY)
1581 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1582 else
1583 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1584 /* low sh */
1585 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1586 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1587 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1589 /* mid sh */
1590 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1591 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1592 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1594 /* high sh */
1595 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1596 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1597 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1599 /* low mh */
1600 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1601 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1602 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1604 /* mid mh */
1605 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1606 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1607 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1609 /* high mh */
1610 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1611 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1612 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1614}
1615
1616/**
1617 * evergreen_pm_misc - set additional pm hw parameters callback.
1618 *
1619 * @rdev: radeon_device pointer
1620 *
1621 * Set non-clock parameters associated with a power state
1622 * (voltage, etc.) (evergreen+).
1623 */
1624void evergreen_pm_misc(struct radeon_device *rdev)
1625{
1626 int req_ps_idx = rdev->pm.requested_power_state_index;
1627 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1628 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1629 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1630
1631 if (voltage->type == VOLTAGE_SW) {
1632 /* 0xff0x are flags rather then an actual voltage */
1633 if ((voltage->voltage & 0xff00) == 0xff00)
1634 return;
1635 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1636 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1637 rdev->pm.current_vddc = voltage->voltage;
1638 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1639 }
1640
1641 /* starting with BTC, there is one state that is used for both
1642 * MH and SH. Difference is that we always use the high clock index for
1643 * mclk and vddci.
1644 */
1645 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1646 (rdev->family >= CHIP_BARTS) &&
1647 rdev->pm.active_crtc_count &&
1648 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1649 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1650 voltage = &rdev->pm.power_state[req_ps_idx].
1651 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1652
1653 /* 0xff0x are flags rather then an actual voltage */
1654 if ((voltage->vddci & 0xff00) == 0xff00)
1655 return;
1656 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1657 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1658 rdev->pm.current_vddci = voltage->vddci;
1659 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1660 }
1661 }
1662}
1663
1664/**
1665 * evergreen_pm_prepare - pre-power state change callback.
1666 *
1667 * @rdev: radeon_device pointer
1668 *
1669 * Prepare for a power state change (evergreen+).
1670 */
1671void evergreen_pm_prepare(struct radeon_device *rdev)
1672{
1673 struct drm_device *ddev = rdev->ddev;
1674 struct drm_crtc *crtc;
1675 struct radeon_crtc *radeon_crtc;
1676 u32 tmp;
1677
1678 /* disable any active CRTCs */
1679 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1680 radeon_crtc = to_radeon_crtc(crtc);
1681 if (radeon_crtc->enabled) {
1682 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1683 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1684 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1685 }
1686 }
1687}
1688
1689/**
1690 * evergreen_pm_finish - post-power state change callback.
1691 *
1692 * @rdev: radeon_device pointer
1693 *
1694 * Clean up after a power state change (evergreen+).
1695 */
1696void evergreen_pm_finish(struct radeon_device *rdev)
1697{
1698 struct drm_device *ddev = rdev->ddev;
1699 struct drm_crtc *crtc;
1700 struct radeon_crtc *radeon_crtc;
1701 u32 tmp;
1702
1703 /* enable any active CRTCs */
1704 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1705 radeon_crtc = to_radeon_crtc(crtc);
1706 if (radeon_crtc->enabled) {
1707 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1708 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1709 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1710 }
1711 }
1712}
1713
1714/**
1715 * evergreen_hpd_sense - hpd sense callback.
1716 *
1717 * @rdev: radeon_device pointer
1718 * @hpd: hpd (hotplug detect) pin
1719 *
1720 * Checks if a digital monitor is connected (evergreen+).
1721 * Returns true if connected, false if not connected.
1722 */
1723bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1724{
1725 if (hpd == RADEON_HPD_NONE)
1726 return false;
1727
1728 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1729}
1730
1731/**
1732 * evergreen_hpd_set_polarity - hpd set polarity callback.
1733 *
1734 * @rdev: radeon_device pointer
1735 * @hpd: hpd (hotplug detect) pin
1736 *
1737 * Set the polarity of the hpd pin (evergreen+).
1738 */
1739void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1740 enum radeon_hpd_id hpd)
1741{
1742 bool connected = evergreen_hpd_sense(rdev, hpd);
1743
1744 if (hpd == RADEON_HPD_NONE)
1745 return;
1746
1747 if (connected)
1748 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1749 else
1750 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1751}
1752
1753/**
1754 * evergreen_hpd_init - hpd setup callback.
1755 *
1756 * @rdev: radeon_device pointer
1757 *
1758 * Setup the hpd pins used by the card (evergreen+).
1759 * Enable the pin, set the polarity, and enable the hpd interrupts.
1760 */
1761void evergreen_hpd_init(struct radeon_device *rdev)
1762{
1763 struct drm_device *dev = rdev->ddev;
1764 struct drm_connector *connector;
1765 unsigned enabled = 0;
1766 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1767 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1768
1769 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1770 enum radeon_hpd_id hpd =
1771 to_radeon_connector(connector)->hpd.hpd;
1772
1773 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1774 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1775 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1776 * aux dp channel on imac and help (but not completely fix)
1777 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1778 * also avoid interrupt storms during dpms.
1779 */
1780 continue;
1781 }
1782
1783 if (hpd == RADEON_HPD_NONE)
1784 continue;
1785
1786 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1787 enabled |= 1 << hpd;
1788
1789 radeon_hpd_set_polarity(rdev, hpd);
1790 }
1791 radeon_irq_kms_enable_hpd(rdev, enabled);
1792}
1793
1794/**
1795 * evergreen_hpd_fini - hpd tear down callback.
1796 *
1797 * @rdev: radeon_device pointer
1798 *
1799 * Tear down the hpd pins used by the card (evergreen+).
1800 * Disable the hpd interrupts.
1801 */
1802void evergreen_hpd_fini(struct radeon_device *rdev)
1803{
1804 struct drm_device *dev = rdev->ddev;
1805 struct drm_connector *connector;
1806 unsigned disabled = 0;
1807
1808 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1809 enum radeon_hpd_id hpd =
1810 to_radeon_connector(connector)->hpd.hpd;
1811
1812 if (hpd == RADEON_HPD_NONE)
1813 continue;
1814
1815 WREG32(DC_HPDx_CONTROL(hpd), 0);
1816 disabled |= 1 << hpd;
1817 }
1818 radeon_irq_kms_disable_hpd(rdev, disabled);
1819}
1820
1821/* watermark setup */
1822
1823static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1824 struct radeon_crtc *radeon_crtc,
1825 struct drm_display_mode *mode,
1826 struct drm_display_mode *other_mode)
1827{
1828 u32 tmp, buffer_alloc, i;
1829 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1830 /*
1831 * Line Buffer Setup
1832 * There are 3 line buffers, each one shared by 2 display controllers.
1833 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1834 * the display controllers. The paritioning is done via one of four
1835 * preset allocations specified in bits 2:0:
1836 * first display controller
1837 * 0 - first half of lb (3840 * 2)
1838 * 1 - first 3/4 of lb (5760 * 2)
1839 * 2 - whole lb (7680 * 2), other crtc must be disabled
1840 * 3 - first 1/4 of lb (1920 * 2)
1841 * second display controller
1842 * 4 - second half of lb (3840 * 2)
1843 * 5 - second 3/4 of lb (5760 * 2)
1844 * 6 - whole lb (7680 * 2), other crtc must be disabled
1845 * 7 - last 1/4 of lb (1920 * 2)
1846 */
1847 /* this can get tricky if we have two large displays on a paired group
1848 * of crtcs. Ideally for multiple large displays we'd assign them to
1849 * non-linked crtcs for maximum line buffer allocation.
1850 */
1851 if (radeon_crtc->base.enabled && mode) {
1852 if (other_mode) {
1853 tmp = 0; /* 1/2 */
1854 buffer_alloc = 1;
1855 } else {
1856 tmp = 2; /* whole */
1857 buffer_alloc = 2;
1858 }
1859 } else {
1860 tmp = 0;
1861 buffer_alloc = 0;
1862 }
1863
1864 /* second controller of the pair uses second half of the lb */
1865 if (radeon_crtc->crtc_id % 2)
1866 tmp += 4;
1867 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1868
1869 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1870 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1871 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1872 for (i = 0; i < rdev->usec_timeout; i++) {
1873 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1874 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1875 break;
1876 udelay(1);
1877 }
1878 }
1879
1880 if (radeon_crtc->base.enabled && mode) {
1881 switch (tmp) {
1882 case 0:
1883 case 4:
1884 default:
1885 if (ASIC_IS_DCE5(rdev))
1886 return 4096 * 2;
1887 else
1888 return 3840 * 2;
1889 case 1:
1890 case 5:
1891 if (ASIC_IS_DCE5(rdev))
1892 return 6144 * 2;
1893 else
1894 return 5760 * 2;
1895 case 2:
1896 case 6:
1897 if (ASIC_IS_DCE5(rdev))
1898 return 8192 * 2;
1899 else
1900 return 7680 * 2;
1901 case 3:
1902 case 7:
1903 if (ASIC_IS_DCE5(rdev))
1904 return 2048 * 2;
1905 else
1906 return 1920 * 2;
1907 }
1908 }
1909
1910 /* controller not enabled, so no lb used */
1911 return 0;
1912}
1913
1914u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1915{
1916 u32 tmp = RREG32(MC_SHARED_CHMAP);
1917
1918 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1919 case 0:
1920 default:
1921 return 1;
1922 case 1:
1923 return 2;
1924 case 2:
1925 return 4;
1926 case 3:
1927 return 8;
1928 }
1929}
1930
1931struct evergreen_wm_params {
1932 u32 dram_channels; /* number of dram channels */
1933 u32 yclk; /* bandwidth per dram data pin in kHz */
1934 u32 sclk; /* engine clock in kHz */
1935 u32 disp_clk; /* display clock in kHz */
1936 u32 src_width; /* viewport width */
1937 u32 active_time; /* active display time in ns */
1938 u32 blank_time; /* blank time in ns */
1939 bool interlaced; /* mode is interlaced */
1940 fixed20_12 vsc; /* vertical scale ratio */
1941 u32 num_heads; /* number of active crtcs */
1942 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1943 u32 lb_size; /* line buffer allocated to pipe */
1944 u32 vtaps; /* vertical scaler taps */
1945};
1946
1947static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1948{
1949 /* Calculate DRAM Bandwidth and the part allocated to display. */
1950 fixed20_12 dram_efficiency; /* 0.7 */
1951 fixed20_12 yclk, dram_channels, bandwidth;
1952 fixed20_12 a;
1953
1954 a.full = dfixed_const(1000);
1955 yclk.full = dfixed_const(wm->yclk);
1956 yclk.full = dfixed_div(yclk, a);
1957 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1958 a.full = dfixed_const(10);
1959 dram_efficiency.full = dfixed_const(7);
1960 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1961 bandwidth.full = dfixed_mul(dram_channels, yclk);
1962 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1963
1964 return dfixed_trunc(bandwidth);
1965}
1966
1967static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1968{
1969 /* Calculate DRAM Bandwidth and the part allocated to display. */
1970 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1971 fixed20_12 yclk, dram_channels, bandwidth;
1972 fixed20_12 a;
1973
1974 a.full = dfixed_const(1000);
1975 yclk.full = dfixed_const(wm->yclk);
1976 yclk.full = dfixed_div(yclk, a);
1977 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978 a.full = dfixed_const(10);
1979 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1980 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1981 bandwidth.full = dfixed_mul(dram_channels, yclk);
1982 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1983
1984 return dfixed_trunc(bandwidth);
1985}
1986
1987static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1988{
1989 /* Calculate the display Data return Bandwidth */
1990 fixed20_12 return_efficiency; /* 0.8 */
1991 fixed20_12 sclk, bandwidth;
1992 fixed20_12 a;
1993
1994 a.full = dfixed_const(1000);
1995 sclk.full = dfixed_const(wm->sclk);
1996 sclk.full = dfixed_div(sclk, a);
1997 a.full = dfixed_const(10);
1998 return_efficiency.full = dfixed_const(8);
1999 return_efficiency.full = dfixed_div(return_efficiency, a);
2000 a.full = dfixed_const(32);
2001 bandwidth.full = dfixed_mul(a, sclk);
2002 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2003
2004 return dfixed_trunc(bandwidth);
2005}
2006
2007static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2008{
2009 /* Calculate the DMIF Request Bandwidth */
2010 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2011 fixed20_12 disp_clk, bandwidth;
2012 fixed20_12 a;
2013
2014 a.full = dfixed_const(1000);
2015 disp_clk.full = dfixed_const(wm->disp_clk);
2016 disp_clk.full = dfixed_div(disp_clk, a);
2017 a.full = dfixed_const(10);
2018 disp_clk_request_efficiency.full = dfixed_const(8);
2019 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2020 a.full = dfixed_const(32);
2021 bandwidth.full = dfixed_mul(a, disp_clk);
2022 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2023
2024 return dfixed_trunc(bandwidth);
2025}
2026
2027static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2028{
2029 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2030 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2031 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2032 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2033
2034 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2035}
2036
2037static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2038{
2039 /* Calculate the display mode Average Bandwidth
2040 * DisplayMode should contain the source and destination dimensions,
2041 * timing, etc.
2042 */
2043 fixed20_12 bpp;
2044 fixed20_12 line_time;
2045 fixed20_12 src_width;
2046 fixed20_12 bandwidth;
2047 fixed20_12 a;
2048
2049 a.full = dfixed_const(1000);
2050 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2051 line_time.full = dfixed_div(line_time, a);
2052 bpp.full = dfixed_const(wm->bytes_per_pixel);
2053 src_width.full = dfixed_const(wm->src_width);
2054 bandwidth.full = dfixed_mul(src_width, bpp);
2055 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2056 bandwidth.full = dfixed_div(bandwidth, line_time);
2057
2058 return dfixed_trunc(bandwidth);
2059}
2060
2061static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2062{
2063 /* First calcualte the latency in ns */
2064 u32 mc_latency = 2000; /* 2000 ns. */
2065 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2066 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2067 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2068 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2069 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2070 (wm->num_heads * cursor_line_pair_return_time);
2071 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2072 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2073 fixed20_12 a, b, c;
2074
2075 if (wm->num_heads == 0)
2076 return 0;
2077
2078 a.full = dfixed_const(2);
2079 b.full = dfixed_const(1);
2080 if ((wm->vsc.full > a.full) ||
2081 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2082 (wm->vtaps >= 5) ||
2083 ((wm->vsc.full >= a.full) && wm->interlaced))
2084 max_src_lines_per_dst_line = 4;
2085 else
2086 max_src_lines_per_dst_line = 2;
2087
2088 a.full = dfixed_const(available_bandwidth);
2089 b.full = dfixed_const(wm->num_heads);
2090 a.full = dfixed_div(a, b);
2091
2092 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2093
2094 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2095 b.full = dfixed_const(1000);
2096 c.full = dfixed_const(lb_fill_bw);
2097 b.full = dfixed_div(c, b);
2098 a.full = dfixed_div(a, b);
2099 line_fill_time = dfixed_trunc(a);
2100
2101 if (line_fill_time < wm->active_time)
2102 return latency;
2103 else
2104 return latency + (line_fill_time - wm->active_time);
2105
2106}
2107
2108static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2109{
2110 if (evergreen_average_bandwidth(wm) <=
2111 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2112 return true;
2113 else
2114 return false;
2115};
2116
2117static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2118{
2119 if (evergreen_average_bandwidth(wm) <=
2120 (evergreen_available_bandwidth(wm) / wm->num_heads))
2121 return true;
2122 else
2123 return false;
2124};
2125
2126static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2127{
2128 u32 lb_partitions = wm->lb_size / wm->src_width;
2129 u32 line_time = wm->active_time + wm->blank_time;
2130 u32 latency_tolerant_lines;
2131 u32 latency_hiding;
2132 fixed20_12 a;
2133
2134 a.full = dfixed_const(1);
2135 if (wm->vsc.full > a.full)
2136 latency_tolerant_lines = 1;
2137 else {
2138 if (lb_partitions <= (wm->vtaps + 1))
2139 latency_tolerant_lines = 1;
2140 else
2141 latency_tolerant_lines = 2;
2142 }
2143
2144 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2145
2146 if (evergreen_latency_watermark(wm) <= latency_hiding)
2147 return true;
2148 else
2149 return false;
2150}
2151
2152static void evergreen_program_watermarks(struct radeon_device *rdev,
2153 struct radeon_crtc *radeon_crtc,
2154 u32 lb_size, u32 num_heads)
2155{
2156 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2157 struct evergreen_wm_params wm_low, wm_high;
2158 u32 dram_channels;
2159 u32 active_time;
2160 u32 line_time = 0;
2161 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2162 u32 priority_a_mark = 0, priority_b_mark = 0;
2163 u32 priority_a_cnt = PRIORITY_OFF;
2164 u32 priority_b_cnt = PRIORITY_OFF;
2165 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2166 u32 tmp, arb_control3;
2167 fixed20_12 a, b, c;
2168
2169 if (radeon_crtc->base.enabled && num_heads && mode) {
2170 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2171 (u32)mode->clock);
2172 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2173 (u32)mode->clock);
2174 line_time = min(line_time, (u32)65535);
2175 priority_a_cnt = 0;
2176 priority_b_cnt = 0;
2177 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2178
2179 /* watermark for high clocks */
2180 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2181 wm_high.yclk =
2182 radeon_dpm_get_mclk(rdev, false) * 10;
2183 wm_high.sclk =
2184 radeon_dpm_get_sclk(rdev, false) * 10;
2185 } else {
2186 wm_high.yclk = rdev->pm.current_mclk * 10;
2187 wm_high.sclk = rdev->pm.current_sclk * 10;
2188 }
2189
2190 wm_high.disp_clk = mode->clock;
2191 wm_high.src_width = mode->crtc_hdisplay;
2192 wm_high.active_time = active_time;
2193 wm_high.blank_time = line_time - wm_high.active_time;
2194 wm_high.interlaced = false;
2195 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2196 wm_high.interlaced = true;
2197 wm_high.vsc = radeon_crtc->vsc;
2198 wm_high.vtaps = 1;
2199 if (radeon_crtc->rmx_type != RMX_OFF)
2200 wm_high.vtaps = 2;
2201 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2202 wm_high.lb_size = lb_size;
2203 wm_high.dram_channels = dram_channels;
2204 wm_high.num_heads = num_heads;
2205
2206 /* watermark for low clocks */
2207 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2208 wm_low.yclk =
2209 radeon_dpm_get_mclk(rdev, true) * 10;
2210 wm_low.sclk =
2211 radeon_dpm_get_sclk(rdev, true) * 10;
2212 } else {
2213 wm_low.yclk = rdev->pm.current_mclk * 10;
2214 wm_low.sclk = rdev->pm.current_sclk * 10;
2215 }
2216
2217 wm_low.disp_clk = mode->clock;
2218 wm_low.src_width = mode->crtc_hdisplay;
2219 wm_low.active_time = active_time;
2220 wm_low.blank_time = line_time - wm_low.active_time;
2221 wm_low.interlaced = false;
2222 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2223 wm_low.interlaced = true;
2224 wm_low.vsc = radeon_crtc->vsc;
2225 wm_low.vtaps = 1;
2226 if (radeon_crtc->rmx_type != RMX_OFF)
2227 wm_low.vtaps = 2;
2228 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2229 wm_low.lb_size = lb_size;
2230 wm_low.dram_channels = dram_channels;
2231 wm_low.num_heads = num_heads;
2232
2233 /* set for high clocks */
2234 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2235 /* set for low clocks */
2236 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2237
2238 /* possibly force display priority to high */
2239 /* should really do this at mode validation time... */
2240 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2241 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2242 !evergreen_check_latency_hiding(&wm_high) ||
2243 (rdev->disp_priority == 2)) {
2244 DRM_DEBUG_KMS("force priority a to high\n");
2245 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2246 }
2247 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2248 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2249 !evergreen_check_latency_hiding(&wm_low) ||
2250 (rdev->disp_priority == 2)) {
2251 DRM_DEBUG_KMS("force priority b to high\n");
2252 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2253 }
2254
2255 a.full = dfixed_const(1000);
2256 b.full = dfixed_const(mode->clock);
2257 b.full = dfixed_div(b, a);
2258 c.full = dfixed_const(latency_watermark_a);
2259 c.full = dfixed_mul(c, b);
2260 c.full = dfixed_mul(c, radeon_crtc->hsc);
2261 c.full = dfixed_div(c, a);
2262 a.full = dfixed_const(16);
2263 c.full = dfixed_div(c, a);
2264 priority_a_mark = dfixed_trunc(c);
2265 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2266
2267 a.full = dfixed_const(1000);
2268 b.full = dfixed_const(mode->clock);
2269 b.full = dfixed_div(b, a);
2270 c.full = dfixed_const(latency_watermark_b);
2271 c.full = dfixed_mul(c, b);
2272 c.full = dfixed_mul(c, radeon_crtc->hsc);
2273 c.full = dfixed_div(c, a);
2274 a.full = dfixed_const(16);
2275 c.full = dfixed_div(c, a);
2276 priority_b_mark = dfixed_trunc(c);
2277 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2278
2279 /* Save number of lines the linebuffer leads before the scanout */
2280 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2281 }
2282
2283 /* select wm A */
2284 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2285 tmp = arb_control3;
2286 tmp &= ~LATENCY_WATERMARK_MASK(3);
2287 tmp |= LATENCY_WATERMARK_MASK(1);
2288 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2289 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2290 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2291 LATENCY_HIGH_WATERMARK(line_time)));
2292 /* select wm B */
2293 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2294 tmp &= ~LATENCY_WATERMARK_MASK(3);
2295 tmp |= LATENCY_WATERMARK_MASK(2);
2296 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2297 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2298 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2299 LATENCY_HIGH_WATERMARK(line_time)));
2300 /* restore original selection */
2301 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2302
2303 /* write the priority marks */
2304 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2305 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2306
2307 /* save values for DPM */
2308 radeon_crtc->line_time = line_time;
2309 radeon_crtc->wm_high = latency_watermark_a;
2310 radeon_crtc->wm_low = latency_watermark_b;
2311}
2312
2313/**
2314 * evergreen_bandwidth_update - update display watermarks callback.
2315 *
2316 * @rdev: radeon_device pointer
2317 *
2318 * Update the display watermarks based on the requested mode(s)
2319 * (evergreen+).
2320 */
2321void evergreen_bandwidth_update(struct radeon_device *rdev)
2322{
2323 struct drm_display_mode *mode0 = NULL;
2324 struct drm_display_mode *mode1 = NULL;
2325 u32 num_heads = 0, lb_size;
2326 int i;
2327
2328 if (!rdev->mode_info.mode_config_initialized)
2329 return;
2330
2331 radeon_update_display_priority(rdev);
2332
2333 for (i = 0; i < rdev->num_crtc; i++) {
2334 if (rdev->mode_info.crtcs[i]->base.enabled)
2335 num_heads++;
2336 }
2337 for (i = 0; i < rdev->num_crtc; i += 2) {
2338 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2339 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2340 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2341 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2342 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2343 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2344 }
2345}
2346
2347/**
2348 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2349 *
2350 * @rdev: radeon_device pointer
2351 *
2352 * Wait for the MC (memory controller) to be idle.
2353 * (evergreen+).
2354 * Returns 0 if the MC is idle, -1 if not.
2355 */
2356int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2357{
2358 unsigned i;
2359 u32 tmp;
2360
2361 for (i = 0; i < rdev->usec_timeout; i++) {
2362 /* read MC_STATUS */
2363 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2364 if (!tmp)
2365 return 0;
2366 udelay(1);
2367 }
2368 return -1;
2369}
2370
2371/*
2372 * GART
2373 */
2374void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2375{
2376 unsigned i;
2377 u32 tmp;
2378
2379 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2380
2381 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2382 for (i = 0; i < rdev->usec_timeout; i++) {
2383 /* read MC_STATUS */
2384 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2385 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2386 if (tmp == 2) {
2387 pr_warn("[drm] r600 flush TLB failed\n");
2388 return;
2389 }
2390 if (tmp) {
2391 return;
2392 }
2393 udelay(1);
2394 }
2395}
2396
2397static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2398{
2399 u32 tmp;
2400 int r;
2401
2402 if (rdev->gart.robj == NULL) {
2403 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2404 return -EINVAL;
2405 }
2406 r = radeon_gart_table_vram_pin(rdev);
2407 if (r)
2408 return r;
2409 /* Setup L2 cache */
2410 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2411 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2412 EFFECTIVE_L2_QUEUE_SIZE(7));
2413 WREG32(VM_L2_CNTL2, 0);
2414 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2415 /* Setup TLB control */
2416 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2417 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2418 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2419 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2420 if (rdev->flags & RADEON_IS_IGP) {
2421 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2422 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2423 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2424 } else {
2425 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2426 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2427 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2428 if ((rdev->family == CHIP_JUNIPER) ||
2429 (rdev->family == CHIP_CYPRESS) ||
2430 (rdev->family == CHIP_HEMLOCK) ||
2431 (rdev->family == CHIP_BARTS))
2432 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2433 }
2434 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2435 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2436 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2438 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2439 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2440 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2441 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2442 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2443 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2444 (u32)(rdev->dummy_page.addr >> 12));
2445 WREG32(VM_CONTEXT1_CNTL, 0);
2446
2447 evergreen_pcie_gart_tlb_flush(rdev);
2448 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2449 (unsigned)(rdev->mc.gtt_size >> 20),
2450 (unsigned long long)rdev->gart.table_addr);
2451 rdev->gart.ready = true;
2452 return 0;
2453}
2454
2455static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2456{
2457 u32 tmp;
2458
2459 /* Disable all tables */
2460 WREG32(VM_CONTEXT0_CNTL, 0);
2461 WREG32(VM_CONTEXT1_CNTL, 0);
2462
2463 /* Setup L2 cache */
2464 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2465 EFFECTIVE_L2_QUEUE_SIZE(7));
2466 WREG32(VM_L2_CNTL2, 0);
2467 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2468 /* Setup TLB control */
2469 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2470 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2471 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2472 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2473 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2474 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2475 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2476 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2477 radeon_gart_table_vram_unpin(rdev);
2478}
2479
2480static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2481{
2482 evergreen_pcie_gart_disable(rdev);
2483 radeon_gart_table_vram_free(rdev);
2484 radeon_gart_fini(rdev);
2485}
2486
2487
2488static void evergreen_agp_enable(struct radeon_device *rdev)
2489{
2490 u32 tmp;
2491
2492 /* Setup L2 cache */
2493 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2494 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2495 EFFECTIVE_L2_QUEUE_SIZE(7));
2496 WREG32(VM_L2_CNTL2, 0);
2497 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2498 /* Setup TLB control */
2499 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2500 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2501 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2502 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2503 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2504 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2505 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2506 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2507 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2508 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2509 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2510 WREG32(VM_CONTEXT0_CNTL, 0);
2511 WREG32(VM_CONTEXT1_CNTL, 0);
2512}
2513
2514static const unsigned ni_dig_offsets[] =
2515{
2516 NI_DIG0_REGISTER_OFFSET,
2517 NI_DIG1_REGISTER_OFFSET,
2518 NI_DIG2_REGISTER_OFFSET,
2519 NI_DIG3_REGISTER_OFFSET,
2520 NI_DIG4_REGISTER_OFFSET,
2521 NI_DIG5_REGISTER_OFFSET
2522};
2523
2524static const unsigned ni_tx_offsets[] =
2525{
2526 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2527 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2528 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2529 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2530 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2531 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2532};
2533
2534static const unsigned evergreen_dp_offsets[] =
2535{
2536 EVERGREEN_DP0_REGISTER_OFFSET,
2537 EVERGREEN_DP1_REGISTER_OFFSET,
2538 EVERGREEN_DP2_REGISTER_OFFSET,
2539 EVERGREEN_DP3_REGISTER_OFFSET,
2540 EVERGREEN_DP4_REGISTER_OFFSET,
2541 EVERGREEN_DP5_REGISTER_OFFSET
2542};
2543
2544static const unsigned evergreen_disp_int_status[] =
2545{
2546 DISP_INTERRUPT_STATUS,
2547 DISP_INTERRUPT_STATUS_CONTINUE,
2548 DISP_INTERRUPT_STATUS_CONTINUE2,
2549 DISP_INTERRUPT_STATUS_CONTINUE3,
2550 DISP_INTERRUPT_STATUS_CONTINUE4,
2551 DISP_INTERRUPT_STATUS_CONTINUE5
2552};
2553
2554/*
2555 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2556 * We go from crtc to connector and it is not relible since it
2557 * should be an opposite direction .If crtc is enable then
2558 * find the dig_fe which selects this crtc and insure that it enable.
2559 * if such dig_fe is found then find dig_be which selects found dig_be and
2560 * insure that it enable and in DP_SST mode.
2561 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2562 * from dp symbols clocks .
2563 */
2564static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2565 unsigned crtc_id, unsigned *ret_dig_fe)
2566{
2567 unsigned i;
2568 unsigned dig_fe;
2569 unsigned dig_be;
2570 unsigned dig_en_be;
2571 unsigned uniphy_pll;
2572 unsigned digs_fe_selected;
2573 unsigned dig_be_mode;
2574 unsigned dig_fe_mask;
2575 bool is_enabled = false;
2576 bool found_crtc = false;
2577
2578 /* loop through all running dig_fe to find selected crtc */
2579 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2580 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2581 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2582 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2583 /* found running pipe */
2584 found_crtc = true;
2585 dig_fe_mask = 1 << i;
2586 dig_fe = i;
2587 break;
2588 }
2589 }
2590
2591 if (found_crtc) {
2592 /* loop through all running dig_be to find selected dig_fe */
2593 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2594 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2595 /* if dig_fe_selected by dig_be? */
2596 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2597 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2598 if (dig_fe_mask & digs_fe_selected &&
2599 /* if dig_be in sst mode? */
2600 dig_be_mode == NI_DIG_BE_DPSST) {
2601 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2602 ni_dig_offsets[i]);
2603 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2604 ni_tx_offsets[i]);
2605 /* dig_be enable and tx is running */
2606 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2607 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2608 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2609 is_enabled = true;
2610 *ret_dig_fe = dig_fe;
2611 break;
2612 }
2613 }
2614 }
2615 }
2616
2617 return is_enabled;
2618}
2619
2620/*
2621 * Blank dig when in dp sst mode
2622 * Dig ignores crtc timing
2623 */
2624static void evergreen_blank_dp_output(struct radeon_device *rdev,
2625 unsigned dig_fe)
2626{
2627 unsigned stream_ctrl;
2628 unsigned fifo_ctrl;
2629 unsigned counter = 0;
2630
2631 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2632 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2633 return;
2634 }
2635
2636 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2637 evergreen_dp_offsets[dig_fe]);
2638 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2639 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2640 return;
2641 }
2642
2643 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2644 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2645 evergreen_dp_offsets[dig_fe], stream_ctrl);
2646
2647 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648 evergreen_dp_offsets[dig_fe]);
2649 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2650 msleep(1);
2651 counter++;
2652 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2653 evergreen_dp_offsets[dig_fe]);
2654 }
2655 if (counter >= 32 )
2656 DRM_ERROR("counter exceeds %d\n", counter);
2657
2658 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2659 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2660 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2661
2662}
2663
2664void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2665{
2666 u32 crtc_enabled, tmp, frame_count, blackout;
2667 int i, j;
2668 unsigned dig_fe;
2669
2670 if (!ASIC_IS_NODCE(rdev)) {
2671 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2672 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2673
2674 /* disable VGA render */
2675 WREG32(VGA_RENDER_CONTROL, 0);
2676 }
2677 /* blank the display controllers */
2678 for (i = 0; i < rdev->num_crtc; i++) {
2679 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2680 if (crtc_enabled) {
2681 save->crtc_enabled[i] = true;
2682 if (ASIC_IS_DCE6(rdev)) {
2683 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2684 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2685 radeon_wait_for_vblank(rdev, i);
2686 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2687 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2688 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2689 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2690 }
2691 } else {
2692 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2693 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2694 radeon_wait_for_vblank(rdev, i);
2695 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2696 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2697 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2698 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2699 }
2700 }
2701 /* wait for the next frame */
2702 frame_count = radeon_get_vblank_counter(rdev, i);
2703 for (j = 0; j < rdev->usec_timeout; j++) {
2704 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2705 break;
2706 udelay(1);
2707 }
2708 /*we should disable dig if it drives dp sst*/
2709 /*but we are in radeon_device_init and the topology is unknown*/
2710 /*and it is available after radeon_modeset_init*/
2711 /*the following method radeon_atom_encoder_dpms_dig*/
2712 /*does the job if we initialize it properly*/
2713 /*for now we do it this manually*/
2714 /**/
2715 if (ASIC_IS_DCE5(rdev) &&
2716 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2717 evergreen_blank_dp_output(rdev, dig_fe);
2718 /*we could remove 6 lines below*/
2719 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2720 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2721 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2722 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2723 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2724 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2725 save->crtc_enabled[i] = false;
2726 /* ***** */
2727 } else {
2728 save->crtc_enabled[i] = false;
2729 }
2730 }
2731
2732 radeon_mc_wait_for_idle(rdev);
2733
2734 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2735 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2736 /* Block CPU access */
2737 WREG32(BIF_FB_EN, 0);
2738 /* blackout the MC */
2739 blackout &= ~BLACKOUT_MODE_MASK;
2740 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2741 }
2742 /* wait for the MC to settle */
2743 udelay(100);
2744
2745 /* lock double buffered regs */
2746 for (i = 0; i < rdev->num_crtc; i++) {
2747 if (save->crtc_enabled[i]) {
2748 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2749 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2750 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2751 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2752 }
2753 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2754 if (!(tmp & 1)) {
2755 tmp |= 1;
2756 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2757 }
2758 }
2759 }
2760}
2761
2762void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2763{
2764 u32 tmp, frame_count;
2765 int i, j;
2766
2767 /* update crtc base addresses */
2768 for (i = 0; i < rdev->num_crtc; i++) {
2769 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2770 upper_32_bits(rdev->mc.vram_start));
2771 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2772 upper_32_bits(rdev->mc.vram_start));
2773 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2774 (u32)rdev->mc.vram_start);
2775 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2776 (u32)rdev->mc.vram_start);
2777 }
2778
2779 if (!ASIC_IS_NODCE(rdev)) {
2780 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2781 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2782 }
2783
2784 /* unlock regs and wait for update */
2785 for (i = 0; i < rdev->num_crtc; i++) {
2786 if (save->crtc_enabled[i]) {
2787 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2788 if ((tmp & 0x7) != 0) {
2789 tmp &= ~0x7;
2790 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2791 }
2792 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2793 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2794 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2795 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2796 }
2797 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2798 if (tmp & 1) {
2799 tmp &= ~1;
2800 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2801 }
2802 for (j = 0; j < rdev->usec_timeout; j++) {
2803 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2804 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2805 break;
2806 udelay(1);
2807 }
2808 }
2809 }
2810
2811 /* unblackout the MC */
2812 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2813 tmp &= ~BLACKOUT_MODE_MASK;
2814 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2815 /* allow CPU access */
2816 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2817
2818 for (i = 0; i < rdev->num_crtc; i++) {
2819 if (save->crtc_enabled[i]) {
2820 if (ASIC_IS_DCE6(rdev)) {
2821 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2822 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2823 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2824 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2825 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2826 } else {
2827 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2828 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2829 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2830 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2831 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2832 }
2833 /* wait for the next frame */
2834 frame_count = radeon_get_vblank_counter(rdev, i);
2835 for (j = 0; j < rdev->usec_timeout; j++) {
2836 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2837 break;
2838 udelay(1);
2839 }
2840 }
2841 }
2842 if (!ASIC_IS_NODCE(rdev)) {
2843 /* Unlock vga access */
2844 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2845 mdelay(1);
2846 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2847 }
2848}
2849
2850void evergreen_mc_program(struct radeon_device *rdev)
2851{
2852 struct evergreen_mc_save save;
2853 u32 tmp;
2854 int i, j;
2855
2856 /* Initialize HDP */
2857 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2858 WREG32((0x2c14 + j), 0x00000000);
2859 WREG32((0x2c18 + j), 0x00000000);
2860 WREG32((0x2c1c + j), 0x00000000);
2861 WREG32((0x2c20 + j), 0x00000000);
2862 WREG32((0x2c24 + j), 0x00000000);
2863 }
2864 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2865
2866 evergreen_mc_stop(rdev, &save);
2867 if (evergreen_mc_wait_for_idle(rdev)) {
2868 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2869 }
2870 /* Lockout access through VGA aperture*/
2871 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2872 /* Update configuration */
2873 if (rdev->flags & RADEON_IS_AGP) {
2874 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2875 /* VRAM before AGP */
2876 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2877 rdev->mc.vram_start >> 12);
2878 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2879 rdev->mc.gtt_end >> 12);
2880 } else {
2881 /* VRAM after AGP */
2882 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2883 rdev->mc.gtt_start >> 12);
2884 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2885 rdev->mc.vram_end >> 12);
2886 }
2887 } else {
2888 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2889 rdev->mc.vram_start >> 12);
2890 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2891 rdev->mc.vram_end >> 12);
2892 }
2893 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2894 /* llano/ontario only */
2895 if ((rdev->family == CHIP_PALM) ||
2896 (rdev->family == CHIP_SUMO) ||
2897 (rdev->family == CHIP_SUMO2)) {
2898 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2899 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2900 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2901 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2902 }
2903 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2904 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2905 WREG32(MC_VM_FB_LOCATION, tmp);
2906 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2907 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2908 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2909 if (rdev->flags & RADEON_IS_AGP) {
2910 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2911 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2912 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2913 } else {
2914 WREG32(MC_VM_AGP_BASE, 0);
2915 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2916 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2917 }
2918 if (evergreen_mc_wait_for_idle(rdev)) {
2919 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2920 }
2921 evergreen_mc_resume(rdev, &save);
2922 /* we need to own VRAM, so turn off the VGA renderer here
2923 * to stop it overwriting our objects */
2924 rv515_vga_render_disable(rdev);
2925}
2926
2927/*
2928 * CP.
2929 */
2930void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2931{
2932 struct radeon_ring *ring = &rdev->ring[ib->ring];
2933 u32 next_rptr;
2934
2935 /* set to DX10/11 mode */
2936 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2937 radeon_ring_write(ring, 1);
2938
2939 if (ring->rptr_save_reg) {
2940 next_rptr = ring->wptr + 3 + 4;
2941 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2942 radeon_ring_write(ring, ((ring->rptr_save_reg -
2943 PACKET3_SET_CONFIG_REG_START) >> 2));
2944 radeon_ring_write(ring, next_rptr);
2945 } else if (rdev->wb.enabled) {
2946 next_rptr = ring->wptr + 5 + 4;
2947 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2948 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2949 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2950 radeon_ring_write(ring, next_rptr);
2951 radeon_ring_write(ring, 0);
2952 }
2953
2954 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2955 radeon_ring_write(ring,
2956#ifdef __BIG_ENDIAN
2957 (2 << 0) |
2958#endif
2959 (ib->gpu_addr & 0xFFFFFFFC));
2960 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2961 radeon_ring_write(ring, ib->length_dw);
2962}
2963
2964
2965static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2966{
2967 const __be32 *fw_data;
2968 int i;
2969
2970 if (!rdev->me_fw || !rdev->pfp_fw)
2971 return -EINVAL;
2972
2973 r700_cp_stop(rdev);
2974 WREG32(CP_RB_CNTL,
2975#ifdef __BIG_ENDIAN
2976 BUF_SWAP_32BIT |
2977#endif
2978 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2979
2980 fw_data = (const __be32 *)rdev->pfp_fw->data;
2981 WREG32(CP_PFP_UCODE_ADDR, 0);
2982 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2983 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2984 WREG32(CP_PFP_UCODE_ADDR, 0);
2985
2986 fw_data = (const __be32 *)rdev->me_fw->data;
2987 WREG32(CP_ME_RAM_WADDR, 0);
2988 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2989 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2990
2991 WREG32(CP_PFP_UCODE_ADDR, 0);
2992 WREG32(CP_ME_RAM_WADDR, 0);
2993 WREG32(CP_ME_RAM_RADDR, 0);
2994 return 0;
2995}
2996
2997static int evergreen_cp_start(struct radeon_device *rdev)
2998{
2999 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3000 int r, i;
3001 uint32_t cp_me;
3002
3003 r = radeon_ring_lock(rdev, ring, 7);
3004 if (r) {
3005 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3006 return r;
3007 }
3008 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3009 radeon_ring_write(ring, 0x1);
3010 radeon_ring_write(ring, 0x0);
3011 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3012 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3013 radeon_ring_write(ring, 0);
3014 radeon_ring_write(ring, 0);
3015 radeon_ring_unlock_commit(rdev, ring, false);
3016
3017 cp_me = 0xff;
3018 WREG32(CP_ME_CNTL, cp_me);
3019
3020 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3021 if (r) {
3022 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3023 return r;
3024 }
3025
3026 /* setup clear context state */
3027 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3028 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3029
3030 for (i = 0; i < evergreen_default_size; i++)
3031 radeon_ring_write(ring, evergreen_default_state[i]);
3032
3033 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3034 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3035
3036 /* set clear context state */
3037 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3038 radeon_ring_write(ring, 0);
3039
3040 /* SQ_VTX_BASE_VTX_LOC */
3041 radeon_ring_write(ring, 0xc0026f00);
3042 radeon_ring_write(ring, 0x00000000);
3043 radeon_ring_write(ring, 0x00000000);
3044 radeon_ring_write(ring, 0x00000000);
3045
3046 /* Clear consts */
3047 radeon_ring_write(ring, 0xc0036f00);
3048 radeon_ring_write(ring, 0x00000bc4);
3049 radeon_ring_write(ring, 0xffffffff);
3050 radeon_ring_write(ring, 0xffffffff);
3051 radeon_ring_write(ring, 0xffffffff);
3052
3053 radeon_ring_write(ring, 0xc0026900);
3054 radeon_ring_write(ring, 0x00000316);
3055 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3056 radeon_ring_write(ring, 0x00000010); /* */
3057
3058 radeon_ring_unlock_commit(rdev, ring, false);
3059
3060 return 0;
3061}
3062
3063static int evergreen_cp_resume(struct radeon_device *rdev)
3064{
3065 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3066 u32 tmp;
3067 u32 rb_bufsz;
3068 int r;
3069
3070 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3071 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3072 SOFT_RESET_PA |
3073 SOFT_RESET_SH |
3074 SOFT_RESET_VGT |
3075 SOFT_RESET_SPI |
3076 SOFT_RESET_SX));
3077 RREG32(GRBM_SOFT_RESET);
3078 mdelay(15);
3079 WREG32(GRBM_SOFT_RESET, 0);
3080 RREG32(GRBM_SOFT_RESET);
3081
3082 /* Set ring buffer size */
3083 rb_bufsz = order_base_2(ring->ring_size / 8);
3084 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3085#ifdef __BIG_ENDIAN
3086 tmp |= BUF_SWAP_32BIT;
3087#endif
3088 WREG32(CP_RB_CNTL, tmp);
3089 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3090 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3091
3092 /* Set the write pointer delay */
3093 WREG32(CP_RB_WPTR_DELAY, 0);
3094
3095 /* Initialize the ring buffer's read and write pointers */
3096 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3097 WREG32(CP_RB_RPTR_WR, 0);
3098 ring->wptr = 0;
3099 WREG32(CP_RB_WPTR, ring->wptr);
3100
3101 /* set the wb address whether it's enabled or not */
3102 WREG32(CP_RB_RPTR_ADDR,
3103 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3104 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3105 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3106
3107 if (rdev->wb.enabled)
3108 WREG32(SCRATCH_UMSK, 0xff);
3109 else {
3110 tmp |= RB_NO_UPDATE;
3111 WREG32(SCRATCH_UMSK, 0);
3112 }
3113
3114 mdelay(1);
3115 WREG32(CP_RB_CNTL, tmp);
3116
3117 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3118 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3119
3120 evergreen_cp_start(rdev);
3121 ring->ready = true;
3122 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3123 if (r) {
3124 ring->ready = false;
3125 return r;
3126 }
3127 return 0;
3128}
3129
3130/*
3131 * Core functions
3132 */
3133static void evergreen_gpu_init(struct radeon_device *rdev)
3134{
3135 u32 gb_addr_config;
3136 u32 mc_shared_chmap, mc_arb_ramcfg;
3137 u32 sx_debug_1;
3138 u32 smx_dc_ctl0;
3139 u32 sq_config;
3140 u32 sq_lds_resource_mgmt;
3141 u32 sq_gpr_resource_mgmt_1;
3142 u32 sq_gpr_resource_mgmt_2;
3143 u32 sq_gpr_resource_mgmt_3;
3144 u32 sq_thread_resource_mgmt;
3145 u32 sq_thread_resource_mgmt_2;
3146 u32 sq_stack_resource_mgmt_1;
3147 u32 sq_stack_resource_mgmt_2;
3148 u32 sq_stack_resource_mgmt_3;
3149 u32 vgt_cache_invalidation;
3150 u32 hdp_host_path_cntl, tmp;
3151 u32 disabled_rb_mask;
3152 int i, j, ps_thread_count;
3153
3154 switch (rdev->family) {
3155 case CHIP_CYPRESS:
3156 case CHIP_HEMLOCK:
3157 rdev->config.evergreen.num_ses = 2;
3158 rdev->config.evergreen.max_pipes = 4;
3159 rdev->config.evergreen.max_tile_pipes = 8;
3160 rdev->config.evergreen.max_simds = 10;
3161 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3162 rdev->config.evergreen.max_gprs = 256;
3163 rdev->config.evergreen.max_threads = 248;
3164 rdev->config.evergreen.max_gs_threads = 32;
3165 rdev->config.evergreen.max_stack_entries = 512;
3166 rdev->config.evergreen.sx_num_of_sets = 4;
3167 rdev->config.evergreen.sx_max_export_size = 256;
3168 rdev->config.evergreen.sx_max_export_pos_size = 64;
3169 rdev->config.evergreen.sx_max_export_smx_size = 192;
3170 rdev->config.evergreen.max_hw_contexts = 8;
3171 rdev->config.evergreen.sq_num_cf_insts = 2;
3172
3173 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3174 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3175 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3176 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3177 break;
3178 case CHIP_JUNIPER:
3179 rdev->config.evergreen.num_ses = 1;
3180 rdev->config.evergreen.max_pipes = 4;
3181 rdev->config.evergreen.max_tile_pipes = 4;
3182 rdev->config.evergreen.max_simds = 10;
3183 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3184 rdev->config.evergreen.max_gprs = 256;
3185 rdev->config.evergreen.max_threads = 248;
3186 rdev->config.evergreen.max_gs_threads = 32;
3187 rdev->config.evergreen.max_stack_entries = 512;
3188 rdev->config.evergreen.sx_num_of_sets = 4;
3189 rdev->config.evergreen.sx_max_export_size = 256;
3190 rdev->config.evergreen.sx_max_export_pos_size = 64;
3191 rdev->config.evergreen.sx_max_export_smx_size = 192;
3192 rdev->config.evergreen.max_hw_contexts = 8;
3193 rdev->config.evergreen.sq_num_cf_insts = 2;
3194
3195 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3196 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3197 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3198 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3199 break;
3200 case CHIP_REDWOOD:
3201 rdev->config.evergreen.num_ses = 1;
3202 rdev->config.evergreen.max_pipes = 4;
3203 rdev->config.evergreen.max_tile_pipes = 4;
3204 rdev->config.evergreen.max_simds = 5;
3205 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3206 rdev->config.evergreen.max_gprs = 256;
3207 rdev->config.evergreen.max_threads = 248;
3208 rdev->config.evergreen.max_gs_threads = 32;
3209 rdev->config.evergreen.max_stack_entries = 256;
3210 rdev->config.evergreen.sx_num_of_sets = 4;
3211 rdev->config.evergreen.sx_max_export_size = 256;
3212 rdev->config.evergreen.sx_max_export_pos_size = 64;
3213 rdev->config.evergreen.sx_max_export_smx_size = 192;
3214 rdev->config.evergreen.max_hw_contexts = 8;
3215 rdev->config.evergreen.sq_num_cf_insts = 2;
3216
3217 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3218 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3219 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3220 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3221 break;
3222 case CHIP_CEDAR:
3223 default:
3224 rdev->config.evergreen.num_ses = 1;
3225 rdev->config.evergreen.max_pipes = 2;
3226 rdev->config.evergreen.max_tile_pipes = 2;
3227 rdev->config.evergreen.max_simds = 2;
3228 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3229 rdev->config.evergreen.max_gprs = 256;
3230 rdev->config.evergreen.max_threads = 192;
3231 rdev->config.evergreen.max_gs_threads = 16;
3232 rdev->config.evergreen.max_stack_entries = 256;
3233 rdev->config.evergreen.sx_num_of_sets = 4;
3234 rdev->config.evergreen.sx_max_export_size = 128;
3235 rdev->config.evergreen.sx_max_export_pos_size = 32;
3236 rdev->config.evergreen.sx_max_export_smx_size = 96;
3237 rdev->config.evergreen.max_hw_contexts = 4;
3238 rdev->config.evergreen.sq_num_cf_insts = 1;
3239
3240 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3241 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3242 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3243 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3244 break;
3245 case CHIP_PALM:
3246 rdev->config.evergreen.num_ses = 1;
3247 rdev->config.evergreen.max_pipes = 2;
3248 rdev->config.evergreen.max_tile_pipes = 2;
3249 rdev->config.evergreen.max_simds = 2;
3250 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3251 rdev->config.evergreen.max_gprs = 256;
3252 rdev->config.evergreen.max_threads = 192;
3253 rdev->config.evergreen.max_gs_threads = 16;
3254 rdev->config.evergreen.max_stack_entries = 256;
3255 rdev->config.evergreen.sx_num_of_sets = 4;
3256 rdev->config.evergreen.sx_max_export_size = 128;
3257 rdev->config.evergreen.sx_max_export_pos_size = 32;
3258 rdev->config.evergreen.sx_max_export_smx_size = 96;
3259 rdev->config.evergreen.max_hw_contexts = 4;
3260 rdev->config.evergreen.sq_num_cf_insts = 1;
3261
3262 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3263 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3264 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3265 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3266 break;
3267 case CHIP_SUMO:
3268 rdev->config.evergreen.num_ses = 1;
3269 rdev->config.evergreen.max_pipes = 4;
3270 rdev->config.evergreen.max_tile_pipes = 4;
3271 if (rdev->pdev->device == 0x9648)
3272 rdev->config.evergreen.max_simds = 3;
3273 else if ((rdev->pdev->device == 0x9647) ||
3274 (rdev->pdev->device == 0x964a))
3275 rdev->config.evergreen.max_simds = 4;
3276 else
3277 rdev->config.evergreen.max_simds = 5;
3278 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3279 rdev->config.evergreen.max_gprs = 256;
3280 rdev->config.evergreen.max_threads = 248;
3281 rdev->config.evergreen.max_gs_threads = 32;
3282 rdev->config.evergreen.max_stack_entries = 256;
3283 rdev->config.evergreen.sx_num_of_sets = 4;
3284 rdev->config.evergreen.sx_max_export_size = 256;
3285 rdev->config.evergreen.sx_max_export_pos_size = 64;
3286 rdev->config.evergreen.sx_max_export_smx_size = 192;
3287 rdev->config.evergreen.max_hw_contexts = 8;
3288 rdev->config.evergreen.sq_num_cf_insts = 2;
3289
3290 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3291 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3292 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3293 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3294 break;
3295 case CHIP_SUMO2:
3296 rdev->config.evergreen.num_ses = 1;
3297 rdev->config.evergreen.max_pipes = 4;
3298 rdev->config.evergreen.max_tile_pipes = 4;
3299 rdev->config.evergreen.max_simds = 2;
3300 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3301 rdev->config.evergreen.max_gprs = 256;
3302 rdev->config.evergreen.max_threads = 248;
3303 rdev->config.evergreen.max_gs_threads = 32;
3304 rdev->config.evergreen.max_stack_entries = 512;
3305 rdev->config.evergreen.sx_num_of_sets = 4;
3306 rdev->config.evergreen.sx_max_export_size = 256;
3307 rdev->config.evergreen.sx_max_export_pos_size = 64;
3308 rdev->config.evergreen.sx_max_export_smx_size = 192;
3309 rdev->config.evergreen.max_hw_contexts = 4;
3310 rdev->config.evergreen.sq_num_cf_insts = 2;
3311
3312 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3313 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3314 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3315 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3316 break;
3317 case CHIP_BARTS:
3318 rdev->config.evergreen.num_ses = 2;
3319 rdev->config.evergreen.max_pipes = 4;
3320 rdev->config.evergreen.max_tile_pipes = 8;
3321 rdev->config.evergreen.max_simds = 7;
3322 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3323 rdev->config.evergreen.max_gprs = 256;
3324 rdev->config.evergreen.max_threads = 248;
3325 rdev->config.evergreen.max_gs_threads = 32;
3326 rdev->config.evergreen.max_stack_entries = 512;
3327 rdev->config.evergreen.sx_num_of_sets = 4;
3328 rdev->config.evergreen.sx_max_export_size = 256;
3329 rdev->config.evergreen.sx_max_export_pos_size = 64;
3330 rdev->config.evergreen.sx_max_export_smx_size = 192;
3331 rdev->config.evergreen.max_hw_contexts = 8;
3332 rdev->config.evergreen.sq_num_cf_insts = 2;
3333
3334 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3335 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3336 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3337 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3338 break;
3339 case CHIP_TURKS:
3340 rdev->config.evergreen.num_ses = 1;
3341 rdev->config.evergreen.max_pipes = 4;
3342 rdev->config.evergreen.max_tile_pipes = 4;
3343 rdev->config.evergreen.max_simds = 6;
3344 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3345 rdev->config.evergreen.max_gprs = 256;
3346 rdev->config.evergreen.max_threads = 248;
3347 rdev->config.evergreen.max_gs_threads = 32;
3348 rdev->config.evergreen.max_stack_entries = 256;
3349 rdev->config.evergreen.sx_num_of_sets = 4;
3350 rdev->config.evergreen.sx_max_export_size = 256;
3351 rdev->config.evergreen.sx_max_export_pos_size = 64;
3352 rdev->config.evergreen.sx_max_export_smx_size = 192;
3353 rdev->config.evergreen.max_hw_contexts = 8;
3354 rdev->config.evergreen.sq_num_cf_insts = 2;
3355
3356 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3357 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3358 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3359 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3360 break;
3361 case CHIP_CAICOS:
3362 rdev->config.evergreen.num_ses = 1;
3363 rdev->config.evergreen.max_pipes = 2;
3364 rdev->config.evergreen.max_tile_pipes = 2;
3365 rdev->config.evergreen.max_simds = 2;
3366 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3367 rdev->config.evergreen.max_gprs = 256;
3368 rdev->config.evergreen.max_threads = 192;
3369 rdev->config.evergreen.max_gs_threads = 16;
3370 rdev->config.evergreen.max_stack_entries = 256;
3371 rdev->config.evergreen.sx_num_of_sets = 4;
3372 rdev->config.evergreen.sx_max_export_size = 128;
3373 rdev->config.evergreen.sx_max_export_pos_size = 32;
3374 rdev->config.evergreen.sx_max_export_smx_size = 96;
3375 rdev->config.evergreen.max_hw_contexts = 4;
3376 rdev->config.evergreen.sq_num_cf_insts = 1;
3377
3378 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3379 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3380 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3381 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3382 break;
3383 }
3384
3385 /* Initialize HDP */
3386 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3387 WREG32((0x2c14 + j), 0x00000000);
3388 WREG32((0x2c18 + j), 0x00000000);
3389 WREG32((0x2c1c + j), 0x00000000);
3390 WREG32((0x2c20 + j), 0x00000000);
3391 WREG32((0x2c24 + j), 0x00000000);
3392 }
3393
3394 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3395 WREG32(SRBM_INT_CNTL, 0x1);
3396 WREG32(SRBM_INT_ACK, 0x1);
3397
3398 evergreen_fix_pci_max_read_req_size(rdev);
3399
3400 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3401 if ((rdev->family == CHIP_PALM) ||
3402 (rdev->family == CHIP_SUMO) ||
3403 (rdev->family == CHIP_SUMO2))
3404 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3405 else
3406 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3407
3408 /* setup tiling info dword. gb_addr_config is not adequate since it does
3409 * not have bank info, so create a custom tiling dword.
3410 * bits 3:0 num_pipes
3411 * bits 7:4 num_banks
3412 * bits 11:8 group_size
3413 * bits 15:12 row_size
3414 */
3415 rdev->config.evergreen.tile_config = 0;
3416 switch (rdev->config.evergreen.max_tile_pipes) {
3417 case 1:
3418 default:
3419 rdev->config.evergreen.tile_config |= (0 << 0);
3420 break;
3421 case 2:
3422 rdev->config.evergreen.tile_config |= (1 << 0);
3423 break;
3424 case 4:
3425 rdev->config.evergreen.tile_config |= (2 << 0);
3426 break;
3427 case 8:
3428 rdev->config.evergreen.tile_config |= (3 << 0);
3429 break;
3430 }
3431 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3432 if (rdev->flags & RADEON_IS_IGP)
3433 rdev->config.evergreen.tile_config |= 1 << 4;
3434 else {
3435 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3436 case 0: /* four banks */
3437 rdev->config.evergreen.tile_config |= 0 << 4;
3438 break;
3439 case 1: /* eight banks */
3440 rdev->config.evergreen.tile_config |= 1 << 4;
3441 break;
3442 case 2: /* sixteen banks */
3443 default:
3444 rdev->config.evergreen.tile_config |= 2 << 4;
3445 break;
3446 }
3447 }
3448 rdev->config.evergreen.tile_config |= 0 << 8;
3449 rdev->config.evergreen.tile_config |=
3450 ((gb_addr_config & 0x30000000) >> 28) << 12;
3451
3452 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3453 u32 efuse_straps_4;
3454 u32 efuse_straps_3;
3455
3456 efuse_straps_4 = RREG32_RCU(0x204);
3457 efuse_straps_3 = RREG32_RCU(0x203);
3458 tmp = (((efuse_straps_4 & 0xf) << 4) |
3459 ((efuse_straps_3 & 0xf0000000) >> 28));
3460 } else {
3461 tmp = 0;
3462 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3463 u32 rb_disable_bitmap;
3464
3465 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3467 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3468 tmp <<= 4;
3469 tmp |= rb_disable_bitmap;
3470 }
3471 }
3472 /* enabled rb are just the one not disabled :) */
3473 disabled_rb_mask = tmp;
3474 tmp = 0;
3475 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3476 tmp |= (1 << i);
3477 /* if all the backends are disabled, fix it up here */
3478 if ((disabled_rb_mask & tmp) == tmp) {
3479 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3480 disabled_rb_mask &= ~(1 << i);
3481 }
3482
3483 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3484 u32 simd_disable_bitmap;
3485
3486 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3488 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3489 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3490 tmp <<= 16;
3491 tmp |= simd_disable_bitmap;
3492 }
3493 rdev->config.evergreen.active_simds = hweight32(~tmp);
3494
3495 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3496 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497
3498 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3499 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3500 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3501 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3502 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3503 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3504 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3505
3506 if ((rdev->config.evergreen.max_backends == 1) &&
3507 (rdev->flags & RADEON_IS_IGP)) {
3508 if ((disabled_rb_mask & 3) == 1) {
3509 /* RB0 disabled, RB1 enabled */
3510 tmp = 0x11111111;
3511 } else {
3512 /* RB1 disabled, RB0 enabled */
3513 tmp = 0x00000000;
3514 }
3515 } else {
3516 tmp = gb_addr_config & NUM_PIPES_MASK;
3517 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3518 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3519 }
3520 rdev->config.evergreen.backend_map = tmp;
3521 WREG32(GB_BACKEND_MAP, tmp);
3522
3523 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3524 WREG32(CGTS_TCC_DISABLE, 0);
3525 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3526 WREG32(CGTS_USER_TCC_DISABLE, 0);
3527
3528 /* set HW defaults for 3D engine */
3529 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3530 ROQ_IB2_START(0x2b)));
3531
3532 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3533
3534 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3535 SYNC_GRADIENT |
3536 SYNC_WALKER |
3537 SYNC_ALIGNER));
3538
3539 sx_debug_1 = RREG32(SX_DEBUG_1);
3540 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3541 WREG32(SX_DEBUG_1, sx_debug_1);
3542
3543
3544 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3545 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3546 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3547 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3548
3549 if (rdev->family <= CHIP_SUMO2)
3550 WREG32(SMX_SAR_CTL0, 0x00010000);
3551
3552 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3553 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3554 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3555
3556 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3557 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3558 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3559
3560 WREG32(VGT_NUM_INSTANCES, 1);
3561 WREG32(SPI_CONFIG_CNTL, 0);
3562 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3563 WREG32(CP_PERFMON_CNTL, 0);
3564
3565 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3566 FETCH_FIFO_HIWATER(0x4) |
3567 DONE_FIFO_HIWATER(0xe0) |
3568 ALU_UPDATE_FIFO_HIWATER(0x8)));
3569
3570 sq_config = RREG32(SQ_CONFIG);
3571 sq_config &= ~(PS_PRIO(3) |
3572 VS_PRIO(3) |
3573 GS_PRIO(3) |
3574 ES_PRIO(3));
3575 sq_config |= (VC_ENABLE |
3576 EXPORT_SRC_C |
3577 PS_PRIO(0) |
3578 VS_PRIO(1) |
3579 GS_PRIO(2) |
3580 ES_PRIO(3));
3581
3582 switch (rdev->family) {
3583 case CHIP_CEDAR:
3584 case CHIP_PALM:
3585 case CHIP_SUMO:
3586 case CHIP_SUMO2:
3587 case CHIP_CAICOS:
3588 /* no vertex cache */
3589 sq_config &= ~VC_ENABLE;
3590 break;
3591 default:
3592 break;
3593 }
3594
3595 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3596
3597 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3598 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3599 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3600 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3602 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3603 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604
3605 switch (rdev->family) {
3606 case CHIP_CEDAR:
3607 case CHIP_PALM:
3608 case CHIP_SUMO:
3609 case CHIP_SUMO2:
3610 ps_thread_count = 96;
3611 break;
3612 default:
3613 ps_thread_count = 128;
3614 break;
3615 }
3616
3617 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3618 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623
3624 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630
3631 WREG32(SQ_CONFIG, sq_config);
3632 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3633 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3634 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3635 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3636 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3637 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3638 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3639 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3640 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3641 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3642
3643 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3644 FORCE_EOV_MAX_REZ_CNT(255)));
3645
3646 switch (rdev->family) {
3647 case CHIP_CEDAR:
3648 case CHIP_PALM:
3649 case CHIP_SUMO:
3650 case CHIP_SUMO2:
3651 case CHIP_CAICOS:
3652 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3653 break;
3654 default:
3655 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3656 break;
3657 }
3658 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3659 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3660
3661 WREG32(VGT_GS_VERTEX_REUSE, 16);
3662 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3663 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3664
3665 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3666 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3667
3668 WREG32(CB_PERF_CTR0_SEL_0, 0);
3669 WREG32(CB_PERF_CTR0_SEL_1, 0);
3670 WREG32(CB_PERF_CTR1_SEL_0, 0);
3671 WREG32(CB_PERF_CTR1_SEL_1, 0);
3672 WREG32(CB_PERF_CTR2_SEL_0, 0);
3673 WREG32(CB_PERF_CTR2_SEL_1, 0);
3674 WREG32(CB_PERF_CTR3_SEL_0, 0);
3675 WREG32(CB_PERF_CTR3_SEL_1, 0);
3676
3677 /* clear render buffer base addresses */
3678 WREG32(CB_COLOR0_BASE, 0);
3679 WREG32(CB_COLOR1_BASE, 0);
3680 WREG32(CB_COLOR2_BASE, 0);
3681 WREG32(CB_COLOR3_BASE, 0);
3682 WREG32(CB_COLOR4_BASE, 0);
3683 WREG32(CB_COLOR5_BASE, 0);
3684 WREG32(CB_COLOR6_BASE, 0);
3685 WREG32(CB_COLOR7_BASE, 0);
3686 WREG32(CB_COLOR8_BASE, 0);
3687 WREG32(CB_COLOR9_BASE, 0);
3688 WREG32(CB_COLOR10_BASE, 0);
3689 WREG32(CB_COLOR11_BASE, 0);
3690
3691 /* set the shader const cache sizes to 0 */
3692 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3693 WREG32(i, 0);
3694 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3695 WREG32(i, 0);
3696
3697 tmp = RREG32(HDP_MISC_CNTL);
3698 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3699 WREG32(HDP_MISC_CNTL, tmp);
3700
3701 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3702 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3703
3704 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3705
3706 udelay(50);
3707
3708}
3709
3710int evergreen_mc_init(struct radeon_device *rdev)
3711{
3712 u32 tmp;
3713 int chansize, numchan;
3714
3715 /* Get VRAM informations */
3716 rdev->mc.vram_is_ddr = true;
3717 if ((rdev->family == CHIP_PALM) ||
3718 (rdev->family == CHIP_SUMO) ||
3719 (rdev->family == CHIP_SUMO2))
3720 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3721 else
3722 tmp = RREG32(MC_ARB_RAMCFG);
3723 if (tmp & CHANSIZE_OVERRIDE) {
3724 chansize = 16;
3725 } else if (tmp & CHANSIZE_MASK) {
3726 chansize = 64;
3727 } else {
3728 chansize = 32;
3729 }
3730 tmp = RREG32(MC_SHARED_CHMAP);
3731 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3732 case 0:
3733 default:
3734 numchan = 1;
3735 break;
3736 case 1:
3737 numchan = 2;
3738 break;
3739 case 2:
3740 numchan = 4;
3741 break;
3742 case 3:
3743 numchan = 8;
3744 break;
3745 }
3746 rdev->mc.vram_width = numchan * chansize;
3747 /* Could aper size report 0 ? */
3748 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3749 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3750 /* Setup GPU memory space */
3751 if ((rdev->family == CHIP_PALM) ||
3752 (rdev->family == CHIP_SUMO) ||
3753 (rdev->family == CHIP_SUMO2)) {
3754 /* size in bytes on fusion */
3755 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3756 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3757 } else {
3758 /* size in MB on evergreen/cayman/tn */
3759 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3760 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761 }
3762 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3763 r700_vram_gtt_location(rdev, &rdev->mc);
3764 radeon_update_bandwidth_info(rdev);
3765
3766 return 0;
3767}
3768
3769void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3770{
3771 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3772 RREG32(GRBM_STATUS));
3773 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3774 RREG32(GRBM_STATUS_SE0));
3775 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3776 RREG32(GRBM_STATUS_SE1));
3777 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3778 RREG32(SRBM_STATUS));
3779 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3780 RREG32(SRBM_STATUS2));
3781 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3782 RREG32(CP_STALLED_STAT1));
3783 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3784 RREG32(CP_STALLED_STAT2));
3785 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3786 RREG32(CP_BUSY_STAT));
3787 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3788 RREG32(CP_STAT));
3789 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3790 RREG32(DMA_STATUS_REG));
3791 if (rdev->family >= CHIP_CAYMAN) {
3792 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3793 RREG32(DMA_STATUS_REG + 0x800));
3794 }
3795}
3796
3797bool evergreen_is_display_hung(struct radeon_device *rdev)
3798{
3799 u32 crtc_hung = 0;
3800 u32 crtc_status[6];
3801 u32 i, j, tmp;
3802
3803 for (i = 0; i < rdev->num_crtc; i++) {
3804 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3805 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3806 crtc_hung |= (1 << i);
3807 }
3808 }
3809
3810 for (j = 0; j < 10; j++) {
3811 for (i = 0; i < rdev->num_crtc; i++) {
3812 if (crtc_hung & (1 << i)) {
3813 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3814 if (tmp != crtc_status[i])
3815 crtc_hung &= ~(1 << i);
3816 }
3817 }
3818 if (crtc_hung == 0)
3819 return false;
3820 udelay(100);
3821 }
3822
3823 return true;
3824}
3825
3826u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3827{
3828 u32 reset_mask = 0;
3829 u32 tmp;
3830
3831 /* GRBM_STATUS */
3832 tmp = RREG32(GRBM_STATUS);
3833 if (tmp & (PA_BUSY | SC_BUSY |
3834 SH_BUSY | SX_BUSY |
3835 TA_BUSY | VGT_BUSY |
3836 DB_BUSY | CB_BUSY |
3837 SPI_BUSY | VGT_BUSY_NO_DMA))
3838 reset_mask |= RADEON_RESET_GFX;
3839
3840 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3841 CP_BUSY | CP_COHERENCY_BUSY))
3842 reset_mask |= RADEON_RESET_CP;
3843
3844 if (tmp & GRBM_EE_BUSY)
3845 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3846
3847 /* DMA_STATUS_REG */
3848 tmp = RREG32(DMA_STATUS_REG);
3849 if (!(tmp & DMA_IDLE))
3850 reset_mask |= RADEON_RESET_DMA;
3851
3852 /* SRBM_STATUS2 */
3853 tmp = RREG32(SRBM_STATUS2);
3854 if (tmp & DMA_BUSY)
3855 reset_mask |= RADEON_RESET_DMA;
3856
3857 /* SRBM_STATUS */
3858 tmp = RREG32(SRBM_STATUS);
3859 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3860 reset_mask |= RADEON_RESET_RLC;
3861
3862 if (tmp & IH_BUSY)
3863 reset_mask |= RADEON_RESET_IH;
3864
3865 if (tmp & SEM_BUSY)
3866 reset_mask |= RADEON_RESET_SEM;
3867
3868 if (tmp & GRBM_RQ_PENDING)
3869 reset_mask |= RADEON_RESET_GRBM;
3870
3871 if (tmp & VMC_BUSY)
3872 reset_mask |= RADEON_RESET_VMC;
3873
3874 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3875 MCC_BUSY | MCD_BUSY))
3876 reset_mask |= RADEON_RESET_MC;
3877
3878 if (evergreen_is_display_hung(rdev))
3879 reset_mask |= RADEON_RESET_DISPLAY;
3880
3881 /* VM_L2_STATUS */
3882 tmp = RREG32(VM_L2_STATUS);
3883 if (tmp & L2_BUSY)
3884 reset_mask |= RADEON_RESET_VMC;
3885
3886 /* Skip MC reset as it's mostly likely not hung, just busy */
3887 if (reset_mask & RADEON_RESET_MC) {
3888 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3889 reset_mask &= ~RADEON_RESET_MC;
3890 }
3891
3892 return reset_mask;
3893}
3894
3895static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3896{
3897 struct evergreen_mc_save save;
3898 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3899 u32 tmp;
3900
3901 if (reset_mask == 0)
3902 return;
3903
3904 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3905
3906 evergreen_print_gpu_status_regs(rdev);
3907
3908 /* Disable CP parsing/prefetching */
3909 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3910
3911 if (reset_mask & RADEON_RESET_DMA) {
3912 /* Disable DMA */
3913 tmp = RREG32(DMA_RB_CNTL);
3914 tmp &= ~DMA_RB_ENABLE;
3915 WREG32(DMA_RB_CNTL, tmp);
3916 }
3917
3918 udelay(50);
3919
3920 evergreen_mc_stop(rdev, &save);
3921 if (evergreen_mc_wait_for_idle(rdev)) {
3922 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3923 }
3924
3925 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3926 grbm_soft_reset |= SOFT_RESET_DB |
3927 SOFT_RESET_CB |
3928 SOFT_RESET_PA |
3929 SOFT_RESET_SC |
3930 SOFT_RESET_SPI |
3931 SOFT_RESET_SX |
3932 SOFT_RESET_SH |
3933 SOFT_RESET_TC |
3934 SOFT_RESET_TA |
3935 SOFT_RESET_VC |
3936 SOFT_RESET_VGT;
3937 }
3938
3939 if (reset_mask & RADEON_RESET_CP) {
3940 grbm_soft_reset |= SOFT_RESET_CP |
3941 SOFT_RESET_VGT;
3942
3943 srbm_soft_reset |= SOFT_RESET_GRBM;
3944 }
3945
3946 if (reset_mask & RADEON_RESET_DMA)
3947 srbm_soft_reset |= SOFT_RESET_DMA;
3948
3949 if (reset_mask & RADEON_RESET_DISPLAY)
3950 srbm_soft_reset |= SOFT_RESET_DC;
3951
3952 if (reset_mask & RADEON_RESET_RLC)
3953 srbm_soft_reset |= SOFT_RESET_RLC;
3954
3955 if (reset_mask & RADEON_RESET_SEM)
3956 srbm_soft_reset |= SOFT_RESET_SEM;
3957
3958 if (reset_mask & RADEON_RESET_IH)
3959 srbm_soft_reset |= SOFT_RESET_IH;
3960
3961 if (reset_mask & RADEON_RESET_GRBM)
3962 srbm_soft_reset |= SOFT_RESET_GRBM;
3963
3964 if (reset_mask & RADEON_RESET_VMC)
3965 srbm_soft_reset |= SOFT_RESET_VMC;
3966
3967 if (!(rdev->flags & RADEON_IS_IGP)) {
3968 if (reset_mask & RADEON_RESET_MC)
3969 srbm_soft_reset |= SOFT_RESET_MC;
3970 }
3971
3972 if (grbm_soft_reset) {
3973 tmp = RREG32(GRBM_SOFT_RESET);
3974 tmp |= grbm_soft_reset;
3975 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3976 WREG32(GRBM_SOFT_RESET, tmp);
3977 tmp = RREG32(GRBM_SOFT_RESET);
3978
3979 udelay(50);
3980
3981 tmp &= ~grbm_soft_reset;
3982 WREG32(GRBM_SOFT_RESET, tmp);
3983 tmp = RREG32(GRBM_SOFT_RESET);
3984 }
3985
3986 if (srbm_soft_reset) {
3987 tmp = RREG32(SRBM_SOFT_RESET);
3988 tmp |= srbm_soft_reset;
3989 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3990 WREG32(SRBM_SOFT_RESET, tmp);
3991 tmp = RREG32(SRBM_SOFT_RESET);
3992
3993 udelay(50);
3994
3995 tmp &= ~srbm_soft_reset;
3996 WREG32(SRBM_SOFT_RESET, tmp);
3997 tmp = RREG32(SRBM_SOFT_RESET);
3998 }
3999
4000 /* Wait a little for things to settle down */
4001 udelay(50);
4002
4003 evergreen_mc_resume(rdev, &save);
4004 udelay(50);
4005
4006 evergreen_print_gpu_status_regs(rdev);
4007}
4008
4009void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4010{
4011 struct evergreen_mc_save save;
4012 u32 tmp, i;
4013
4014 dev_info(rdev->dev, "GPU pci config reset\n");
4015
4016 /* disable dpm? */
4017
4018 /* Disable CP parsing/prefetching */
4019 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4020 udelay(50);
4021 /* Disable DMA */
4022 tmp = RREG32(DMA_RB_CNTL);
4023 tmp &= ~DMA_RB_ENABLE;
4024 WREG32(DMA_RB_CNTL, tmp);
4025 /* XXX other engines? */
4026
4027 /* halt the rlc */
4028 r600_rlc_stop(rdev);
4029
4030 udelay(50);
4031
4032 /* set mclk/sclk to bypass */
4033 rv770_set_clk_bypass_mode(rdev);
4034 /* disable BM */
4035 pci_clear_master(rdev->pdev);
4036 /* disable mem access */
4037 evergreen_mc_stop(rdev, &save);
4038 if (evergreen_mc_wait_for_idle(rdev)) {
4039 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4040 }
4041 /* reset */
4042 radeon_pci_config_reset(rdev);
4043 /* wait for asic to come out of reset */
4044 for (i = 0; i < rdev->usec_timeout; i++) {
4045 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4046 break;
4047 udelay(1);
4048 }
4049}
4050
4051int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4052{
4053 u32 reset_mask;
4054
4055 if (hard) {
4056 evergreen_gpu_pci_config_reset(rdev);
4057 return 0;
4058 }
4059
4060 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4061
4062 if (reset_mask)
4063 r600_set_bios_scratch_engine_hung(rdev, true);
4064
4065 /* try soft reset */
4066 evergreen_gpu_soft_reset(rdev, reset_mask);
4067
4068 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4069
4070 /* try pci config reset */
4071 if (reset_mask && radeon_hard_reset)
4072 evergreen_gpu_pci_config_reset(rdev);
4073
4074 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4075
4076 if (!reset_mask)
4077 r600_set_bios_scratch_engine_hung(rdev, false);
4078
4079 return 0;
4080}
4081
4082/**
4083 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4084 *
4085 * @rdev: radeon_device pointer
4086 * @ring: radeon_ring structure holding ring information
4087 *
4088 * Check if the GFX engine is locked up.
4089 * Returns true if the engine appears to be locked up, false if not.
4090 */
4091bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4092{
4093 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4094
4095 if (!(reset_mask & (RADEON_RESET_GFX |
4096 RADEON_RESET_COMPUTE |
4097 RADEON_RESET_CP))) {
4098 radeon_ring_lockup_update(rdev, ring);
4099 return false;
4100 }
4101 return radeon_ring_test_lockup(rdev, ring);
4102}
4103
4104/*
4105 * RLC
4106 */
4107#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4108#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4109
4110void sumo_rlc_fini(struct radeon_device *rdev)
4111{
4112 int r;
4113
4114 /* save restore block */
4115 if (rdev->rlc.save_restore_obj) {
4116 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4117 if (unlikely(r != 0))
4118 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4119 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4120 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4121
4122 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4123 rdev->rlc.save_restore_obj = NULL;
4124 }
4125
4126 /* clear state block */
4127 if (rdev->rlc.clear_state_obj) {
4128 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4129 if (unlikely(r != 0))
4130 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4131 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4132 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4133
4134 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4135 rdev->rlc.clear_state_obj = NULL;
4136 }
4137
4138 /* clear state block */
4139 if (rdev->rlc.cp_table_obj) {
4140 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4141 if (unlikely(r != 0))
4142 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4143 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4144 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4145
4146 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4147 rdev->rlc.cp_table_obj = NULL;
4148 }
4149}
4150
4151#define CP_ME_TABLE_SIZE 96
4152
4153int sumo_rlc_init(struct radeon_device *rdev)
4154{
4155 const u32 *src_ptr;
4156 volatile u32 *dst_ptr;
4157 u32 dws, data, i, j, k, reg_num;
4158 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4159 u64 reg_list_mc_addr;
4160 const struct cs_section_def *cs_data;
4161 int r;
4162
4163 src_ptr = rdev->rlc.reg_list;
4164 dws = rdev->rlc.reg_list_size;
4165 if (rdev->family >= CHIP_BONAIRE) {
4166 dws += (5 * 16) + 48 + 48 + 64;
4167 }
4168 cs_data = rdev->rlc.cs_data;
4169
4170 if (src_ptr) {
4171 /* save restore block */
4172 if (rdev->rlc.save_restore_obj == NULL) {
4173 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4174 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4175 NULL, &rdev->rlc.save_restore_obj);
4176 if (r) {
4177 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4178 return r;
4179 }
4180 }
4181
4182 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4183 if (unlikely(r != 0)) {
4184 sumo_rlc_fini(rdev);
4185 return r;
4186 }
4187 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4188 &rdev->rlc.save_restore_gpu_addr);
4189 if (r) {
4190 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4191 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4192 sumo_rlc_fini(rdev);
4193 return r;
4194 }
4195
4196 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4197 if (r) {
4198 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4199 sumo_rlc_fini(rdev);
4200 return r;
4201 }
4202 /* write the sr buffer */
4203 dst_ptr = rdev->rlc.sr_ptr;
4204 if (rdev->family >= CHIP_TAHITI) {
4205 /* SI */
4206 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4207 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4208 } else {
4209 /* ON/LN/TN */
4210 /* format:
4211 * dw0: (reg2 << 16) | reg1
4212 * dw1: reg1 save space
4213 * dw2: reg2 save space
4214 */
4215 for (i = 0; i < dws; i++) {
4216 data = src_ptr[i] >> 2;
4217 i++;
4218 if (i < dws)
4219 data |= (src_ptr[i] >> 2) << 16;
4220 j = (((i - 1) * 3) / 2);
4221 dst_ptr[j] = cpu_to_le32(data);
4222 }
4223 j = ((i * 3) / 2);
4224 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4225 }
4226 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4227 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4228 }
4229
4230 if (cs_data) {
4231 /* clear state block */
4232 if (rdev->family >= CHIP_BONAIRE) {
4233 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4234 } else if (rdev->family >= CHIP_TAHITI) {
4235 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4236 dws = rdev->rlc.clear_state_size + (256 / 4);
4237 } else {
4238 reg_list_num = 0;
4239 dws = 0;
4240 for (i = 0; cs_data[i].section != NULL; i++) {
4241 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4242 reg_list_num++;
4243 dws += cs_data[i].section[j].reg_count;
4244 }
4245 }
4246 reg_list_blk_index = (3 * reg_list_num + 2);
4247 dws += reg_list_blk_index;
4248 rdev->rlc.clear_state_size = dws;
4249 }
4250
4251 if (rdev->rlc.clear_state_obj == NULL) {
4252 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4253 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4254 NULL, &rdev->rlc.clear_state_obj);
4255 if (r) {
4256 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4257 sumo_rlc_fini(rdev);
4258 return r;
4259 }
4260 }
4261 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4262 if (unlikely(r != 0)) {
4263 sumo_rlc_fini(rdev);
4264 return r;
4265 }
4266 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4267 &rdev->rlc.clear_state_gpu_addr);
4268 if (r) {
4269 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4270 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4271 sumo_rlc_fini(rdev);
4272 return r;
4273 }
4274
4275 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4276 if (r) {
4277 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4278 sumo_rlc_fini(rdev);
4279 return r;
4280 }
4281 /* set up the cs buffer */
4282 dst_ptr = rdev->rlc.cs_ptr;
4283 if (rdev->family >= CHIP_BONAIRE) {
4284 cik_get_csb_buffer(rdev, dst_ptr);
4285 } else if (rdev->family >= CHIP_TAHITI) {
4286 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4287 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4288 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4289 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4290 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4291 } else {
4292 reg_list_hdr_blk_index = 0;
4293 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4294 data = upper_32_bits(reg_list_mc_addr);
4295 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4296 reg_list_hdr_blk_index++;
4297 for (i = 0; cs_data[i].section != NULL; i++) {
4298 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4299 reg_num = cs_data[i].section[j].reg_count;
4300 data = reg_list_mc_addr & 0xffffffff;
4301 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4302 reg_list_hdr_blk_index++;
4303
4304 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4305 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4306 reg_list_hdr_blk_index++;
4307
4308 data = 0x08000000 | (reg_num * 4);
4309 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4310 reg_list_hdr_blk_index++;
4311
4312 for (k = 0; k < reg_num; k++) {
4313 data = cs_data[i].section[j].extent[k];
4314 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4315 }
4316 reg_list_mc_addr += reg_num * 4;
4317 reg_list_blk_index += reg_num;
4318 }
4319 }
4320 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4321 }
4322 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4323 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4324 }
4325
4326 if (rdev->rlc.cp_table_size) {
4327 if (rdev->rlc.cp_table_obj == NULL) {
4328 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4329 PAGE_SIZE, true,
4330 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4331 NULL, &rdev->rlc.cp_table_obj);
4332 if (r) {
4333 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4334 sumo_rlc_fini(rdev);
4335 return r;
4336 }
4337 }
4338
4339 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4340 if (unlikely(r != 0)) {
4341 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4342 sumo_rlc_fini(rdev);
4343 return r;
4344 }
4345 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4346 &rdev->rlc.cp_table_gpu_addr);
4347 if (r) {
4348 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4349 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4350 sumo_rlc_fini(rdev);
4351 return r;
4352 }
4353 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4354 if (r) {
4355 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4356 sumo_rlc_fini(rdev);
4357 return r;
4358 }
4359
4360 cik_init_cp_pg_table(rdev);
4361
4362 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4363 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4364
4365 }
4366
4367 return 0;
4368}
4369
4370static void evergreen_rlc_start(struct radeon_device *rdev)
4371{
4372 u32 mask = RLC_ENABLE;
4373
4374 if (rdev->flags & RADEON_IS_IGP) {
4375 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4376 }
4377
4378 WREG32(RLC_CNTL, mask);
4379}
4380
4381int evergreen_rlc_resume(struct radeon_device *rdev)
4382{
4383 u32 i;
4384 const __be32 *fw_data;
4385
4386 if (!rdev->rlc_fw)
4387 return -EINVAL;
4388
4389 r600_rlc_stop(rdev);
4390
4391 WREG32(RLC_HB_CNTL, 0);
4392
4393 if (rdev->flags & RADEON_IS_IGP) {
4394 if (rdev->family == CHIP_ARUBA) {
4395 u32 always_on_bitmap =
4396 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4397 /* find out the number of active simds */
4398 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4399 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4400 tmp = hweight32(~tmp);
4401 if (tmp == rdev->config.cayman.max_simds_per_se) {
4402 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4403 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4404 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4405 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4406 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4407 }
4408 } else {
4409 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4410 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4411 }
4412 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4413 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4414 } else {
4415 WREG32(RLC_HB_BASE, 0);
4416 WREG32(RLC_HB_RPTR, 0);
4417 WREG32(RLC_HB_WPTR, 0);
4418 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4419 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4420 }
4421 WREG32(RLC_MC_CNTL, 0);
4422 WREG32(RLC_UCODE_CNTL, 0);
4423
4424 fw_data = (const __be32 *)rdev->rlc_fw->data;
4425 if (rdev->family >= CHIP_ARUBA) {
4426 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4427 WREG32(RLC_UCODE_ADDR, i);
4428 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429 }
4430 } else if (rdev->family >= CHIP_CAYMAN) {
4431 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4432 WREG32(RLC_UCODE_ADDR, i);
4433 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434 }
4435 } else {
4436 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4437 WREG32(RLC_UCODE_ADDR, i);
4438 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4439 }
4440 }
4441 WREG32(RLC_UCODE_ADDR, 0);
4442
4443 evergreen_rlc_start(rdev);
4444
4445 return 0;
4446}
4447
4448/* Interrupts */
4449
4450u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4451{
4452 if (crtc >= rdev->num_crtc)
4453 return 0;
4454 else
4455 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4456}
4457
4458void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4459{
4460 int i;
4461 u32 tmp;
4462
4463 if (rdev->family >= CHIP_CAYMAN) {
4464 cayman_cp_int_cntl_setup(rdev, 0,
4465 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4466 cayman_cp_int_cntl_setup(rdev, 1, 0);
4467 cayman_cp_int_cntl_setup(rdev, 2, 0);
4468 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4469 WREG32(CAYMAN_DMA1_CNTL, tmp);
4470 } else
4471 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4472 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4473 WREG32(DMA_CNTL, tmp);
4474 WREG32(GRBM_INT_CNTL, 0);
4475 WREG32(SRBM_INT_CNTL, 0);
4476 for (i = 0; i < rdev->num_crtc; i++)
4477 WREG32(INT_MASK + crtc_offsets[i], 0);
4478 for (i = 0; i < rdev->num_crtc; i++)
4479 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4480
4481 /* only one DAC on DCE5 */
4482 if (!ASIC_IS_DCE5(rdev))
4483 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4484 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4485
4486 for (i = 0; i < 6; i++)
4487 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4488}
4489
4490/* Note that the order we write back regs here is important */
4491int evergreen_irq_set(struct radeon_device *rdev)
4492{
4493 int i;
4494 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4495 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4496 u32 grbm_int_cntl = 0;
4497 u32 dma_cntl, dma_cntl1 = 0;
4498 u32 thermal_int = 0;
4499
4500 if (!rdev->irq.installed) {
4501 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4502 return -EINVAL;
4503 }
4504 /* don't enable anything if the ih is disabled */
4505 if (!rdev->ih.enabled) {
4506 r600_disable_interrupts(rdev);
4507 /* force the active interrupt state to all disabled */
4508 evergreen_disable_interrupt_state(rdev);
4509 return 0;
4510 }
4511
4512 if (rdev->family == CHIP_ARUBA)
4513 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4514 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4515 else
4516 thermal_int = RREG32(CG_THERMAL_INT) &
4517 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518
4519 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4520
4521 if (rdev->family >= CHIP_CAYMAN) {
4522 /* enable CP interrupts on all rings */
4523 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4524 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4525 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4526 }
4527 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4528 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4529 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4530 }
4531 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4532 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4533 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4534 }
4535 } else {
4536 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4537 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4538 cp_int_cntl |= RB_INT_ENABLE;
4539 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4540 }
4541 }
4542
4543 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4544 DRM_DEBUG("r600_irq_set: sw int dma\n");
4545 dma_cntl |= TRAP_ENABLE;
4546 }
4547
4548 if (rdev->family >= CHIP_CAYMAN) {
4549 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4550 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4551 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4552 dma_cntl1 |= TRAP_ENABLE;
4553 }
4554 }
4555
4556 if (rdev->irq.dpm_thermal) {
4557 DRM_DEBUG("dpm thermal\n");
4558 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4559 }
4560
4561 if (rdev->family >= CHIP_CAYMAN) {
4562 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4563 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4564 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4565 } else
4566 WREG32(CP_INT_CNTL, cp_int_cntl);
4567
4568 WREG32(DMA_CNTL, dma_cntl);
4569
4570 if (rdev->family >= CHIP_CAYMAN)
4571 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4572
4573 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4574
4575 for (i = 0; i < rdev->num_crtc; i++) {
4576 radeon_irq_kms_set_irq_n_enabled(
4577 rdev, INT_MASK + crtc_offsets[i],
4578 VBLANK_INT_MASK,
4579 rdev->irq.crtc_vblank_int[i] ||
4580 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4581 }
4582
4583 for (i = 0; i < rdev->num_crtc; i++)
4584 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4585
4586 for (i = 0; i < 6; i++) {
4587 radeon_irq_kms_set_irq_n_enabled(
4588 rdev, DC_HPDx_INT_CONTROL(i),
4589 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4590 rdev->irq.hpd[i], "HPD", i);
4591 }
4592
4593 if (rdev->family == CHIP_ARUBA)
4594 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4595 else
4596 WREG32(CG_THERMAL_INT, thermal_int);
4597
4598 for (i = 0; i < 6; i++) {
4599 radeon_irq_kms_set_irq_n_enabled(
4600 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4601 AFMT_AZ_FORMAT_WTRIG_MASK,
4602 rdev->irq.afmt[i], "HDMI", i);
4603 }
4604
4605 /* posting read */
4606 RREG32(SRBM_STATUS);
4607
4608 return 0;
4609}
4610
4611/* Note that the order we write back regs here is important */
4612static void evergreen_irq_ack(struct radeon_device *rdev)
4613{
4614 int i, j;
4615 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4616 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4617 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4618
4619 for (i = 0; i < 6; i++) {
4620 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4621 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4622 if (i < rdev->num_crtc)
4623 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4624 }
4625
4626 /* We write back each interrupt register in pairs of two */
4627 for (i = 0; i < rdev->num_crtc; i += 2) {
4628 for (j = i; j < (i + 2); j++) {
4629 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4630 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4631 GRPH_PFLIP_INT_CLEAR);
4632 }
4633
4634 for (j = i; j < (i + 2); j++) {
4635 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4636 WREG32(VBLANK_STATUS + crtc_offsets[j],
4637 VBLANK_ACK);
4638 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4639 WREG32(VLINE_STATUS + crtc_offsets[j],
4640 VLINE_ACK);
4641 }
4642 }
4643
4644 for (i = 0; i < 6; i++) {
4645 if (disp_int[i] & DC_HPD1_INTERRUPT)
4646 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4647 }
4648
4649 for (i = 0; i < 6; i++) {
4650 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4651 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4652 }
4653
4654 for (i = 0; i < 6; i++) {
4655 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4656 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4657 AFMT_AZ_FORMAT_WTRIG_ACK);
4658 }
4659}
4660
4661static void evergreen_irq_disable(struct radeon_device *rdev)
4662{
4663 r600_disable_interrupts(rdev);
4664 /* Wait and acknowledge irq */
4665 mdelay(1);
4666 evergreen_irq_ack(rdev);
4667 evergreen_disable_interrupt_state(rdev);
4668}
4669
4670void evergreen_irq_suspend(struct radeon_device *rdev)
4671{
4672 evergreen_irq_disable(rdev);
4673 r600_rlc_stop(rdev);
4674}
4675
4676static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4677{
4678 u32 wptr, tmp;
4679
4680 if (rdev->wb.enabled)
4681 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4682 else
4683 wptr = RREG32(IH_RB_WPTR);
4684
4685 if (wptr & RB_OVERFLOW) {
4686 wptr &= ~RB_OVERFLOW;
4687 /* When a ring buffer overflow happen start parsing interrupt
4688 * from the last not overwritten vector (wptr + 16). Hopefully
4689 * this should allow us to catchup.
4690 */
4691 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4692 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4693 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4694 tmp = RREG32(IH_RB_CNTL);
4695 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4696 WREG32(IH_RB_CNTL, tmp);
4697 }
4698 return (wptr & rdev->ih.ptr_mask);
4699}
4700
4701int evergreen_irq_process(struct radeon_device *rdev)
4702{
4703 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4704 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4705 u32 crtc_idx, hpd_idx, afmt_idx;
4706 u32 mask;
4707 u32 wptr;
4708 u32 rptr;
4709 u32 src_id, src_data;
4710 u32 ring_index;
4711 bool queue_hotplug = false;
4712 bool queue_hdmi = false;
4713 bool queue_dp = false;
4714 bool queue_thermal = false;
4715 u32 status, addr;
4716 const char *event_name;
4717
4718 if (!rdev->ih.enabled || rdev->shutdown)
4719 return IRQ_NONE;
4720
4721 wptr = evergreen_get_ih_wptr(rdev);
4722
4723restart_ih:
4724 /* is somebody else already processing irqs? */
4725 if (atomic_xchg(&rdev->ih.lock, 1))
4726 return IRQ_NONE;
4727
4728 rptr = rdev->ih.rptr;
4729 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4730
4731 /* Order reading of wptr vs. reading of IH ring data */
4732 rmb();
4733
4734 /* display interrupts */
4735 evergreen_irq_ack(rdev);
4736
4737 while (rptr != wptr) {
4738 /* wptr/rptr are in bytes! */
4739 ring_index = rptr / 4;
4740 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4741 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4742
4743 switch (src_id) {
4744 case 1: /* D1 vblank/vline */
4745 case 2: /* D2 vblank/vline */
4746 case 3: /* D3 vblank/vline */
4747 case 4: /* D4 vblank/vline */
4748 case 5: /* D5 vblank/vline */
4749 case 6: /* D6 vblank/vline */
4750 crtc_idx = src_id - 1;
4751
4752 if (src_data == 0) { /* vblank */
4753 mask = LB_D1_VBLANK_INTERRUPT;
4754 event_name = "vblank";
4755
4756 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4757 drm_handle_vblank(rdev->ddev, crtc_idx);
4758 rdev->pm.vblank_sync = true;
4759 wake_up(&rdev->irq.vblank_queue);
4760 }
4761 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4762 radeon_crtc_handle_vblank(rdev,
4763 crtc_idx);
4764 }
4765
4766 } else if (src_data == 1) { /* vline */
4767 mask = LB_D1_VLINE_INTERRUPT;
4768 event_name = "vline";
4769 } else {
4770 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4771 src_id, src_data);
4772 break;
4773 }
4774
4775 if (!(disp_int[crtc_idx] & mask)) {
4776 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4777 crtc_idx + 1, event_name);
4778 }
4779
4780 disp_int[crtc_idx] &= ~mask;
4781 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4782
4783 break;
4784 case 8: /* D1 page flip */
4785 case 10: /* D2 page flip */
4786 case 12: /* D3 page flip */
4787 case 14: /* D4 page flip */
4788 case 16: /* D5 page flip */
4789 case 18: /* D6 page flip */
4790 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4791 if (radeon_use_pflipirq > 0)
4792 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4793 break;
4794 case 42: /* HPD hotplug */
4795 if (src_data <= 5) {
4796 hpd_idx = src_data;
4797 mask = DC_HPD1_INTERRUPT;
4798 queue_hotplug = true;
4799 event_name = "HPD";
4800
4801 } else if (src_data <= 11) {
4802 hpd_idx = src_data - 6;
4803 mask = DC_HPD1_RX_INTERRUPT;
4804 queue_dp = true;
4805 event_name = "HPD_RX";
4806
4807 } else {
4808 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4809 src_id, src_data);
4810 break;
4811 }
4812
4813 if (!(disp_int[hpd_idx] & mask))
4814 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4815
4816 disp_int[hpd_idx] &= ~mask;
4817 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4818
4819 break;
4820 case 44: /* hdmi */
4821 afmt_idx = src_data;
4822 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4823 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4824
4825 if (afmt_idx > 5) {
4826 DRM_ERROR("Unhandled interrupt: %d %d\n",
4827 src_id, src_data);
4828 break;
4829 }
4830 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4831 queue_hdmi = true;
4832 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4833 break;
4834 case 96:
4835 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836 WREG32(SRBM_INT_ACK, 0x1);
4837 break;
4838 case 124: /* UVD */
4839 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4841 break;
4842 case 146:
4843 case 147:
4844 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846 /* reset addr and status */
4847 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848 if (addr == 0x0 && status == 0x0)
4849 break;
4850 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4852 addr);
4853 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4854 status);
4855 cayman_vm_decode_fault(rdev, status, addr);
4856 break;
4857 case 176: /* CP_INT in ring buffer */
4858 case 177: /* CP_INT in IB1 */
4859 case 178: /* CP_INT in IB2 */
4860 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4862 break;
4863 case 181: /* CP EOP event */
4864 DRM_DEBUG("IH: CP EOP\n");
4865 if (rdev->family >= CHIP_CAYMAN) {
4866 switch (src_data) {
4867 case 0:
4868 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4869 break;
4870 case 1:
4871 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4872 break;
4873 case 2:
4874 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4875 break;
4876 }
4877 } else
4878 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4879 break;
4880 case 224: /* DMA trap event */
4881 DRM_DEBUG("IH: DMA trap\n");
4882 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4883 break;
4884 case 230: /* thermal low to high */
4885 DRM_DEBUG("IH: thermal low to high\n");
4886 rdev->pm.dpm.thermal.high_to_low = false;
4887 queue_thermal = true;
4888 break;
4889 case 231: /* thermal high to low */
4890 DRM_DEBUG("IH: thermal high to low\n");
4891 rdev->pm.dpm.thermal.high_to_low = true;
4892 queue_thermal = true;
4893 break;
4894 case 233: /* GUI IDLE */
4895 DRM_DEBUG("IH: GUI idle\n");
4896 break;
4897 case 244: /* DMA trap event */
4898 if (rdev->family >= CHIP_CAYMAN) {
4899 DRM_DEBUG("IH: DMA1 trap\n");
4900 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4901 }
4902 break;
4903 default:
4904 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4905 break;
4906 }
4907
4908 /* wptr/rptr are in bytes! */
4909 rptr += 16;
4910 rptr &= rdev->ih.ptr_mask;
4911 WREG32(IH_RB_RPTR, rptr);
4912 }
4913 if (queue_dp)
4914 schedule_work(&rdev->dp_work);
4915 if (queue_hotplug)
4916 schedule_delayed_work(&rdev->hotplug_work, 0);
4917 if (queue_hdmi)
4918 schedule_work(&rdev->audio_work);
4919 if (queue_thermal && rdev->pm.dpm_enabled)
4920 schedule_work(&rdev->pm.dpm.thermal.work);
4921 rdev->ih.rptr = rptr;
4922 atomic_set(&rdev->ih.lock, 0);
4923
4924 /* make sure wptr hasn't changed while processing */
4925 wptr = evergreen_get_ih_wptr(rdev);
4926 if (wptr != rptr)
4927 goto restart_ih;
4928
4929 return IRQ_HANDLED;
4930}
4931
4932static void evergreen_uvd_init(struct radeon_device *rdev)
4933{
4934 int r;
4935
4936 if (!rdev->has_uvd)
4937 return;
4938
4939 r = radeon_uvd_init(rdev);
4940 if (r) {
4941 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4942 /*
4943 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944 * to early fails uvd_v2_2_resume() and thus nothing happens
4945 * there. So it is pointless to try to go through that code
4946 * hence why we disable uvd here.
4947 */
4948 rdev->has_uvd = false;
4949 return;
4950 }
4951 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4953}
4954
4955static void evergreen_uvd_start(struct radeon_device *rdev)
4956{
4957 int r;
4958
4959 if (!rdev->has_uvd)
4960 return;
4961
4962 r = uvd_v2_2_resume(rdev);
4963 if (r) {
4964 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4965 goto error;
4966 }
4967 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4968 if (r) {
4969 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4970 goto error;
4971 }
4972 return;
4973
4974error:
4975 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4976}
4977
4978static void evergreen_uvd_resume(struct radeon_device *rdev)
4979{
4980 struct radeon_ring *ring;
4981 int r;
4982
4983 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4984 return;
4985
4986 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4988 if (r) {
4989 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4990 return;
4991 }
4992 r = uvd_v1_0_init(rdev);
4993 if (r) {
4994 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4995 return;
4996 }
4997}
4998
4999static int evergreen_startup(struct radeon_device *rdev)
5000{
5001 struct radeon_ring *ring;
5002 int r;
5003
5004 /* enable pcie gen2 link */
5005 evergreen_pcie_gen2_enable(rdev);
5006 /* enable aspm */
5007 evergreen_program_aspm(rdev);
5008
5009 /* scratch needs to be initialized before MC */
5010 r = r600_vram_scratch_init(rdev);
5011 if (r)
5012 return r;
5013
5014 evergreen_mc_program(rdev);
5015
5016 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017 r = ni_mc_load_microcode(rdev);
5018 if (r) {
5019 DRM_ERROR("Failed to load MC firmware!\n");
5020 return r;
5021 }
5022 }
5023
5024 if (rdev->flags & RADEON_IS_AGP) {
5025 evergreen_agp_enable(rdev);
5026 } else {
5027 r = evergreen_pcie_gart_enable(rdev);
5028 if (r)
5029 return r;
5030 }
5031 evergreen_gpu_init(rdev);
5032
5033 /* allocate rlc buffers */
5034 if (rdev->flags & RADEON_IS_IGP) {
5035 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036 rdev->rlc.reg_list_size =
5037 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038 rdev->rlc.cs_data = evergreen_cs_data;
5039 r = sumo_rlc_init(rdev);
5040 if (r) {
5041 DRM_ERROR("Failed to init rlc BOs!\n");
5042 return r;
5043 }
5044 }
5045
5046 /* allocate wb buffer */
5047 r = radeon_wb_init(rdev);
5048 if (r)
5049 return r;
5050
5051 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5052 if (r) {
5053 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5054 return r;
5055 }
5056
5057 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5058 if (r) {
5059 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5060 return r;
5061 }
5062
5063 evergreen_uvd_start(rdev);
5064
5065 /* Enable IRQ */
5066 if (!rdev->irq.installed) {
5067 r = radeon_irq_kms_init(rdev);
5068 if (r)
5069 return r;
5070 }
5071
5072 r = r600_irq_init(rdev);
5073 if (r) {
5074 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075 radeon_irq_kms_fini(rdev);
5076 return r;
5077 }
5078 evergreen_irq_set(rdev);
5079
5080 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5082 RADEON_CP_PACKET2);
5083 if (r)
5084 return r;
5085
5086 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5089 if (r)
5090 return r;
5091
5092 r = evergreen_cp_load_microcode(rdev);
5093 if (r)
5094 return r;
5095 r = evergreen_cp_resume(rdev);
5096 if (r)
5097 return r;
5098 r = r600_dma_resume(rdev);
5099 if (r)
5100 return r;
5101
5102 evergreen_uvd_resume(rdev);
5103
5104 r = radeon_ib_pool_init(rdev);
5105 if (r) {
5106 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5107 return r;
5108 }
5109
5110 r = radeon_audio_init(rdev);
5111 if (r) {
5112 DRM_ERROR("radeon: audio init failed\n");
5113 return r;
5114 }
5115
5116 return 0;
5117}
5118
5119int evergreen_resume(struct radeon_device *rdev)
5120{
5121 int r;
5122
5123 /* reset the asic, the gfx blocks are often in a bad state
5124 * after the driver is unloaded or after a resume
5125 */
5126 if (radeon_asic_reset(rdev))
5127 dev_warn(rdev->dev, "GPU reset failed !\n");
5128 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129 * posting will perform necessary task to bring back GPU into good
5130 * shape.
5131 */
5132 /* post card */
5133 atom_asic_init(rdev->mode_info.atom_context);
5134
5135 /* init golden registers */
5136 evergreen_init_golden_registers(rdev);
5137
5138 if (rdev->pm.pm_method == PM_METHOD_DPM)
5139 radeon_pm_resume(rdev);
5140
5141 rdev->accel_working = true;
5142 r = evergreen_startup(rdev);
5143 if (r) {
5144 DRM_ERROR("evergreen startup failed on resume\n");
5145 rdev->accel_working = false;
5146 return r;
5147 }
5148
5149 return r;
5150
5151}
5152
5153int evergreen_suspend(struct radeon_device *rdev)
5154{
5155 radeon_pm_suspend(rdev);
5156 radeon_audio_fini(rdev);
5157 if (rdev->has_uvd) {
5158 uvd_v1_0_fini(rdev);
5159 radeon_uvd_suspend(rdev);
5160 }
5161 r700_cp_stop(rdev);
5162 r600_dma_stop(rdev);
5163 evergreen_irq_suspend(rdev);
5164 radeon_wb_disable(rdev);
5165 evergreen_pcie_gart_disable(rdev);
5166
5167 return 0;
5168}
5169
5170/* Plan is to move initialization in that function and use
5171 * helper function so that radeon_device_init pretty much
5172 * do nothing more than calling asic specific function. This
5173 * should also allow to remove a bunch of callback function
5174 * like vram_info.
5175 */
5176int evergreen_init(struct radeon_device *rdev)
5177{
5178 int r;
5179
5180 /* Read BIOS */
5181 if (!radeon_get_bios(rdev)) {
5182 if (ASIC_IS_AVIVO(rdev))
5183 return -EINVAL;
5184 }
5185 /* Must be an ATOMBIOS */
5186 if (!rdev->is_atom_bios) {
5187 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5188 return -EINVAL;
5189 }
5190 r = radeon_atombios_init(rdev);
5191 if (r)
5192 return r;
5193 /* reset the asic, the gfx blocks are often in a bad state
5194 * after the driver is unloaded or after a resume
5195 */
5196 if (radeon_asic_reset(rdev))
5197 dev_warn(rdev->dev, "GPU reset failed !\n");
5198 /* Post card if necessary */
5199 if (!radeon_card_posted(rdev)) {
5200 if (!rdev->bios) {
5201 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5202 return -EINVAL;
5203 }
5204 DRM_INFO("GPU not posted. posting now...\n");
5205 atom_asic_init(rdev->mode_info.atom_context);
5206 }
5207 /* init golden registers */
5208 evergreen_init_golden_registers(rdev);
5209 /* Initialize scratch registers */
5210 r600_scratch_init(rdev);
5211 /* Initialize surface registers */
5212 radeon_surface_init(rdev);
5213 /* Initialize clocks */
5214 radeon_get_clock_info(rdev->ddev);
5215 /* Fence driver */
5216 r = radeon_fence_driver_init(rdev);
5217 if (r)
5218 return r;
5219 /* initialize AGP */
5220 if (rdev->flags & RADEON_IS_AGP) {
5221 r = radeon_agp_init(rdev);
5222 if (r)
5223 radeon_agp_disable(rdev);
5224 }
5225 /* initialize memory controller */
5226 r = evergreen_mc_init(rdev);
5227 if (r)
5228 return r;
5229 /* Memory manager */
5230 r = radeon_bo_init(rdev);
5231 if (r)
5232 return r;
5233
5234 if (ASIC_IS_DCE5(rdev)) {
5235 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5236 r = ni_init_microcode(rdev);
5237 if (r) {
5238 DRM_ERROR("Failed to load firmware!\n");
5239 return r;
5240 }
5241 }
5242 } else {
5243 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5244 r = r600_init_microcode(rdev);
5245 if (r) {
5246 DRM_ERROR("Failed to load firmware!\n");
5247 return r;
5248 }
5249 }
5250 }
5251
5252 /* Initialize power management */
5253 radeon_pm_init(rdev);
5254
5255 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5256 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5257
5258 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5259 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5260
5261 evergreen_uvd_init(rdev);
5262
5263 rdev->ih.ring_obj = NULL;
5264 r600_ih_ring_init(rdev, 64 * 1024);
5265
5266 r = r600_pcie_gart_init(rdev);
5267 if (r)
5268 return r;
5269
5270 rdev->accel_working = true;
5271 r = evergreen_startup(rdev);
5272 if (r) {
5273 dev_err(rdev->dev, "disabling GPU acceleration\n");
5274 r700_cp_fini(rdev);
5275 r600_dma_fini(rdev);
5276 r600_irq_fini(rdev);
5277 if (rdev->flags & RADEON_IS_IGP)
5278 sumo_rlc_fini(rdev);
5279 radeon_wb_fini(rdev);
5280 radeon_ib_pool_fini(rdev);
5281 radeon_irq_kms_fini(rdev);
5282 evergreen_pcie_gart_fini(rdev);
5283 rdev->accel_working = false;
5284 }
5285
5286 /* Don't start up if the MC ucode is missing on BTC parts.
5287 * The default clocks and voltages before the MC ucode
5288 * is loaded are not suffient for advanced operations.
5289 */
5290 if (ASIC_IS_DCE5(rdev)) {
5291 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5292 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5293 return -EINVAL;
5294 }
5295 }
5296
5297 return 0;
5298}
5299
5300void evergreen_fini(struct radeon_device *rdev)
5301{
5302 radeon_pm_fini(rdev);
5303 radeon_audio_fini(rdev);
5304 r700_cp_fini(rdev);
5305 r600_dma_fini(rdev);
5306 r600_irq_fini(rdev);
5307 if (rdev->flags & RADEON_IS_IGP)
5308 sumo_rlc_fini(rdev);
5309 radeon_wb_fini(rdev);
5310 radeon_ib_pool_fini(rdev);
5311 radeon_irq_kms_fini(rdev);
5312 uvd_v1_0_fini(rdev);
5313 radeon_uvd_fini(rdev);
5314 evergreen_pcie_gart_fini(rdev);
5315 r600_vram_scratch_fini(rdev);
5316 radeon_gem_fini(rdev);
5317 radeon_fence_driver_fini(rdev);
5318 radeon_agp_fini(rdev);
5319 radeon_bo_fini(rdev);
5320 radeon_atombios_fini(rdev);
5321 kfree(rdev->bios);
5322 rdev->bios = NULL;
5323}
5324
5325void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5326{
5327 u32 link_width_cntl, speed_cntl;
5328
5329 if (radeon_pcie_gen2 == 0)
5330 return;
5331
5332 if (rdev->flags & RADEON_IS_IGP)
5333 return;
5334
5335 if (!(rdev->flags & RADEON_IS_PCIE))
5336 return;
5337
5338 /* x2 cards have a special sequence */
5339 if (ASIC_IS_X2(rdev))
5340 return;
5341
5342 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5343 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5344 return;
5345
5346 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5347 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5348 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5349 return;
5350 }
5351
5352 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5353
5354 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5355 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5356
5357 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5358 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5359 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5360
5361 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5362 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5363 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5364
5365 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5366 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5367 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5368
5369 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5370 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5371 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5372
5373 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5374 speed_cntl |= LC_GEN2_EN_STRAP;
5375 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5376
5377 } else {
5378 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5379 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5380 if (1)
5381 link_width_cntl |= LC_UPCONFIGURE_DIS;
5382 else
5383 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5384 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5385 }
5386}
5387
5388void evergreen_program_aspm(struct radeon_device *rdev)
5389{
5390 u32 data, orig;
5391 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5392 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5393 /* fusion_platform = true
5394 * if the system is a fusion system
5395 * (APU or DGPU in a fusion system).
5396 * todo: check if the system is a fusion platform.
5397 */
5398 bool fusion_platform = false;
5399
5400 if (radeon_aspm == 0)
5401 return;
5402
5403 if (!(rdev->flags & RADEON_IS_PCIE))
5404 return;
5405
5406 switch (rdev->family) {
5407 case CHIP_CYPRESS:
5408 case CHIP_HEMLOCK:
5409 case CHIP_JUNIPER:
5410 case CHIP_REDWOOD:
5411 case CHIP_CEDAR:
5412 case CHIP_SUMO:
5413 case CHIP_SUMO2:
5414 case CHIP_PALM:
5415 case CHIP_ARUBA:
5416 disable_l0s = true;
5417 break;
5418 default:
5419 disable_l0s = false;
5420 break;
5421 }
5422
5423 if (rdev->flags & RADEON_IS_IGP)
5424 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5425
5426 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5427 if (fusion_platform)
5428 data &= ~MULTI_PIF;
5429 else
5430 data |= MULTI_PIF;
5431 if (data != orig)
5432 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5433
5434 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5435 if (fusion_platform)
5436 data &= ~MULTI_PIF;
5437 else
5438 data |= MULTI_PIF;
5439 if (data != orig)
5440 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5441
5442 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5443 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5444 if (!disable_l0s) {
5445 if (rdev->family >= CHIP_BARTS)
5446 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5447 else
5448 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5449 }
5450
5451 if (!disable_l1) {
5452 if (rdev->family >= CHIP_BARTS)
5453 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5454 else
5455 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5456
5457 if (!disable_plloff_in_l1) {
5458 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5459 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5460 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5461 if (data != orig)
5462 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5463
5464 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5465 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5466 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5467 if (data != orig)
5468 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5469
5470 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5471 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5472 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5473 if (data != orig)
5474 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5475
5476 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5477 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5478 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5479 if (data != orig)
5480 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5481
5482 if (rdev->family >= CHIP_BARTS) {
5483 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5484 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5485 data |= PLL_RAMP_UP_TIME_0(4);
5486 if (data != orig)
5487 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5488
5489 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5490 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5491 data |= PLL_RAMP_UP_TIME_1(4);
5492 if (data != orig)
5493 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5494
5495 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5496 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5497 data |= PLL_RAMP_UP_TIME_0(4);
5498 if (data != orig)
5499 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5500
5501 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5502 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5503 data |= PLL_RAMP_UP_TIME_1(4);
5504 if (data != orig)
5505 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5506 }
5507
5508 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5509 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5510 data |= LC_DYN_LANES_PWR_STATE(3);
5511 if (data != orig)
5512 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5513
5514 if (rdev->family >= CHIP_BARTS) {
5515 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5516 data &= ~LS2_EXIT_TIME_MASK;
5517 data |= LS2_EXIT_TIME(1);
5518 if (data != orig)
5519 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5520
5521 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5522 data &= ~LS2_EXIT_TIME_MASK;
5523 data |= LS2_EXIT_TIME(1);
5524 if (data != orig)
5525 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5526 }
5527 }
5528 }
5529
5530 /* evergreen parts only */
5531 if (rdev->family < CHIP_BARTS)
5532 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5533
5534 if (pcie_lc_cntl != pcie_lc_cntl_old)
5535 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5536}