Loading...
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include "drmP.h"
28#include "radeon.h"
29#include "radeon_asic.h"
30#include "radeon_drm.h"
31#include "evergreend.h"
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
35#include "evergreen_blit_shaders.h"
36
37#define EVERGREEN_PFP_UCODE_SIZE 1120
38#define EVERGREEN_PM4_UCODE_SIZE 1376
39
40static void evergreen_gpu_init(struct radeon_device *rdev);
41void evergreen_fini(struct radeon_device *rdev);
42static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
43
44void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
45{
46 u16 ctl, v;
47 int cap, err;
48
49 cap = pci_pcie_cap(rdev->pdev);
50 if (!cap)
51 return;
52
53 err = pci_read_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, &ctl);
54 if (err)
55 return;
56
57 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
58
59 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
60 * to avoid hangs or perfomance issues
61 */
62 if ((v == 0) || (v == 6) || (v == 7)) {
63 ctl &= ~PCI_EXP_DEVCTL_READRQ;
64 ctl |= (2 << 12);
65 pci_write_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, ctl);
66 }
67}
68
69void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
70{
71 /* enable the pflip int */
72 radeon_irq_kms_pflip_irq_get(rdev, crtc);
73}
74
75void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
76{
77 /* disable the pflip int */
78 radeon_irq_kms_pflip_irq_put(rdev, crtc);
79}
80
81u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
82{
83 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
84 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
85
86 /* Lock the graphics update lock */
87 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
88 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
89
90 /* update the scanout addresses */
91 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
92 upper_32_bits(crtc_base));
93 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
94 (u32)crtc_base);
95
96 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
97 upper_32_bits(crtc_base));
98 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
99 (u32)crtc_base);
100
101 /* Wait for update_pending to go high. */
102 while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
103 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
104
105 /* Unlock the lock, so double-buffering can take place inside vblank */
106 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
107 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
108
109 /* Return current update_pending status: */
110 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
111}
112
113/* get temperature in millidegrees */
114int evergreen_get_temp(struct radeon_device *rdev)
115{
116 u32 temp, toffset;
117 int actual_temp = 0;
118
119 if (rdev->family == CHIP_JUNIPER) {
120 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
121 TOFFSET_SHIFT;
122 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
123 TS0_ADC_DOUT_SHIFT;
124
125 if (toffset & 0x100)
126 actual_temp = temp / 2 - (0x200 - toffset);
127 else
128 actual_temp = temp / 2 + toffset;
129
130 actual_temp = actual_temp * 1000;
131
132 } else {
133 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
134 ASIC_T_SHIFT;
135
136 if (temp & 0x400)
137 actual_temp = -256;
138 else if (temp & 0x200)
139 actual_temp = 255;
140 else if (temp & 0x100) {
141 actual_temp = temp & 0x1ff;
142 actual_temp |= ~0x1ff;
143 } else
144 actual_temp = temp & 0xff;
145
146 actual_temp = (actual_temp * 1000) / 2;
147 }
148
149 return actual_temp;
150}
151
152int sumo_get_temp(struct radeon_device *rdev)
153{
154 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
155 int actual_temp = temp - 49;
156
157 return actual_temp * 1000;
158}
159
160void evergreen_pm_misc(struct radeon_device *rdev)
161{
162 int req_ps_idx = rdev->pm.requested_power_state_index;
163 int req_cm_idx = rdev->pm.requested_clock_mode_index;
164 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
165 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
166
167 if (voltage->type == VOLTAGE_SW) {
168 /* 0xff01 is a flag rather then an actual voltage */
169 if (voltage->voltage == 0xff01)
170 return;
171 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
172 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
173 rdev->pm.current_vddc = voltage->voltage;
174 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
175 }
176 /* 0xff01 is a flag rather then an actual voltage */
177 if (voltage->vddci == 0xff01)
178 return;
179 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
180 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
181 rdev->pm.current_vddci = voltage->vddci;
182 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
183 }
184 }
185}
186
187void evergreen_pm_prepare(struct radeon_device *rdev)
188{
189 struct drm_device *ddev = rdev->ddev;
190 struct drm_crtc *crtc;
191 struct radeon_crtc *radeon_crtc;
192 u32 tmp;
193
194 /* disable any active CRTCs */
195 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
196 radeon_crtc = to_radeon_crtc(crtc);
197 if (radeon_crtc->enabled) {
198 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
199 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
200 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
201 }
202 }
203}
204
205void evergreen_pm_finish(struct radeon_device *rdev)
206{
207 struct drm_device *ddev = rdev->ddev;
208 struct drm_crtc *crtc;
209 struct radeon_crtc *radeon_crtc;
210 u32 tmp;
211
212 /* enable any active CRTCs */
213 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
214 radeon_crtc = to_radeon_crtc(crtc);
215 if (radeon_crtc->enabled) {
216 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
217 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
218 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
219 }
220 }
221}
222
223bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
224{
225 bool connected = false;
226
227 switch (hpd) {
228 case RADEON_HPD_1:
229 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
230 connected = true;
231 break;
232 case RADEON_HPD_2:
233 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
234 connected = true;
235 break;
236 case RADEON_HPD_3:
237 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
238 connected = true;
239 break;
240 case RADEON_HPD_4:
241 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
242 connected = true;
243 break;
244 case RADEON_HPD_5:
245 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
246 connected = true;
247 break;
248 case RADEON_HPD_6:
249 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
250 connected = true;
251 break;
252 default:
253 break;
254 }
255
256 return connected;
257}
258
259void evergreen_hpd_set_polarity(struct radeon_device *rdev,
260 enum radeon_hpd_id hpd)
261{
262 u32 tmp;
263 bool connected = evergreen_hpd_sense(rdev, hpd);
264
265 switch (hpd) {
266 case RADEON_HPD_1:
267 tmp = RREG32(DC_HPD1_INT_CONTROL);
268 if (connected)
269 tmp &= ~DC_HPDx_INT_POLARITY;
270 else
271 tmp |= DC_HPDx_INT_POLARITY;
272 WREG32(DC_HPD1_INT_CONTROL, tmp);
273 break;
274 case RADEON_HPD_2:
275 tmp = RREG32(DC_HPD2_INT_CONTROL);
276 if (connected)
277 tmp &= ~DC_HPDx_INT_POLARITY;
278 else
279 tmp |= DC_HPDx_INT_POLARITY;
280 WREG32(DC_HPD2_INT_CONTROL, tmp);
281 break;
282 case RADEON_HPD_3:
283 tmp = RREG32(DC_HPD3_INT_CONTROL);
284 if (connected)
285 tmp &= ~DC_HPDx_INT_POLARITY;
286 else
287 tmp |= DC_HPDx_INT_POLARITY;
288 WREG32(DC_HPD3_INT_CONTROL, tmp);
289 break;
290 case RADEON_HPD_4:
291 tmp = RREG32(DC_HPD4_INT_CONTROL);
292 if (connected)
293 tmp &= ~DC_HPDx_INT_POLARITY;
294 else
295 tmp |= DC_HPDx_INT_POLARITY;
296 WREG32(DC_HPD4_INT_CONTROL, tmp);
297 break;
298 case RADEON_HPD_5:
299 tmp = RREG32(DC_HPD5_INT_CONTROL);
300 if (connected)
301 tmp &= ~DC_HPDx_INT_POLARITY;
302 else
303 tmp |= DC_HPDx_INT_POLARITY;
304 WREG32(DC_HPD5_INT_CONTROL, tmp);
305 break;
306 case RADEON_HPD_6:
307 tmp = RREG32(DC_HPD6_INT_CONTROL);
308 if (connected)
309 tmp &= ~DC_HPDx_INT_POLARITY;
310 else
311 tmp |= DC_HPDx_INT_POLARITY;
312 WREG32(DC_HPD6_INT_CONTROL, tmp);
313 break;
314 default:
315 break;
316 }
317}
318
319void evergreen_hpd_init(struct radeon_device *rdev)
320{
321 struct drm_device *dev = rdev->ddev;
322 struct drm_connector *connector;
323 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
324 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
325
326 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
327 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
328 switch (radeon_connector->hpd.hpd) {
329 case RADEON_HPD_1:
330 WREG32(DC_HPD1_CONTROL, tmp);
331 rdev->irq.hpd[0] = true;
332 break;
333 case RADEON_HPD_2:
334 WREG32(DC_HPD2_CONTROL, tmp);
335 rdev->irq.hpd[1] = true;
336 break;
337 case RADEON_HPD_3:
338 WREG32(DC_HPD3_CONTROL, tmp);
339 rdev->irq.hpd[2] = true;
340 break;
341 case RADEON_HPD_4:
342 WREG32(DC_HPD4_CONTROL, tmp);
343 rdev->irq.hpd[3] = true;
344 break;
345 case RADEON_HPD_5:
346 WREG32(DC_HPD5_CONTROL, tmp);
347 rdev->irq.hpd[4] = true;
348 break;
349 case RADEON_HPD_6:
350 WREG32(DC_HPD6_CONTROL, tmp);
351 rdev->irq.hpd[5] = true;
352 break;
353 default:
354 break;
355 }
356 }
357 if (rdev->irq.installed)
358 evergreen_irq_set(rdev);
359}
360
361void evergreen_hpd_fini(struct radeon_device *rdev)
362{
363 struct drm_device *dev = rdev->ddev;
364 struct drm_connector *connector;
365
366 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
367 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
368 switch (radeon_connector->hpd.hpd) {
369 case RADEON_HPD_1:
370 WREG32(DC_HPD1_CONTROL, 0);
371 rdev->irq.hpd[0] = false;
372 break;
373 case RADEON_HPD_2:
374 WREG32(DC_HPD2_CONTROL, 0);
375 rdev->irq.hpd[1] = false;
376 break;
377 case RADEON_HPD_3:
378 WREG32(DC_HPD3_CONTROL, 0);
379 rdev->irq.hpd[2] = false;
380 break;
381 case RADEON_HPD_4:
382 WREG32(DC_HPD4_CONTROL, 0);
383 rdev->irq.hpd[3] = false;
384 break;
385 case RADEON_HPD_5:
386 WREG32(DC_HPD5_CONTROL, 0);
387 rdev->irq.hpd[4] = false;
388 break;
389 case RADEON_HPD_6:
390 WREG32(DC_HPD6_CONTROL, 0);
391 rdev->irq.hpd[5] = false;
392 break;
393 default:
394 break;
395 }
396 }
397}
398
399/* watermark setup */
400
401static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
402 struct radeon_crtc *radeon_crtc,
403 struct drm_display_mode *mode,
404 struct drm_display_mode *other_mode)
405{
406 u32 tmp;
407 /*
408 * Line Buffer Setup
409 * There are 3 line buffers, each one shared by 2 display controllers.
410 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
411 * the display controllers. The paritioning is done via one of four
412 * preset allocations specified in bits 2:0:
413 * first display controller
414 * 0 - first half of lb (3840 * 2)
415 * 1 - first 3/4 of lb (5760 * 2)
416 * 2 - whole lb (7680 * 2), other crtc must be disabled
417 * 3 - first 1/4 of lb (1920 * 2)
418 * second display controller
419 * 4 - second half of lb (3840 * 2)
420 * 5 - second 3/4 of lb (5760 * 2)
421 * 6 - whole lb (7680 * 2), other crtc must be disabled
422 * 7 - last 1/4 of lb (1920 * 2)
423 */
424 /* this can get tricky if we have two large displays on a paired group
425 * of crtcs. Ideally for multiple large displays we'd assign them to
426 * non-linked crtcs for maximum line buffer allocation.
427 */
428 if (radeon_crtc->base.enabled && mode) {
429 if (other_mode)
430 tmp = 0; /* 1/2 */
431 else
432 tmp = 2; /* whole */
433 } else
434 tmp = 0;
435
436 /* second controller of the pair uses second half of the lb */
437 if (radeon_crtc->crtc_id % 2)
438 tmp += 4;
439 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
440
441 if (radeon_crtc->base.enabled && mode) {
442 switch (tmp) {
443 case 0:
444 case 4:
445 default:
446 if (ASIC_IS_DCE5(rdev))
447 return 4096 * 2;
448 else
449 return 3840 * 2;
450 case 1:
451 case 5:
452 if (ASIC_IS_DCE5(rdev))
453 return 6144 * 2;
454 else
455 return 5760 * 2;
456 case 2:
457 case 6:
458 if (ASIC_IS_DCE5(rdev))
459 return 8192 * 2;
460 else
461 return 7680 * 2;
462 case 3:
463 case 7:
464 if (ASIC_IS_DCE5(rdev))
465 return 2048 * 2;
466 else
467 return 1920 * 2;
468 }
469 }
470
471 /* controller not enabled, so no lb used */
472 return 0;
473}
474
475static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
476{
477 u32 tmp = RREG32(MC_SHARED_CHMAP);
478
479 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
480 case 0:
481 default:
482 return 1;
483 case 1:
484 return 2;
485 case 2:
486 return 4;
487 case 3:
488 return 8;
489 }
490}
491
492struct evergreen_wm_params {
493 u32 dram_channels; /* number of dram channels */
494 u32 yclk; /* bandwidth per dram data pin in kHz */
495 u32 sclk; /* engine clock in kHz */
496 u32 disp_clk; /* display clock in kHz */
497 u32 src_width; /* viewport width */
498 u32 active_time; /* active display time in ns */
499 u32 blank_time; /* blank time in ns */
500 bool interlaced; /* mode is interlaced */
501 fixed20_12 vsc; /* vertical scale ratio */
502 u32 num_heads; /* number of active crtcs */
503 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
504 u32 lb_size; /* line buffer allocated to pipe */
505 u32 vtaps; /* vertical scaler taps */
506};
507
508static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
509{
510 /* Calculate DRAM Bandwidth and the part allocated to display. */
511 fixed20_12 dram_efficiency; /* 0.7 */
512 fixed20_12 yclk, dram_channels, bandwidth;
513 fixed20_12 a;
514
515 a.full = dfixed_const(1000);
516 yclk.full = dfixed_const(wm->yclk);
517 yclk.full = dfixed_div(yclk, a);
518 dram_channels.full = dfixed_const(wm->dram_channels * 4);
519 a.full = dfixed_const(10);
520 dram_efficiency.full = dfixed_const(7);
521 dram_efficiency.full = dfixed_div(dram_efficiency, a);
522 bandwidth.full = dfixed_mul(dram_channels, yclk);
523 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
524
525 return dfixed_trunc(bandwidth);
526}
527
528static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
529{
530 /* Calculate DRAM Bandwidth and the part allocated to display. */
531 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
532 fixed20_12 yclk, dram_channels, bandwidth;
533 fixed20_12 a;
534
535 a.full = dfixed_const(1000);
536 yclk.full = dfixed_const(wm->yclk);
537 yclk.full = dfixed_div(yclk, a);
538 dram_channels.full = dfixed_const(wm->dram_channels * 4);
539 a.full = dfixed_const(10);
540 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
541 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
542 bandwidth.full = dfixed_mul(dram_channels, yclk);
543 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
544
545 return dfixed_trunc(bandwidth);
546}
547
548static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
549{
550 /* Calculate the display Data return Bandwidth */
551 fixed20_12 return_efficiency; /* 0.8 */
552 fixed20_12 sclk, bandwidth;
553 fixed20_12 a;
554
555 a.full = dfixed_const(1000);
556 sclk.full = dfixed_const(wm->sclk);
557 sclk.full = dfixed_div(sclk, a);
558 a.full = dfixed_const(10);
559 return_efficiency.full = dfixed_const(8);
560 return_efficiency.full = dfixed_div(return_efficiency, a);
561 a.full = dfixed_const(32);
562 bandwidth.full = dfixed_mul(a, sclk);
563 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
564
565 return dfixed_trunc(bandwidth);
566}
567
568static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
569{
570 /* Calculate the DMIF Request Bandwidth */
571 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
572 fixed20_12 disp_clk, bandwidth;
573 fixed20_12 a;
574
575 a.full = dfixed_const(1000);
576 disp_clk.full = dfixed_const(wm->disp_clk);
577 disp_clk.full = dfixed_div(disp_clk, a);
578 a.full = dfixed_const(10);
579 disp_clk_request_efficiency.full = dfixed_const(8);
580 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
581 a.full = dfixed_const(32);
582 bandwidth.full = dfixed_mul(a, disp_clk);
583 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
584
585 return dfixed_trunc(bandwidth);
586}
587
588static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
589{
590 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
591 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
592 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
593 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
594
595 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
596}
597
598static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
599{
600 /* Calculate the display mode Average Bandwidth
601 * DisplayMode should contain the source and destination dimensions,
602 * timing, etc.
603 */
604 fixed20_12 bpp;
605 fixed20_12 line_time;
606 fixed20_12 src_width;
607 fixed20_12 bandwidth;
608 fixed20_12 a;
609
610 a.full = dfixed_const(1000);
611 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
612 line_time.full = dfixed_div(line_time, a);
613 bpp.full = dfixed_const(wm->bytes_per_pixel);
614 src_width.full = dfixed_const(wm->src_width);
615 bandwidth.full = dfixed_mul(src_width, bpp);
616 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
617 bandwidth.full = dfixed_div(bandwidth, line_time);
618
619 return dfixed_trunc(bandwidth);
620}
621
622static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
623{
624 /* First calcualte the latency in ns */
625 u32 mc_latency = 2000; /* 2000 ns. */
626 u32 available_bandwidth = evergreen_available_bandwidth(wm);
627 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
628 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
629 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
630 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
631 (wm->num_heads * cursor_line_pair_return_time);
632 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
633 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
634 fixed20_12 a, b, c;
635
636 if (wm->num_heads == 0)
637 return 0;
638
639 a.full = dfixed_const(2);
640 b.full = dfixed_const(1);
641 if ((wm->vsc.full > a.full) ||
642 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
643 (wm->vtaps >= 5) ||
644 ((wm->vsc.full >= a.full) && wm->interlaced))
645 max_src_lines_per_dst_line = 4;
646 else
647 max_src_lines_per_dst_line = 2;
648
649 a.full = dfixed_const(available_bandwidth);
650 b.full = dfixed_const(wm->num_heads);
651 a.full = dfixed_div(a, b);
652
653 b.full = dfixed_const(1000);
654 c.full = dfixed_const(wm->disp_clk);
655 b.full = dfixed_div(c, b);
656 c.full = dfixed_const(wm->bytes_per_pixel);
657 b.full = dfixed_mul(b, c);
658
659 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
660
661 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
662 b.full = dfixed_const(1000);
663 c.full = dfixed_const(lb_fill_bw);
664 b.full = dfixed_div(c, b);
665 a.full = dfixed_div(a, b);
666 line_fill_time = dfixed_trunc(a);
667
668 if (line_fill_time < wm->active_time)
669 return latency;
670 else
671 return latency + (line_fill_time - wm->active_time);
672
673}
674
675static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
676{
677 if (evergreen_average_bandwidth(wm) <=
678 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
679 return true;
680 else
681 return false;
682};
683
684static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
685{
686 if (evergreen_average_bandwidth(wm) <=
687 (evergreen_available_bandwidth(wm) / wm->num_heads))
688 return true;
689 else
690 return false;
691};
692
693static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
694{
695 u32 lb_partitions = wm->lb_size / wm->src_width;
696 u32 line_time = wm->active_time + wm->blank_time;
697 u32 latency_tolerant_lines;
698 u32 latency_hiding;
699 fixed20_12 a;
700
701 a.full = dfixed_const(1);
702 if (wm->vsc.full > a.full)
703 latency_tolerant_lines = 1;
704 else {
705 if (lb_partitions <= (wm->vtaps + 1))
706 latency_tolerant_lines = 1;
707 else
708 latency_tolerant_lines = 2;
709 }
710
711 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
712
713 if (evergreen_latency_watermark(wm) <= latency_hiding)
714 return true;
715 else
716 return false;
717}
718
719static void evergreen_program_watermarks(struct radeon_device *rdev,
720 struct radeon_crtc *radeon_crtc,
721 u32 lb_size, u32 num_heads)
722{
723 struct drm_display_mode *mode = &radeon_crtc->base.mode;
724 struct evergreen_wm_params wm;
725 u32 pixel_period;
726 u32 line_time = 0;
727 u32 latency_watermark_a = 0, latency_watermark_b = 0;
728 u32 priority_a_mark = 0, priority_b_mark = 0;
729 u32 priority_a_cnt = PRIORITY_OFF;
730 u32 priority_b_cnt = PRIORITY_OFF;
731 u32 pipe_offset = radeon_crtc->crtc_id * 16;
732 u32 tmp, arb_control3;
733 fixed20_12 a, b, c;
734
735 if (radeon_crtc->base.enabled && num_heads && mode) {
736 pixel_period = 1000000 / (u32)mode->clock;
737 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
738 priority_a_cnt = 0;
739 priority_b_cnt = 0;
740
741 wm.yclk = rdev->pm.current_mclk * 10;
742 wm.sclk = rdev->pm.current_sclk * 10;
743 wm.disp_clk = mode->clock;
744 wm.src_width = mode->crtc_hdisplay;
745 wm.active_time = mode->crtc_hdisplay * pixel_period;
746 wm.blank_time = line_time - wm.active_time;
747 wm.interlaced = false;
748 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
749 wm.interlaced = true;
750 wm.vsc = radeon_crtc->vsc;
751 wm.vtaps = 1;
752 if (radeon_crtc->rmx_type != RMX_OFF)
753 wm.vtaps = 2;
754 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
755 wm.lb_size = lb_size;
756 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
757 wm.num_heads = num_heads;
758
759 /* set for high clocks */
760 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
761 /* set for low clocks */
762 /* wm.yclk = low clk; wm.sclk = low clk */
763 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
764
765 /* possibly force display priority to high */
766 /* should really do this at mode validation time... */
767 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
768 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
769 !evergreen_check_latency_hiding(&wm) ||
770 (rdev->disp_priority == 2)) {
771 DRM_DEBUG_KMS("force priority to high\n");
772 priority_a_cnt |= PRIORITY_ALWAYS_ON;
773 priority_b_cnt |= PRIORITY_ALWAYS_ON;
774 }
775
776 a.full = dfixed_const(1000);
777 b.full = dfixed_const(mode->clock);
778 b.full = dfixed_div(b, a);
779 c.full = dfixed_const(latency_watermark_a);
780 c.full = dfixed_mul(c, b);
781 c.full = dfixed_mul(c, radeon_crtc->hsc);
782 c.full = dfixed_div(c, a);
783 a.full = dfixed_const(16);
784 c.full = dfixed_div(c, a);
785 priority_a_mark = dfixed_trunc(c);
786 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
787
788 a.full = dfixed_const(1000);
789 b.full = dfixed_const(mode->clock);
790 b.full = dfixed_div(b, a);
791 c.full = dfixed_const(latency_watermark_b);
792 c.full = dfixed_mul(c, b);
793 c.full = dfixed_mul(c, radeon_crtc->hsc);
794 c.full = dfixed_div(c, a);
795 a.full = dfixed_const(16);
796 c.full = dfixed_div(c, a);
797 priority_b_mark = dfixed_trunc(c);
798 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
799 }
800
801 /* select wm A */
802 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
803 tmp = arb_control3;
804 tmp &= ~LATENCY_WATERMARK_MASK(3);
805 tmp |= LATENCY_WATERMARK_MASK(1);
806 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
807 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
808 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
809 LATENCY_HIGH_WATERMARK(line_time)));
810 /* select wm B */
811 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
812 tmp &= ~LATENCY_WATERMARK_MASK(3);
813 tmp |= LATENCY_WATERMARK_MASK(2);
814 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
815 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
816 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
817 LATENCY_HIGH_WATERMARK(line_time)));
818 /* restore original selection */
819 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
820
821 /* write the priority marks */
822 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
823 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
824
825}
826
827void evergreen_bandwidth_update(struct radeon_device *rdev)
828{
829 struct drm_display_mode *mode0 = NULL;
830 struct drm_display_mode *mode1 = NULL;
831 u32 num_heads = 0, lb_size;
832 int i;
833
834 radeon_update_display_priority(rdev);
835
836 for (i = 0; i < rdev->num_crtc; i++) {
837 if (rdev->mode_info.crtcs[i]->base.enabled)
838 num_heads++;
839 }
840 for (i = 0; i < rdev->num_crtc; i += 2) {
841 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
842 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
843 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
844 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
845 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
846 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
847 }
848}
849
850int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
851{
852 unsigned i;
853 u32 tmp;
854
855 for (i = 0; i < rdev->usec_timeout; i++) {
856 /* read MC_STATUS */
857 tmp = RREG32(SRBM_STATUS) & 0x1F00;
858 if (!tmp)
859 return 0;
860 udelay(1);
861 }
862 return -1;
863}
864
865/*
866 * GART
867 */
868void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
869{
870 unsigned i;
871 u32 tmp;
872
873 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
874
875 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
876 for (i = 0; i < rdev->usec_timeout; i++) {
877 /* read MC_STATUS */
878 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
879 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
880 if (tmp == 2) {
881 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
882 return;
883 }
884 if (tmp) {
885 return;
886 }
887 udelay(1);
888 }
889}
890
891int evergreen_pcie_gart_enable(struct radeon_device *rdev)
892{
893 u32 tmp;
894 int r;
895
896 if (rdev->gart.table.vram.robj == NULL) {
897 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
898 return -EINVAL;
899 }
900 r = radeon_gart_table_vram_pin(rdev);
901 if (r)
902 return r;
903 radeon_gart_restore(rdev);
904 /* Setup L2 cache */
905 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
906 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
907 EFFECTIVE_L2_QUEUE_SIZE(7));
908 WREG32(VM_L2_CNTL2, 0);
909 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
910 /* Setup TLB control */
911 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
912 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
913 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
914 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
915 if (rdev->flags & RADEON_IS_IGP) {
916 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
917 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
918 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
919 } else {
920 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
921 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
922 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
923 }
924 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
925 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
926 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
927 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
928 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
929 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
930 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
931 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
932 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
933 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
934 (u32)(rdev->dummy_page.addr >> 12));
935 WREG32(VM_CONTEXT1_CNTL, 0);
936
937 evergreen_pcie_gart_tlb_flush(rdev);
938 rdev->gart.ready = true;
939 return 0;
940}
941
942void evergreen_pcie_gart_disable(struct radeon_device *rdev)
943{
944 u32 tmp;
945 int r;
946
947 /* Disable all tables */
948 WREG32(VM_CONTEXT0_CNTL, 0);
949 WREG32(VM_CONTEXT1_CNTL, 0);
950
951 /* Setup L2 cache */
952 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
953 EFFECTIVE_L2_QUEUE_SIZE(7));
954 WREG32(VM_L2_CNTL2, 0);
955 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
956 /* Setup TLB control */
957 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
958 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
959 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
960 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
961 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
962 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
963 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
964 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
965 if (rdev->gart.table.vram.robj) {
966 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
967 if (likely(r == 0)) {
968 radeon_bo_kunmap(rdev->gart.table.vram.robj);
969 radeon_bo_unpin(rdev->gart.table.vram.robj);
970 radeon_bo_unreserve(rdev->gart.table.vram.robj);
971 }
972 }
973}
974
975void evergreen_pcie_gart_fini(struct radeon_device *rdev)
976{
977 evergreen_pcie_gart_disable(rdev);
978 radeon_gart_table_vram_free(rdev);
979 radeon_gart_fini(rdev);
980}
981
982
983void evergreen_agp_enable(struct radeon_device *rdev)
984{
985 u32 tmp;
986
987 /* Setup L2 cache */
988 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
989 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
990 EFFECTIVE_L2_QUEUE_SIZE(7));
991 WREG32(VM_L2_CNTL2, 0);
992 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
993 /* Setup TLB control */
994 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
995 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
996 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
997 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
998 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
999 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1000 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1001 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1002 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1003 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1004 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1005 WREG32(VM_CONTEXT0_CNTL, 0);
1006 WREG32(VM_CONTEXT1_CNTL, 0);
1007}
1008
1009void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1010{
1011 save->vga_control[0] = RREG32(D1VGA_CONTROL);
1012 save->vga_control[1] = RREG32(D2VGA_CONTROL);
1013 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
1014 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
1015 save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
1016 save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
1017 if (rdev->num_crtc >= 4) {
1018 save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
1019 save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
1020 save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
1021 save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
1022 }
1023 if (rdev->num_crtc >= 6) {
1024 save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
1025 save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
1026 save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
1027 save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1028 }
1029
1030 /* Stop all video */
1031 WREG32(VGA_RENDER_CONTROL, 0);
1032 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1033 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1034 if (rdev->num_crtc >= 4) {
1035 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1036 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1037 }
1038 if (rdev->num_crtc >= 6) {
1039 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1040 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1041 }
1042 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1043 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1044 if (rdev->num_crtc >= 4) {
1045 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1046 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1047 }
1048 if (rdev->num_crtc >= 6) {
1049 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1050 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1051 }
1052 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1053 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1054 if (rdev->num_crtc >= 4) {
1055 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1056 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1057 }
1058 if (rdev->num_crtc >= 6) {
1059 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1060 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1061 }
1062
1063 WREG32(D1VGA_CONTROL, 0);
1064 WREG32(D2VGA_CONTROL, 0);
1065 if (rdev->num_crtc >= 4) {
1066 WREG32(EVERGREEN_D3VGA_CONTROL, 0);
1067 WREG32(EVERGREEN_D4VGA_CONTROL, 0);
1068 }
1069 if (rdev->num_crtc >= 6) {
1070 WREG32(EVERGREEN_D5VGA_CONTROL, 0);
1071 WREG32(EVERGREEN_D6VGA_CONTROL, 0);
1072 }
1073}
1074
1075void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1076{
1077 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1078 upper_32_bits(rdev->mc.vram_start));
1079 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1080 upper_32_bits(rdev->mc.vram_start));
1081 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1082 (u32)rdev->mc.vram_start);
1083 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1084 (u32)rdev->mc.vram_start);
1085
1086 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1087 upper_32_bits(rdev->mc.vram_start));
1088 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1089 upper_32_bits(rdev->mc.vram_start));
1090 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1091 (u32)rdev->mc.vram_start);
1092 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1093 (u32)rdev->mc.vram_start);
1094
1095 if (rdev->num_crtc >= 4) {
1096 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1097 upper_32_bits(rdev->mc.vram_start));
1098 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1099 upper_32_bits(rdev->mc.vram_start));
1100 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1101 (u32)rdev->mc.vram_start);
1102 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1103 (u32)rdev->mc.vram_start);
1104
1105 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1106 upper_32_bits(rdev->mc.vram_start));
1107 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1108 upper_32_bits(rdev->mc.vram_start));
1109 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1110 (u32)rdev->mc.vram_start);
1111 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1112 (u32)rdev->mc.vram_start);
1113 }
1114 if (rdev->num_crtc >= 6) {
1115 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1116 upper_32_bits(rdev->mc.vram_start));
1117 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1118 upper_32_bits(rdev->mc.vram_start));
1119 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1120 (u32)rdev->mc.vram_start);
1121 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1122 (u32)rdev->mc.vram_start);
1123
1124 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1125 upper_32_bits(rdev->mc.vram_start));
1126 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1127 upper_32_bits(rdev->mc.vram_start));
1128 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1129 (u32)rdev->mc.vram_start);
1130 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1131 (u32)rdev->mc.vram_start);
1132 }
1133
1134 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1135 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1136 /* Unlock host access */
1137 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1138 mdelay(1);
1139 /* Restore video state */
1140 WREG32(D1VGA_CONTROL, save->vga_control[0]);
1141 WREG32(D2VGA_CONTROL, save->vga_control[1]);
1142 if (rdev->num_crtc >= 4) {
1143 WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
1144 WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
1145 }
1146 if (rdev->num_crtc >= 6) {
1147 WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
1148 WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
1149 }
1150 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1151 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1152 if (rdev->num_crtc >= 4) {
1153 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1154 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1155 }
1156 if (rdev->num_crtc >= 6) {
1157 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1158 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1159 }
1160 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
1161 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1162 if (rdev->num_crtc >= 4) {
1163 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
1164 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
1165 }
1166 if (rdev->num_crtc >= 6) {
1167 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
1168 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1169 }
1170 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1171 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1172 if (rdev->num_crtc >= 4) {
1173 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1174 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1175 }
1176 if (rdev->num_crtc >= 6) {
1177 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1178 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1179 }
1180 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1181}
1182
1183void evergreen_mc_program(struct radeon_device *rdev)
1184{
1185 struct evergreen_mc_save save;
1186 u32 tmp;
1187 int i, j;
1188
1189 /* Initialize HDP */
1190 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1191 WREG32((0x2c14 + j), 0x00000000);
1192 WREG32((0x2c18 + j), 0x00000000);
1193 WREG32((0x2c1c + j), 0x00000000);
1194 WREG32((0x2c20 + j), 0x00000000);
1195 WREG32((0x2c24 + j), 0x00000000);
1196 }
1197 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1198
1199 evergreen_mc_stop(rdev, &save);
1200 if (evergreen_mc_wait_for_idle(rdev)) {
1201 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1202 }
1203 /* Lockout access through VGA aperture*/
1204 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1205 /* Update configuration */
1206 if (rdev->flags & RADEON_IS_AGP) {
1207 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1208 /* VRAM before AGP */
1209 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1210 rdev->mc.vram_start >> 12);
1211 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1212 rdev->mc.gtt_end >> 12);
1213 } else {
1214 /* VRAM after AGP */
1215 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1216 rdev->mc.gtt_start >> 12);
1217 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1218 rdev->mc.vram_end >> 12);
1219 }
1220 } else {
1221 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1222 rdev->mc.vram_start >> 12);
1223 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1224 rdev->mc.vram_end >> 12);
1225 }
1226 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1227 if (rdev->flags & RADEON_IS_IGP) {
1228 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1229 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1230 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1231 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1232 }
1233 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1234 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1235 WREG32(MC_VM_FB_LOCATION, tmp);
1236 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1237 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1238 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1239 if (rdev->flags & RADEON_IS_AGP) {
1240 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1241 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1242 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1243 } else {
1244 WREG32(MC_VM_AGP_BASE, 0);
1245 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1246 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1247 }
1248 if (evergreen_mc_wait_for_idle(rdev)) {
1249 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1250 }
1251 evergreen_mc_resume(rdev, &save);
1252 /* we need to own VRAM, so turn off the VGA renderer here
1253 * to stop it overwriting our objects */
1254 rv515_vga_render_disable(rdev);
1255}
1256
1257/*
1258 * CP.
1259 */
1260void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1261{
1262 /* set to DX10/11 mode */
1263 radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
1264 radeon_ring_write(rdev, 1);
1265 /* FIXME: implement */
1266 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1267 radeon_ring_write(rdev,
1268#ifdef __BIG_ENDIAN
1269 (2 << 0) |
1270#endif
1271 (ib->gpu_addr & 0xFFFFFFFC));
1272 radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
1273 radeon_ring_write(rdev, ib->length_dw);
1274}
1275
1276
1277static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1278{
1279 const __be32 *fw_data;
1280 int i;
1281
1282 if (!rdev->me_fw || !rdev->pfp_fw)
1283 return -EINVAL;
1284
1285 r700_cp_stop(rdev);
1286 WREG32(CP_RB_CNTL,
1287#ifdef __BIG_ENDIAN
1288 BUF_SWAP_32BIT |
1289#endif
1290 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1291
1292 fw_data = (const __be32 *)rdev->pfp_fw->data;
1293 WREG32(CP_PFP_UCODE_ADDR, 0);
1294 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1295 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1296 WREG32(CP_PFP_UCODE_ADDR, 0);
1297
1298 fw_data = (const __be32 *)rdev->me_fw->data;
1299 WREG32(CP_ME_RAM_WADDR, 0);
1300 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1301 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1302
1303 WREG32(CP_PFP_UCODE_ADDR, 0);
1304 WREG32(CP_ME_RAM_WADDR, 0);
1305 WREG32(CP_ME_RAM_RADDR, 0);
1306 return 0;
1307}
1308
1309static int evergreen_cp_start(struct radeon_device *rdev)
1310{
1311 int r, i;
1312 uint32_t cp_me;
1313
1314 r = radeon_ring_lock(rdev, 7);
1315 if (r) {
1316 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1317 return r;
1318 }
1319 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1320 radeon_ring_write(rdev, 0x1);
1321 radeon_ring_write(rdev, 0x0);
1322 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1323 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1324 radeon_ring_write(rdev, 0);
1325 radeon_ring_write(rdev, 0);
1326 radeon_ring_unlock_commit(rdev);
1327
1328 cp_me = 0xff;
1329 WREG32(CP_ME_CNTL, cp_me);
1330
1331 r = radeon_ring_lock(rdev, evergreen_default_size + 19);
1332 if (r) {
1333 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1334 return r;
1335 }
1336
1337 /* setup clear context state */
1338 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1339 radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1340
1341 for (i = 0; i < evergreen_default_size; i++)
1342 radeon_ring_write(rdev, evergreen_default_state[i]);
1343
1344 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1345 radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
1346
1347 /* set clear context state */
1348 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
1349 radeon_ring_write(rdev, 0);
1350
1351 /* SQ_VTX_BASE_VTX_LOC */
1352 radeon_ring_write(rdev, 0xc0026f00);
1353 radeon_ring_write(rdev, 0x00000000);
1354 radeon_ring_write(rdev, 0x00000000);
1355 radeon_ring_write(rdev, 0x00000000);
1356
1357 /* Clear consts */
1358 radeon_ring_write(rdev, 0xc0036f00);
1359 radeon_ring_write(rdev, 0x00000bc4);
1360 radeon_ring_write(rdev, 0xffffffff);
1361 radeon_ring_write(rdev, 0xffffffff);
1362 radeon_ring_write(rdev, 0xffffffff);
1363
1364 radeon_ring_write(rdev, 0xc0026900);
1365 radeon_ring_write(rdev, 0x00000316);
1366 radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1367 radeon_ring_write(rdev, 0x00000010); /* */
1368
1369 radeon_ring_unlock_commit(rdev);
1370
1371 return 0;
1372}
1373
1374int evergreen_cp_resume(struct radeon_device *rdev)
1375{
1376 u32 tmp;
1377 u32 rb_bufsz;
1378 int r;
1379
1380 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1381 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1382 SOFT_RESET_PA |
1383 SOFT_RESET_SH |
1384 SOFT_RESET_VGT |
1385 SOFT_RESET_SPI |
1386 SOFT_RESET_SX));
1387 RREG32(GRBM_SOFT_RESET);
1388 mdelay(15);
1389 WREG32(GRBM_SOFT_RESET, 0);
1390 RREG32(GRBM_SOFT_RESET);
1391
1392 /* Set ring buffer size */
1393 rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1394 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1395#ifdef __BIG_ENDIAN
1396 tmp |= BUF_SWAP_32BIT;
1397#endif
1398 WREG32(CP_RB_CNTL, tmp);
1399 WREG32(CP_SEM_WAIT_TIMER, 0x4);
1400
1401 /* Set the write pointer delay */
1402 WREG32(CP_RB_WPTR_DELAY, 0);
1403
1404 /* Initialize the ring buffer's read and write pointers */
1405 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1406 WREG32(CP_RB_RPTR_WR, 0);
1407 rdev->cp.wptr = 0;
1408 WREG32(CP_RB_WPTR, rdev->cp.wptr);
1409
1410 /* set the wb address wether it's enabled or not */
1411 WREG32(CP_RB_RPTR_ADDR,
1412 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1413 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1414 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1415
1416 if (rdev->wb.enabled)
1417 WREG32(SCRATCH_UMSK, 0xff);
1418 else {
1419 tmp |= RB_NO_UPDATE;
1420 WREG32(SCRATCH_UMSK, 0);
1421 }
1422
1423 mdelay(1);
1424 WREG32(CP_RB_CNTL, tmp);
1425
1426 WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1427 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1428
1429 rdev->cp.rptr = RREG32(CP_RB_RPTR);
1430
1431 evergreen_cp_start(rdev);
1432 rdev->cp.ready = true;
1433 r = radeon_ring_test(rdev);
1434 if (r) {
1435 rdev->cp.ready = false;
1436 return r;
1437 }
1438 return 0;
1439}
1440
1441/*
1442 * Core functions
1443 */
1444static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1445 u32 num_tile_pipes,
1446 u32 num_backends,
1447 u32 backend_disable_mask)
1448{
1449 u32 backend_map = 0;
1450 u32 enabled_backends_mask = 0;
1451 u32 enabled_backends_count = 0;
1452 u32 cur_pipe;
1453 u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
1454 u32 cur_backend = 0;
1455 u32 i;
1456 bool force_no_swizzle;
1457
1458 if (num_tile_pipes > EVERGREEN_MAX_PIPES)
1459 num_tile_pipes = EVERGREEN_MAX_PIPES;
1460 if (num_tile_pipes < 1)
1461 num_tile_pipes = 1;
1462 if (num_backends > EVERGREEN_MAX_BACKENDS)
1463 num_backends = EVERGREEN_MAX_BACKENDS;
1464 if (num_backends < 1)
1465 num_backends = 1;
1466
1467 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1468 if (((backend_disable_mask >> i) & 1) == 0) {
1469 enabled_backends_mask |= (1 << i);
1470 ++enabled_backends_count;
1471 }
1472 if (enabled_backends_count == num_backends)
1473 break;
1474 }
1475
1476 if (enabled_backends_count == 0) {
1477 enabled_backends_mask = 1;
1478 enabled_backends_count = 1;
1479 }
1480
1481 if (enabled_backends_count != num_backends)
1482 num_backends = enabled_backends_count;
1483
1484 memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * EVERGREEN_MAX_PIPES);
1485 switch (rdev->family) {
1486 case CHIP_CEDAR:
1487 case CHIP_REDWOOD:
1488 case CHIP_PALM:
1489 case CHIP_SUMO:
1490 case CHIP_SUMO2:
1491 case CHIP_TURKS:
1492 case CHIP_CAICOS:
1493 force_no_swizzle = false;
1494 break;
1495 case CHIP_CYPRESS:
1496 case CHIP_HEMLOCK:
1497 case CHIP_JUNIPER:
1498 case CHIP_BARTS:
1499 default:
1500 force_no_swizzle = true;
1501 break;
1502 }
1503 if (force_no_swizzle) {
1504 bool last_backend_enabled = false;
1505
1506 force_no_swizzle = false;
1507 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1508 if (((enabled_backends_mask >> i) & 1) == 1) {
1509 if (last_backend_enabled)
1510 force_no_swizzle = true;
1511 last_backend_enabled = true;
1512 } else
1513 last_backend_enabled = false;
1514 }
1515 }
1516
1517 switch (num_tile_pipes) {
1518 case 1:
1519 case 3:
1520 case 5:
1521 case 7:
1522 DRM_ERROR("odd number of pipes!\n");
1523 break;
1524 case 2:
1525 swizzle_pipe[0] = 0;
1526 swizzle_pipe[1] = 1;
1527 break;
1528 case 4:
1529 if (force_no_swizzle) {
1530 swizzle_pipe[0] = 0;
1531 swizzle_pipe[1] = 1;
1532 swizzle_pipe[2] = 2;
1533 swizzle_pipe[3] = 3;
1534 } else {
1535 swizzle_pipe[0] = 0;
1536 swizzle_pipe[1] = 2;
1537 swizzle_pipe[2] = 1;
1538 swizzle_pipe[3] = 3;
1539 }
1540 break;
1541 case 6:
1542 if (force_no_swizzle) {
1543 swizzle_pipe[0] = 0;
1544 swizzle_pipe[1] = 1;
1545 swizzle_pipe[2] = 2;
1546 swizzle_pipe[3] = 3;
1547 swizzle_pipe[4] = 4;
1548 swizzle_pipe[5] = 5;
1549 } else {
1550 swizzle_pipe[0] = 0;
1551 swizzle_pipe[1] = 2;
1552 swizzle_pipe[2] = 4;
1553 swizzle_pipe[3] = 1;
1554 swizzle_pipe[4] = 3;
1555 swizzle_pipe[5] = 5;
1556 }
1557 break;
1558 case 8:
1559 if (force_no_swizzle) {
1560 swizzle_pipe[0] = 0;
1561 swizzle_pipe[1] = 1;
1562 swizzle_pipe[2] = 2;
1563 swizzle_pipe[3] = 3;
1564 swizzle_pipe[4] = 4;
1565 swizzle_pipe[5] = 5;
1566 swizzle_pipe[6] = 6;
1567 swizzle_pipe[7] = 7;
1568 } else {
1569 swizzle_pipe[0] = 0;
1570 swizzle_pipe[1] = 2;
1571 swizzle_pipe[2] = 4;
1572 swizzle_pipe[3] = 6;
1573 swizzle_pipe[4] = 1;
1574 swizzle_pipe[5] = 3;
1575 swizzle_pipe[6] = 5;
1576 swizzle_pipe[7] = 7;
1577 }
1578 break;
1579 }
1580
1581 for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1582 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1583 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1584
1585 backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
1586
1587 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1588 }
1589
1590 return backend_map;
1591}
1592
1593static void evergreen_gpu_init(struct radeon_device *rdev)
1594{
1595 u32 cc_rb_backend_disable = 0;
1596 u32 cc_gc_shader_pipe_config;
1597 u32 gb_addr_config = 0;
1598 u32 mc_shared_chmap, mc_arb_ramcfg;
1599 u32 gb_backend_map;
1600 u32 grbm_gfx_index;
1601 u32 sx_debug_1;
1602 u32 smx_dc_ctl0;
1603 u32 sq_config;
1604 u32 sq_lds_resource_mgmt;
1605 u32 sq_gpr_resource_mgmt_1;
1606 u32 sq_gpr_resource_mgmt_2;
1607 u32 sq_gpr_resource_mgmt_3;
1608 u32 sq_thread_resource_mgmt;
1609 u32 sq_thread_resource_mgmt_2;
1610 u32 sq_stack_resource_mgmt_1;
1611 u32 sq_stack_resource_mgmt_2;
1612 u32 sq_stack_resource_mgmt_3;
1613 u32 vgt_cache_invalidation;
1614 u32 hdp_host_path_cntl, tmp;
1615 int i, j, num_shader_engines, ps_thread_count;
1616
1617 switch (rdev->family) {
1618 case CHIP_CYPRESS:
1619 case CHIP_HEMLOCK:
1620 rdev->config.evergreen.num_ses = 2;
1621 rdev->config.evergreen.max_pipes = 4;
1622 rdev->config.evergreen.max_tile_pipes = 8;
1623 rdev->config.evergreen.max_simds = 10;
1624 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1625 rdev->config.evergreen.max_gprs = 256;
1626 rdev->config.evergreen.max_threads = 248;
1627 rdev->config.evergreen.max_gs_threads = 32;
1628 rdev->config.evergreen.max_stack_entries = 512;
1629 rdev->config.evergreen.sx_num_of_sets = 4;
1630 rdev->config.evergreen.sx_max_export_size = 256;
1631 rdev->config.evergreen.sx_max_export_pos_size = 64;
1632 rdev->config.evergreen.sx_max_export_smx_size = 192;
1633 rdev->config.evergreen.max_hw_contexts = 8;
1634 rdev->config.evergreen.sq_num_cf_insts = 2;
1635
1636 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1637 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1638 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1639 break;
1640 case CHIP_JUNIPER:
1641 rdev->config.evergreen.num_ses = 1;
1642 rdev->config.evergreen.max_pipes = 4;
1643 rdev->config.evergreen.max_tile_pipes = 4;
1644 rdev->config.evergreen.max_simds = 10;
1645 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1646 rdev->config.evergreen.max_gprs = 256;
1647 rdev->config.evergreen.max_threads = 248;
1648 rdev->config.evergreen.max_gs_threads = 32;
1649 rdev->config.evergreen.max_stack_entries = 512;
1650 rdev->config.evergreen.sx_num_of_sets = 4;
1651 rdev->config.evergreen.sx_max_export_size = 256;
1652 rdev->config.evergreen.sx_max_export_pos_size = 64;
1653 rdev->config.evergreen.sx_max_export_smx_size = 192;
1654 rdev->config.evergreen.max_hw_contexts = 8;
1655 rdev->config.evergreen.sq_num_cf_insts = 2;
1656
1657 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1658 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1659 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1660 break;
1661 case CHIP_REDWOOD:
1662 rdev->config.evergreen.num_ses = 1;
1663 rdev->config.evergreen.max_pipes = 4;
1664 rdev->config.evergreen.max_tile_pipes = 4;
1665 rdev->config.evergreen.max_simds = 5;
1666 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1667 rdev->config.evergreen.max_gprs = 256;
1668 rdev->config.evergreen.max_threads = 248;
1669 rdev->config.evergreen.max_gs_threads = 32;
1670 rdev->config.evergreen.max_stack_entries = 256;
1671 rdev->config.evergreen.sx_num_of_sets = 4;
1672 rdev->config.evergreen.sx_max_export_size = 256;
1673 rdev->config.evergreen.sx_max_export_pos_size = 64;
1674 rdev->config.evergreen.sx_max_export_smx_size = 192;
1675 rdev->config.evergreen.max_hw_contexts = 8;
1676 rdev->config.evergreen.sq_num_cf_insts = 2;
1677
1678 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1679 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1680 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1681 break;
1682 case CHIP_CEDAR:
1683 default:
1684 rdev->config.evergreen.num_ses = 1;
1685 rdev->config.evergreen.max_pipes = 2;
1686 rdev->config.evergreen.max_tile_pipes = 2;
1687 rdev->config.evergreen.max_simds = 2;
1688 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1689 rdev->config.evergreen.max_gprs = 256;
1690 rdev->config.evergreen.max_threads = 192;
1691 rdev->config.evergreen.max_gs_threads = 16;
1692 rdev->config.evergreen.max_stack_entries = 256;
1693 rdev->config.evergreen.sx_num_of_sets = 4;
1694 rdev->config.evergreen.sx_max_export_size = 128;
1695 rdev->config.evergreen.sx_max_export_pos_size = 32;
1696 rdev->config.evergreen.sx_max_export_smx_size = 96;
1697 rdev->config.evergreen.max_hw_contexts = 4;
1698 rdev->config.evergreen.sq_num_cf_insts = 1;
1699
1700 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1701 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1702 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1703 break;
1704 case CHIP_PALM:
1705 rdev->config.evergreen.num_ses = 1;
1706 rdev->config.evergreen.max_pipes = 2;
1707 rdev->config.evergreen.max_tile_pipes = 2;
1708 rdev->config.evergreen.max_simds = 2;
1709 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1710 rdev->config.evergreen.max_gprs = 256;
1711 rdev->config.evergreen.max_threads = 192;
1712 rdev->config.evergreen.max_gs_threads = 16;
1713 rdev->config.evergreen.max_stack_entries = 256;
1714 rdev->config.evergreen.sx_num_of_sets = 4;
1715 rdev->config.evergreen.sx_max_export_size = 128;
1716 rdev->config.evergreen.sx_max_export_pos_size = 32;
1717 rdev->config.evergreen.sx_max_export_smx_size = 96;
1718 rdev->config.evergreen.max_hw_contexts = 4;
1719 rdev->config.evergreen.sq_num_cf_insts = 1;
1720
1721 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1722 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1723 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1724 break;
1725 case CHIP_SUMO:
1726 rdev->config.evergreen.num_ses = 1;
1727 rdev->config.evergreen.max_pipes = 4;
1728 rdev->config.evergreen.max_tile_pipes = 2;
1729 if (rdev->pdev->device == 0x9648)
1730 rdev->config.evergreen.max_simds = 3;
1731 else if ((rdev->pdev->device == 0x9647) ||
1732 (rdev->pdev->device == 0x964a))
1733 rdev->config.evergreen.max_simds = 4;
1734 else
1735 rdev->config.evergreen.max_simds = 5;
1736 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1737 rdev->config.evergreen.max_gprs = 256;
1738 rdev->config.evergreen.max_threads = 248;
1739 rdev->config.evergreen.max_gs_threads = 32;
1740 rdev->config.evergreen.max_stack_entries = 256;
1741 rdev->config.evergreen.sx_num_of_sets = 4;
1742 rdev->config.evergreen.sx_max_export_size = 256;
1743 rdev->config.evergreen.sx_max_export_pos_size = 64;
1744 rdev->config.evergreen.sx_max_export_smx_size = 192;
1745 rdev->config.evergreen.max_hw_contexts = 8;
1746 rdev->config.evergreen.sq_num_cf_insts = 2;
1747
1748 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1749 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1750 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1751 break;
1752 case CHIP_SUMO2:
1753 rdev->config.evergreen.num_ses = 1;
1754 rdev->config.evergreen.max_pipes = 4;
1755 rdev->config.evergreen.max_tile_pipes = 4;
1756 rdev->config.evergreen.max_simds = 2;
1757 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1758 rdev->config.evergreen.max_gprs = 256;
1759 rdev->config.evergreen.max_threads = 248;
1760 rdev->config.evergreen.max_gs_threads = 32;
1761 rdev->config.evergreen.max_stack_entries = 512;
1762 rdev->config.evergreen.sx_num_of_sets = 4;
1763 rdev->config.evergreen.sx_max_export_size = 256;
1764 rdev->config.evergreen.sx_max_export_pos_size = 64;
1765 rdev->config.evergreen.sx_max_export_smx_size = 192;
1766 rdev->config.evergreen.max_hw_contexts = 8;
1767 rdev->config.evergreen.sq_num_cf_insts = 2;
1768
1769 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1770 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1771 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1772 break;
1773 case CHIP_BARTS:
1774 rdev->config.evergreen.num_ses = 2;
1775 rdev->config.evergreen.max_pipes = 4;
1776 rdev->config.evergreen.max_tile_pipes = 8;
1777 rdev->config.evergreen.max_simds = 7;
1778 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1779 rdev->config.evergreen.max_gprs = 256;
1780 rdev->config.evergreen.max_threads = 248;
1781 rdev->config.evergreen.max_gs_threads = 32;
1782 rdev->config.evergreen.max_stack_entries = 512;
1783 rdev->config.evergreen.sx_num_of_sets = 4;
1784 rdev->config.evergreen.sx_max_export_size = 256;
1785 rdev->config.evergreen.sx_max_export_pos_size = 64;
1786 rdev->config.evergreen.sx_max_export_smx_size = 192;
1787 rdev->config.evergreen.max_hw_contexts = 8;
1788 rdev->config.evergreen.sq_num_cf_insts = 2;
1789
1790 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1791 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1792 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1793 break;
1794 case CHIP_TURKS:
1795 rdev->config.evergreen.num_ses = 1;
1796 rdev->config.evergreen.max_pipes = 4;
1797 rdev->config.evergreen.max_tile_pipes = 4;
1798 rdev->config.evergreen.max_simds = 6;
1799 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1800 rdev->config.evergreen.max_gprs = 256;
1801 rdev->config.evergreen.max_threads = 248;
1802 rdev->config.evergreen.max_gs_threads = 32;
1803 rdev->config.evergreen.max_stack_entries = 256;
1804 rdev->config.evergreen.sx_num_of_sets = 4;
1805 rdev->config.evergreen.sx_max_export_size = 256;
1806 rdev->config.evergreen.sx_max_export_pos_size = 64;
1807 rdev->config.evergreen.sx_max_export_smx_size = 192;
1808 rdev->config.evergreen.max_hw_contexts = 8;
1809 rdev->config.evergreen.sq_num_cf_insts = 2;
1810
1811 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1812 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1813 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1814 break;
1815 case CHIP_CAICOS:
1816 rdev->config.evergreen.num_ses = 1;
1817 rdev->config.evergreen.max_pipes = 4;
1818 rdev->config.evergreen.max_tile_pipes = 2;
1819 rdev->config.evergreen.max_simds = 2;
1820 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1821 rdev->config.evergreen.max_gprs = 256;
1822 rdev->config.evergreen.max_threads = 192;
1823 rdev->config.evergreen.max_gs_threads = 16;
1824 rdev->config.evergreen.max_stack_entries = 256;
1825 rdev->config.evergreen.sx_num_of_sets = 4;
1826 rdev->config.evergreen.sx_max_export_size = 128;
1827 rdev->config.evergreen.sx_max_export_pos_size = 32;
1828 rdev->config.evergreen.sx_max_export_smx_size = 96;
1829 rdev->config.evergreen.max_hw_contexts = 4;
1830 rdev->config.evergreen.sq_num_cf_insts = 1;
1831
1832 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1833 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1834 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1835 break;
1836 }
1837
1838 /* Initialize HDP */
1839 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1840 WREG32((0x2c14 + j), 0x00000000);
1841 WREG32((0x2c18 + j), 0x00000000);
1842 WREG32((0x2c1c + j), 0x00000000);
1843 WREG32((0x2c20 + j), 0x00000000);
1844 WREG32((0x2c24 + j), 0x00000000);
1845 }
1846
1847 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1848
1849 evergreen_fix_pci_max_read_req_size(rdev);
1850
1851 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
1852
1853 cc_gc_shader_pipe_config |=
1854 INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
1855 & EVERGREEN_MAX_PIPES_MASK);
1856 cc_gc_shader_pipe_config |=
1857 INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
1858 & EVERGREEN_MAX_SIMDS_MASK);
1859
1860 cc_rb_backend_disable =
1861 BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
1862 & EVERGREEN_MAX_BACKENDS_MASK);
1863
1864
1865 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1866 if (rdev->flags & RADEON_IS_IGP)
1867 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
1868 else
1869 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1870
1871 switch (rdev->config.evergreen.max_tile_pipes) {
1872 case 1:
1873 default:
1874 gb_addr_config |= NUM_PIPES(0);
1875 break;
1876 case 2:
1877 gb_addr_config |= NUM_PIPES(1);
1878 break;
1879 case 4:
1880 gb_addr_config |= NUM_PIPES(2);
1881 break;
1882 case 8:
1883 gb_addr_config |= NUM_PIPES(3);
1884 break;
1885 }
1886
1887 gb_addr_config |= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1888 gb_addr_config |= BANK_INTERLEAVE_SIZE(0);
1889 gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
1890 gb_addr_config |= SHADER_ENGINE_TILE_SIZE(1);
1891 gb_addr_config |= NUM_GPUS(0); /* Hemlock? */
1892 gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
1893
1894 if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
1895 gb_addr_config |= ROW_SIZE(2);
1896 else
1897 gb_addr_config |= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
1898
1899 if (rdev->ddev->pdev->device == 0x689e) {
1900 u32 efuse_straps_4;
1901 u32 efuse_straps_3;
1902 u8 efuse_box_bit_131_124;
1903
1904 WREG32(RCU_IND_INDEX, 0x204);
1905 efuse_straps_4 = RREG32(RCU_IND_DATA);
1906 WREG32(RCU_IND_INDEX, 0x203);
1907 efuse_straps_3 = RREG32(RCU_IND_DATA);
1908 efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28));
1909
1910 switch(efuse_box_bit_131_124) {
1911 case 0x00:
1912 gb_backend_map = 0x76543210;
1913 break;
1914 case 0x55:
1915 gb_backend_map = 0x77553311;
1916 break;
1917 case 0x56:
1918 gb_backend_map = 0x77553300;
1919 break;
1920 case 0x59:
1921 gb_backend_map = 0x77552211;
1922 break;
1923 case 0x66:
1924 gb_backend_map = 0x77443300;
1925 break;
1926 case 0x99:
1927 gb_backend_map = 0x66552211;
1928 break;
1929 case 0x5a:
1930 gb_backend_map = 0x77552200;
1931 break;
1932 case 0xaa:
1933 gb_backend_map = 0x66442200;
1934 break;
1935 case 0x95:
1936 gb_backend_map = 0x66553311;
1937 break;
1938 default:
1939 DRM_ERROR("bad backend map, using default\n");
1940 gb_backend_map =
1941 evergreen_get_tile_pipe_to_backend_map(rdev,
1942 rdev->config.evergreen.max_tile_pipes,
1943 rdev->config.evergreen.max_backends,
1944 ((EVERGREEN_MAX_BACKENDS_MASK <<
1945 rdev->config.evergreen.max_backends) &
1946 EVERGREEN_MAX_BACKENDS_MASK));
1947 break;
1948 }
1949 } else if (rdev->ddev->pdev->device == 0x68b9) {
1950 u32 efuse_straps_3;
1951 u8 efuse_box_bit_127_124;
1952
1953 WREG32(RCU_IND_INDEX, 0x203);
1954 efuse_straps_3 = RREG32(RCU_IND_DATA);
1955 efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
1956
1957 switch(efuse_box_bit_127_124) {
1958 case 0x0:
1959 gb_backend_map = 0x00003210;
1960 break;
1961 case 0x5:
1962 case 0x6:
1963 case 0x9:
1964 case 0xa:
1965 gb_backend_map = 0x00003311;
1966 break;
1967 default:
1968 DRM_ERROR("bad backend map, using default\n");
1969 gb_backend_map =
1970 evergreen_get_tile_pipe_to_backend_map(rdev,
1971 rdev->config.evergreen.max_tile_pipes,
1972 rdev->config.evergreen.max_backends,
1973 ((EVERGREEN_MAX_BACKENDS_MASK <<
1974 rdev->config.evergreen.max_backends) &
1975 EVERGREEN_MAX_BACKENDS_MASK));
1976 break;
1977 }
1978 } else {
1979 switch (rdev->family) {
1980 case CHIP_CYPRESS:
1981 case CHIP_HEMLOCK:
1982 case CHIP_BARTS:
1983 gb_backend_map = 0x66442200;
1984 break;
1985 case CHIP_JUNIPER:
1986 gb_backend_map = 0x00002200;
1987 break;
1988 default:
1989 gb_backend_map =
1990 evergreen_get_tile_pipe_to_backend_map(rdev,
1991 rdev->config.evergreen.max_tile_pipes,
1992 rdev->config.evergreen.max_backends,
1993 ((EVERGREEN_MAX_BACKENDS_MASK <<
1994 rdev->config.evergreen.max_backends) &
1995 EVERGREEN_MAX_BACKENDS_MASK));
1996 }
1997 }
1998
1999 /* setup tiling info dword. gb_addr_config is not adequate since it does
2000 * not have bank info, so create a custom tiling dword.
2001 * bits 3:0 num_pipes
2002 * bits 7:4 num_banks
2003 * bits 11:8 group_size
2004 * bits 15:12 row_size
2005 */
2006 rdev->config.evergreen.tile_config = 0;
2007 switch (rdev->config.evergreen.max_tile_pipes) {
2008 case 1:
2009 default:
2010 rdev->config.evergreen.tile_config |= (0 << 0);
2011 break;
2012 case 2:
2013 rdev->config.evergreen.tile_config |= (1 << 0);
2014 break;
2015 case 4:
2016 rdev->config.evergreen.tile_config |= (2 << 0);
2017 break;
2018 case 8:
2019 rdev->config.evergreen.tile_config |= (3 << 0);
2020 break;
2021 }
2022 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
2023 if (rdev->flags & RADEON_IS_IGP)
2024 rdev->config.evergreen.tile_config |= 1 << 4;
2025 else
2026 rdev->config.evergreen.tile_config |=
2027 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
2028 rdev->config.evergreen.tile_config |=
2029 ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
2030 rdev->config.evergreen.tile_config |=
2031 ((gb_addr_config & 0x30000000) >> 28) << 12;
2032
2033 rdev->config.evergreen.backend_map = gb_backend_map;
2034 WREG32(GB_BACKEND_MAP, gb_backend_map);
2035 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2036 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2037 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2038
2039 num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
2040 grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
2041
2042 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
2043 u32 rb = cc_rb_backend_disable | (0xf0 << 16);
2044 u32 sp = cc_gc_shader_pipe_config;
2045 u32 gfx = grbm_gfx_index | SE_INDEX(i);
2046
2047 if (i == num_shader_engines) {
2048 rb |= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
2049 sp |= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
2050 }
2051
2052 WREG32(GRBM_GFX_INDEX, gfx);
2053 WREG32(RLC_GFX_INDEX, gfx);
2054
2055 WREG32(CC_RB_BACKEND_DISABLE, rb);
2056 WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
2057 WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
2058 WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
2059 }
2060
2061 grbm_gfx_index |= SE_BROADCAST_WRITES;
2062 WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
2063 WREG32(RLC_GFX_INDEX, grbm_gfx_index);
2064
2065 WREG32(CGTS_SYS_TCC_DISABLE, 0);
2066 WREG32(CGTS_TCC_DISABLE, 0);
2067 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2068 WREG32(CGTS_USER_TCC_DISABLE, 0);
2069
2070 /* set HW defaults for 3D engine */
2071 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2072 ROQ_IB2_START(0x2b)));
2073
2074 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2075
2076 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2077 SYNC_GRADIENT |
2078 SYNC_WALKER |
2079 SYNC_ALIGNER));
2080
2081 sx_debug_1 = RREG32(SX_DEBUG_1);
2082 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2083 WREG32(SX_DEBUG_1, sx_debug_1);
2084
2085
2086 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2087 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2088 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2089 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2090
2091 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2092 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2093 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2094
2095 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2096 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2097 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2098
2099 WREG32(VGT_NUM_INSTANCES, 1);
2100 WREG32(SPI_CONFIG_CNTL, 0);
2101 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2102 WREG32(CP_PERFMON_CNTL, 0);
2103
2104 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2105 FETCH_FIFO_HIWATER(0x4) |
2106 DONE_FIFO_HIWATER(0xe0) |
2107 ALU_UPDATE_FIFO_HIWATER(0x8)));
2108
2109 sq_config = RREG32(SQ_CONFIG);
2110 sq_config &= ~(PS_PRIO(3) |
2111 VS_PRIO(3) |
2112 GS_PRIO(3) |
2113 ES_PRIO(3));
2114 sq_config |= (VC_ENABLE |
2115 EXPORT_SRC_C |
2116 PS_PRIO(0) |
2117 VS_PRIO(1) |
2118 GS_PRIO(2) |
2119 ES_PRIO(3));
2120
2121 switch (rdev->family) {
2122 case CHIP_CEDAR:
2123 case CHIP_PALM:
2124 case CHIP_SUMO:
2125 case CHIP_SUMO2:
2126 case CHIP_CAICOS:
2127 /* no vertex cache */
2128 sq_config &= ~VC_ENABLE;
2129 break;
2130 default:
2131 break;
2132 }
2133
2134 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2135
2136 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2137 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2138 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2139 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2140 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2141 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2142 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2143
2144 switch (rdev->family) {
2145 case CHIP_CEDAR:
2146 case CHIP_PALM:
2147 case CHIP_SUMO:
2148 case CHIP_SUMO2:
2149 ps_thread_count = 96;
2150 break;
2151 default:
2152 ps_thread_count = 128;
2153 break;
2154 }
2155
2156 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2157 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2158 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2159 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2160 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2161 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2162
2163 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2164 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2165 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2166 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2167 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2168 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2169
2170 WREG32(SQ_CONFIG, sq_config);
2171 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2172 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2173 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2174 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2175 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2176 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2177 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2178 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2179 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2180 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2181
2182 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2183 FORCE_EOV_MAX_REZ_CNT(255)));
2184
2185 switch (rdev->family) {
2186 case CHIP_CEDAR:
2187 case CHIP_PALM:
2188 case CHIP_SUMO:
2189 case CHIP_SUMO2:
2190 case CHIP_CAICOS:
2191 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2192 break;
2193 default:
2194 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2195 break;
2196 }
2197 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2198 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2199
2200 WREG32(VGT_GS_VERTEX_REUSE, 16);
2201 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2202 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2203
2204 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2205 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2206
2207 WREG32(CB_PERF_CTR0_SEL_0, 0);
2208 WREG32(CB_PERF_CTR0_SEL_1, 0);
2209 WREG32(CB_PERF_CTR1_SEL_0, 0);
2210 WREG32(CB_PERF_CTR1_SEL_1, 0);
2211 WREG32(CB_PERF_CTR2_SEL_0, 0);
2212 WREG32(CB_PERF_CTR2_SEL_1, 0);
2213 WREG32(CB_PERF_CTR3_SEL_0, 0);
2214 WREG32(CB_PERF_CTR3_SEL_1, 0);
2215
2216 /* clear render buffer base addresses */
2217 WREG32(CB_COLOR0_BASE, 0);
2218 WREG32(CB_COLOR1_BASE, 0);
2219 WREG32(CB_COLOR2_BASE, 0);
2220 WREG32(CB_COLOR3_BASE, 0);
2221 WREG32(CB_COLOR4_BASE, 0);
2222 WREG32(CB_COLOR5_BASE, 0);
2223 WREG32(CB_COLOR6_BASE, 0);
2224 WREG32(CB_COLOR7_BASE, 0);
2225 WREG32(CB_COLOR8_BASE, 0);
2226 WREG32(CB_COLOR9_BASE, 0);
2227 WREG32(CB_COLOR10_BASE, 0);
2228 WREG32(CB_COLOR11_BASE, 0);
2229
2230 /* set the shader const cache sizes to 0 */
2231 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2232 WREG32(i, 0);
2233 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2234 WREG32(i, 0);
2235
2236 tmp = RREG32(HDP_MISC_CNTL);
2237 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2238 WREG32(HDP_MISC_CNTL, tmp);
2239
2240 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2241 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2242
2243 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2244
2245 udelay(50);
2246
2247}
2248
2249int evergreen_mc_init(struct radeon_device *rdev)
2250{
2251 u32 tmp;
2252 int chansize, numchan;
2253
2254 /* Get VRAM informations */
2255 rdev->mc.vram_is_ddr = true;
2256 if (rdev->flags & RADEON_IS_IGP)
2257 tmp = RREG32(FUS_MC_ARB_RAMCFG);
2258 else
2259 tmp = RREG32(MC_ARB_RAMCFG);
2260 if (tmp & CHANSIZE_OVERRIDE) {
2261 chansize = 16;
2262 } else if (tmp & CHANSIZE_MASK) {
2263 chansize = 64;
2264 } else {
2265 chansize = 32;
2266 }
2267 tmp = RREG32(MC_SHARED_CHMAP);
2268 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2269 case 0:
2270 default:
2271 numchan = 1;
2272 break;
2273 case 1:
2274 numchan = 2;
2275 break;
2276 case 2:
2277 numchan = 4;
2278 break;
2279 case 3:
2280 numchan = 8;
2281 break;
2282 }
2283 rdev->mc.vram_width = numchan * chansize;
2284 /* Could aper size report 0 ? */
2285 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2286 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
2287 /* Setup GPU memory space */
2288 if (rdev->flags & RADEON_IS_IGP) {
2289 /* size in bytes on fusion */
2290 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2291 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2292 } else {
2293 /* size in MB on evergreen */
2294 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2295 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2296 }
2297 rdev->mc.visible_vram_size = rdev->mc.aper_size;
2298 r700_vram_gtt_location(rdev, &rdev->mc);
2299 radeon_update_bandwidth_info(rdev);
2300
2301 return 0;
2302}
2303
2304bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
2305{
2306 u32 srbm_status;
2307 u32 grbm_status;
2308 u32 grbm_status_se0, grbm_status_se1;
2309 struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
2310 int r;
2311
2312 srbm_status = RREG32(SRBM_STATUS);
2313 grbm_status = RREG32(GRBM_STATUS);
2314 grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2315 grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2316 if (!(grbm_status & GUI_ACTIVE)) {
2317 r100_gpu_lockup_update(lockup, &rdev->cp);
2318 return false;
2319 }
2320 /* force CP activities */
2321 r = radeon_ring_lock(rdev, 2);
2322 if (!r) {
2323 /* PACKET2 NOP */
2324 radeon_ring_write(rdev, 0x80000000);
2325 radeon_ring_write(rdev, 0x80000000);
2326 radeon_ring_unlock_commit(rdev);
2327 }
2328 rdev->cp.rptr = RREG32(CP_RB_RPTR);
2329 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
2330}
2331
2332static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
2333{
2334 struct evergreen_mc_save save;
2335 u32 grbm_reset = 0;
2336
2337 if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
2338 return 0;
2339
2340 dev_info(rdev->dev, "GPU softreset \n");
2341 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
2342 RREG32(GRBM_STATUS));
2343 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2344 RREG32(GRBM_STATUS_SE0));
2345 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2346 RREG32(GRBM_STATUS_SE1));
2347 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
2348 RREG32(SRBM_STATUS));
2349 evergreen_mc_stop(rdev, &save);
2350 if (evergreen_mc_wait_for_idle(rdev)) {
2351 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2352 }
2353 /* Disable CP parsing/prefetching */
2354 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2355
2356 /* reset all the gfx blocks */
2357 grbm_reset = (SOFT_RESET_CP |
2358 SOFT_RESET_CB |
2359 SOFT_RESET_DB |
2360 SOFT_RESET_PA |
2361 SOFT_RESET_SC |
2362 SOFT_RESET_SPI |
2363 SOFT_RESET_SH |
2364 SOFT_RESET_SX |
2365 SOFT_RESET_TC |
2366 SOFT_RESET_TA |
2367 SOFT_RESET_VC |
2368 SOFT_RESET_VGT);
2369
2370 dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
2371 WREG32(GRBM_SOFT_RESET, grbm_reset);
2372 (void)RREG32(GRBM_SOFT_RESET);
2373 udelay(50);
2374 WREG32(GRBM_SOFT_RESET, 0);
2375 (void)RREG32(GRBM_SOFT_RESET);
2376 /* Wait a little for things to settle down */
2377 udelay(50);
2378 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
2379 RREG32(GRBM_STATUS));
2380 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2381 RREG32(GRBM_STATUS_SE0));
2382 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2383 RREG32(GRBM_STATUS_SE1));
2384 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
2385 RREG32(SRBM_STATUS));
2386 evergreen_mc_resume(rdev, &save);
2387 return 0;
2388}
2389
2390int evergreen_asic_reset(struct radeon_device *rdev)
2391{
2392 return evergreen_gpu_soft_reset(rdev);
2393}
2394
2395/* Interrupts */
2396
2397u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2398{
2399 switch (crtc) {
2400 case 0:
2401 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
2402 case 1:
2403 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
2404 case 2:
2405 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
2406 case 3:
2407 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
2408 case 4:
2409 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
2410 case 5:
2411 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
2412 default:
2413 return 0;
2414 }
2415}
2416
2417void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2418{
2419 u32 tmp;
2420
2421 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2422 WREG32(GRBM_INT_CNTL, 0);
2423 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2424 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2425 if (rdev->num_crtc >= 4) {
2426 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2427 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2428 }
2429 if (rdev->num_crtc >= 6) {
2430 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2431 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2432 }
2433
2434 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2435 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2436 if (rdev->num_crtc >= 4) {
2437 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2438 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2439 }
2440 if (rdev->num_crtc >= 6) {
2441 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2442 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2443 }
2444
2445 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2446 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2447
2448 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2449 WREG32(DC_HPD1_INT_CONTROL, tmp);
2450 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2451 WREG32(DC_HPD2_INT_CONTROL, tmp);
2452 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2453 WREG32(DC_HPD3_INT_CONTROL, tmp);
2454 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2455 WREG32(DC_HPD4_INT_CONTROL, tmp);
2456 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2457 WREG32(DC_HPD5_INT_CONTROL, tmp);
2458 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2459 WREG32(DC_HPD6_INT_CONTROL, tmp);
2460
2461}
2462
2463int evergreen_irq_set(struct radeon_device *rdev)
2464{
2465 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2466 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2467 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2468 u32 grbm_int_cntl = 0;
2469 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2470
2471 if (!rdev->irq.installed) {
2472 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2473 return -EINVAL;
2474 }
2475 /* don't enable anything if the ih is disabled */
2476 if (!rdev->ih.enabled) {
2477 r600_disable_interrupts(rdev);
2478 /* force the active interrupt state to all disabled */
2479 evergreen_disable_interrupt_state(rdev);
2480 return 0;
2481 }
2482
2483 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2484 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2485 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2486 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2487 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2488 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2489
2490 if (rdev->irq.sw_int) {
2491 DRM_DEBUG("evergreen_irq_set: sw int\n");
2492 cp_int_cntl |= RB_INT_ENABLE;
2493 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2494 }
2495 if (rdev->irq.crtc_vblank_int[0] ||
2496 rdev->irq.pflip[0]) {
2497 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2498 crtc1 |= VBLANK_INT_MASK;
2499 }
2500 if (rdev->irq.crtc_vblank_int[1] ||
2501 rdev->irq.pflip[1]) {
2502 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2503 crtc2 |= VBLANK_INT_MASK;
2504 }
2505 if (rdev->irq.crtc_vblank_int[2] ||
2506 rdev->irq.pflip[2]) {
2507 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2508 crtc3 |= VBLANK_INT_MASK;
2509 }
2510 if (rdev->irq.crtc_vblank_int[3] ||
2511 rdev->irq.pflip[3]) {
2512 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2513 crtc4 |= VBLANK_INT_MASK;
2514 }
2515 if (rdev->irq.crtc_vblank_int[4] ||
2516 rdev->irq.pflip[4]) {
2517 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2518 crtc5 |= VBLANK_INT_MASK;
2519 }
2520 if (rdev->irq.crtc_vblank_int[5] ||
2521 rdev->irq.pflip[5]) {
2522 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2523 crtc6 |= VBLANK_INT_MASK;
2524 }
2525 if (rdev->irq.hpd[0]) {
2526 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2527 hpd1 |= DC_HPDx_INT_EN;
2528 }
2529 if (rdev->irq.hpd[1]) {
2530 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2531 hpd2 |= DC_HPDx_INT_EN;
2532 }
2533 if (rdev->irq.hpd[2]) {
2534 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2535 hpd3 |= DC_HPDx_INT_EN;
2536 }
2537 if (rdev->irq.hpd[3]) {
2538 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2539 hpd4 |= DC_HPDx_INT_EN;
2540 }
2541 if (rdev->irq.hpd[4]) {
2542 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2543 hpd5 |= DC_HPDx_INT_EN;
2544 }
2545 if (rdev->irq.hpd[5]) {
2546 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2547 hpd6 |= DC_HPDx_INT_EN;
2548 }
2549 if (rdev->irq.gui_idle) {
2550 DRM_DEBUG("gui idle\n");
2551 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2552 }
2553
2554 WREG32(CP_INT_CNTL, cp_int_cntl);
2555 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2556
2557 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2558 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2559 if (rdev->num_crtc >= 4) {
2560 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2561 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2562 }
2563 if (rdev->num_crtc >= 6) {
2564 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2565 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2566 }
2567
2568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2570 if (rdev->num_crtc >= 4) {
2571 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2573 }
2574 if (rdev->num_crtc >= 6) {
2575 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2576 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2577 }
2578
2579 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2580 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2581 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2582 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2583 WREG32(DC_HPD5_INT_CONTROL, hpd5);
2584 WREG32(DC_HPD6_INT_CONTROL, hpd6);
2585
2586 return 0;
2587}
2588
2589static inline void evergreen_irq_ack(struct radeon_device *rdev)
2590{
2591 u32 tmp;
2592
2593 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2594 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2595 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2596 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2597 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2598 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2599 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2600 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2601 if (rdev->num_crtc >= 4) {
2602 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2603 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2604 }
2605 if (rdev->num_crtc >= 6) {
2606 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2607 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2608 }
2609
2610 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2611 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2612 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2613 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2614 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2615 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2616 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2617 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2618 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2619 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2620 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2621 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2622
2623 if (rdev->num_crtc >= 4) {
2624 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2625 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2626 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2627 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2628 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2629 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2630 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2631 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2632 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2633 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2634 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2635 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2636 }
2637
2638 if (rdev->num_crtc >= 6) {
2639 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2640 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2641 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2642 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2643 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2644 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2645 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2646 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2647 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2648 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2649 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2650 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2651 }
2652
2653 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2654 tmp = RREG32(DC_HPD1_INT_CONTROL);
2655 tmp |= DC_HPDx_INT_ACK;
2656 WREG32(DC_HPD1_INT_CONTROL, tmp);
2657 }
2658 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2659 tmp = RREG32(DC_HPD2_INT_CONTROL);
2660 tmp |= DC_HPDx_INT_ACK;
2661 WREG32(DC_HPD2_INT_CONTROL, tmp);
2662 }
2663 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2664 tmp = RREG32(DC_HPD3_INT_CONTROL);
2665 tmp |= DC_HPDx_INT_ACK;
2666 WREG32(DC_HPD3_INT_CONTROL, tmp);
2667 }
2668 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2669 tmp = RREG32(DC_HPD4_INT_CONTROL);
2670 tmp |= DC_HPDx_INT_ACK;
2671 WREG32(DC_HPD4_INT_CONTROL, tmp);
2672 }
2673 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2674 tmp = RREG32(DC_HPD5_INT_CONTROL);
2675 tmp |= DC_HPDx_INT_ACK;
2676 WREG32(DC_HPD5_INT_CONTROL, tmp);
2677 }
2678 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2679 tmp = RREG32(DC_HPD5_INT_CONTROL);
2680 tmp |= DC_HPDx_INT_ACK;
2681 WREG32(DC_HPD6_INT_CONTROL, tmp);
2682 }
2683}
2684
2685void evergreen_irq_disable(struct radeon_device *rdev)
2686{
2687 r600_disable_interrupts(rdev);
2688 /* Wait and acknowledge irq */
2689 mdelay(1);
2690 evergreen_irq_ack(rdev);
2691 evergreen_disable_interrupt_state(rdev);
2692}
2693
2694void evergreen_irq_suspend(struct radeon_device *rdev)
2695{
2696 evergreen_irq_disable(rdev);
2697 r600_rlc_stop(rdev);
2698}
2699
2700static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
2701{
2702 u32 wptr, tmp;
2703
2704 if (rdev->wb.enabled)
2705 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
2706 else
2707 wptr = RREG32(IH_RB_WPTR);
2708
2709 if (wptr & RB_OVERFLOW) {
2710 /* When a ring buffer overflow happen start parsing interrupt
2711 * from the last not overwritten vector (wptr + 16). Hopefully
2712 * this should allow us to catchup.
2713 */
2714 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2715 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2716 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2717 tmp = RREG32(IH_RB_CNTL);
2718 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2719 WREG32(IH_RB_CNTL, tmp);
2720 }
2721 return (wptr & rdev->ih.ptr_mask);
2722}
2723
2724int evergreen_irq_process(struct radeon_device *rdev)
2725{
2726 u32 wptr;
2727 u32 rptr;
2728 u32 src_id, src_data;
2729 u32 ring_index;
2730 unsigned long flags;
2731 bool queue_hotplug = false;
2732
2733 if (!rdev->ih.enabled || rdev->shutdown)
2734 return IRQ_NONE;
2735
2736 wptr = evergreen_get_ih_wptr(rdev);
2737 rptr = rdev->ih.rptr;
2738 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2739
2740 spin_lock_irqsave(&rdev->ih.lock, flags);
2741 if (rptr == wptr) {
2742 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2743 return IRQ_NONE;
2744 }
2745restart_ih:
2746 /* Order reading of wptr vs. reading of IH ring data */
2747 rmb();
2748
2749 /* display interrupts */
2750 evergreen_irq_ack(rdev);
2751
2752 rdev->ih.wptr = wptr;
2753 while (rptr != wptr) {
2754 /* wptr/rptr are in bytes! */
2755 ring_index = rptr / 4;
2756 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
2757 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
2758
2759 switch (src_id) {
2760 case 1: /* D1 vblank/vline */
2761 switch (src_data) {
2762 case 0: /* D1 vblank */
2763 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
2764 if (rdev->irq.crtc_vblank_int[0]) {
2765 drm_handle_vblank(rdev->ddev, 0);
2766 rdev->pm.vblank_sync = true;
2767 wake_up(&rdev->irq.vblank_queue);
2768 }
2769 if (rdev->irq.pflip[0])
2770 radeon_crtc_handle_flip(rdev, 0);
2771 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2772 DRM_DEBUG("IH: D1 vblank\n");
2773 }
2774 break;
2775 case 1: /* D1 vline */
2776 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
2777 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
2778 DRM_DEBUG("IH: D1 vline\n");
2779 }
2780 break;
2781 default:
2782 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2783 break;
2784 }
2785 break;
2786 case 2: /* D2 vblank/vline */
2787 switch (src_data) {
2788 case 0: /* D2 vblank */
2789 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
2790 if (rdev->irq.crtc_vblank_int[1]) {
2791 drm_handle_vblank(rdev->ddev, 1);
2792 rdev->pm.vblank_sync = true;
2793 wake_up(&rdev->irq.vblank_queue);
2794 }
2795 if (rdev->irq.pflip[1])
2796 radeon_crtc_handle_flip(rdev, 1);
2797 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
2798 DRM_DEBUG("IH: D2 vblank\n");
2799 }
2800 break;
2801 case 1: /* D2 vline */
2802 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
2803 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
2804 DRM_DEBUG("IH: D2 vline\n");
2805 }
2806 break;
2807 default:
2808 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2809 break;
2810 }
2811 break;
2812 case 3: /* D3 vblank/vline */
2813 switch (src_data) {
2814 case 0: /* D3 vblank */
2815 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
2816 if (rdev->irq.crtc_vblank_int[2]) {
2817 drm_handle_vblank(rdev->ddev, 2);
2818 rdev->pm.vblank_sync = true;
2819 wake_up(&rdev->irq.vblank_queue);
2820 }
2821 if (rdev->irq.pflip[2])
2822 radeon_crtc_handle_flip(rdev, 2);
2823 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
2824 DRM_DEBUG("IH: D3 vblank\n");
2825 }
2826 break;
2827 case 1: /* D3 vline */
2828 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
2829 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
2830 DRM_DEBUG("IH: D3 vline\n");
2831 }
2832 break;
2833 default:
2834 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2835 break;
2836 }
2837 break;
2838 case 4: /* D4 vblank/vline */
2839 switch (src_data) {
2840 case 0: /* D4 vblank */
2841 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
2842 if (rdev->irq.crtc_vblank_int[3]) {
2843 drm_handle_vblank(rdev->ddev, 3);
2844 rdev->pm.vblank_sync = true;
2845 wake_up(&rdev->irq.vblank_queue);
2846 }
2847 if (rdev->irq.pflip[3])
2848 radeon_crtc_handle_flip(rdev, 3);
2849 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
2850 DRM_DEBUG("IH: D4 vblank\n");
2851 }
2852 break;
2853 case 1: /* D4 vline */
2854 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
2855 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
2856 DRM_DEBUG("IH: D4 vline\n");
2857 }
2858 break;
2859 default:
2860 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2861 break;
2862 }
2863 break;
2864 case 5: /* D5 vblank/vline */
2865 switch (src_data) {
2866 case 0: /* D5 vblank */
2867 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
2868 if (rdev->irq.crtc_vblank_int[4]) {
2869 drm_handle_vblank(rdev->ddev, 4);
2870 rdev->pm.vblank_sync = true;
2871 wake_up(&rdev->irq.vblank_queue);
2872 }
2873 if (rdev->irq.pflip[4])
2874 radeon_crtc_handle_flip(rdev, 4);
2875 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
2876 DRM_DEBUG("IH: D5 vblank\n");
2877 }
2878 break;
2879 case 1: /* D5 vline */
2880 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
2881 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
2882 DRM_DEBUG("IH: D5 vline\n");
2883 }
2884 break;
2885 default:
2886 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2887 break;
2888 }
2889 break;
2890 case 6: /* D6 vblank/vline */
2891 switch (src_data) {
2892 case 0: /* D6 vblank */
2893 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
2894 if (rdev->irq.crtc_vblank_int[5]) {
2895 drm_handle_vblank(rdev->ddev, 5);
2896 rdev->pm.vblank_sync = true;
2897 wake_up(&rdev->irq.vblank_queue);
2898 }
2899 if (rdev->irq.pflip[5])
2900 radeon_crtc_handle_flip(rdev, 5);
2901 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
2902 DRM_DEBUG("IH: D6 vblank\n");
2903 }
2904 break;
2905 case 1: /* D6 vline */
2906 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
2907 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
2908 DRM_DEBUG("IH: D6 vline\n");
2909 }
2910 break;
2911 default:
2912 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2913 break;
2914 }
2915 break;
2916 case 42: /* HPD hotplug */
2917 switch (src_data) {
2918 case 0:
2919 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2920 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
2921 queue_hotplug = true;
2922 DRM_DEBUG("IH: HPD1\n");
2923 }
2924 break;
2925 case 1:
2926 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2927 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
2928 queue_hotplug = true;
2929 DRM_DEBUG("IH: HPD2\n");
2930 }
2931 break;
2932 case 2:
2933 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2934 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
2935 queue_hotplug = true;
2936 DRM_DEBUG("IH: HPD3\n");
2937 }
2938 break;
2939 case 3:
2940 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2941 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
2942 queue_hotplug = true;
2943 DRM_DEBUG("IH: HPD4\n");
2944 }
2945 break;
2946 case 4:
2947 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2948 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
2949 queue_hotplug = true;
2950 DRM_DEBUG("IH: HPD5\n");
2951 }
2952 break;
2953 case 5:
2954 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2955 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
2956 queue_hotplug = true;
2957 DRM_DEBUG("IH: HPD6\n");
2958 }
2959 break;
2960 default:
2961 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2962 break;
2963 }
2964 break;
2965 case 176: /* CP_INT in ring buffer */
2966 case 177: /* CP_INT in IB1 */
2967 case 178: /* CP_INT in IB2 */
2968 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2969 radeon_fence_process(rdev);
2970 break;
2971 case 181: /* CP EOP event */
2972 DRM_DEBUG("IH: CP EOP\n");
2973 radeon_fence_process(rdev);
2974 break;
2975 case 233: /* GUI IDLE */
2976 DRM_DEBUG("IH: GUI idle\n");
2977 rdev->pm.gui_idle = true;
2978 wake_up(&rdev->irq.idle_queue);
2979 break;
2980 default:
2981 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2982 break;
2983 }
2984
2985 /* wptr/rptr are in bytes! */
2986 rptr += 16;
2987 rptr &= rdev->ih.ptr_mask;
2988 }
2989 /* make sure wptr hasn't changed while processing */
2990 wptr = evergreen_get_ih_wptr(rdev);
2991 if (wptr != rdev->ih.wptr)
2992 goto restart_ih;
2993 if (queue_hotplug)
2994 schedule_work(&rdev->hotplug_work);
2995 rdev->ih.rptr = rptr;
2996 WREG32(IH_RB_RPTR, rdev->ih.rptr);
2997 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2998 return IRQ_HANDLED;
2999}
3000
3001static int evergreen_startup(struct radeon_device *rdev)
3002{
3003 int r;
3004
3005 /* enable pcie gen2 link */
3006 if (!ASIC_IS_DCE5(rdev))
3007 evergreen_pcie_gen2_enable(rdev);
3008
3009 if (ASIC_IS_DCE5(rdev)) {
3010 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3011 r = ni_init_microcode(rdev);
3012 if (r) {
3013 DRM_ERROR("Failed to load firmware!\n");
3014 return r;
3015 }
3016 }
3017 r = ni_mc_load_microcode(rdev);
3018 if (r) {
3019 DRM_ERROR("Failed to load MC firmware!\n");
3020 return r;
3021 }
3022 } else {
3023 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3024 r = r600_init_microcode(rdev);
3025 if (r) {
3026 DRM_ERROR("Failed to load firmware!\n");
3027 return r;
3028 }
3029 }
3030 }
3031
3032 evergreen_mc_program(rdev);
3033 if (rdev->flags & RADEON_IS_AGP) {
3034 evergreen_agp_enable(rdev);
3035 } else {
3036 r = evergreen_pcie_gart_enable(rdev);
3037 if (r)
3038 return r;
3039 }
3040 evergreen_gpu_init(rdev);
3041
3042 r = evergreen_blit_init(rdev);
3043 if (r) {
3044 evergreen_blit_fini(rdev);
3045 rdev->asic->copy = NULL;
3046 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3047 }
3048
3049 /* allocate wb buffer */
3050 r = radeon_wb_init(rdev);
3051 if (r)
3052 return r;
3053
3054 /* Enable IRQ */
3055 r = r600_irq_init(rdev);
3056 if (r) {
3057 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3058 radeon_irq_kms_fini(rdev);
3059 return r;
3060 }
3061 evergreen_irq_set(rdev);
3062
3063 r = radeon_ring_init(rdev, rdev->cp.ring_size);
3064 if (r)
3065 return r;
3066 r = evergreen_cp_load_microcode(rdev);
3067 if (r)
3068 return r;
3069 r = evergreen_cp_resume(rdev);
3070 if (r)
3071 return r;
3072
3073 return 0;
3074}
3075
3076int evergreen_resume(struct radeon_device *rdev)
3077{
3078 int r;
3079
3080 /* reset the asic, the gfx blocks are often in a bad state
3081 * after the driver is unloaded or after a resume
3082 */
3083 if (radeon_asic_reset(rdev))
3084 dev_warn(rdev->dev, "GPU reset failed !\n");
3085 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3086 * posting will perform necessary task to bring back GPU into good
3087 * shape.
3088 */
3089 /* post card */
3090 atom_asic_init(rdev->mode_info.atom_context);
3091
3092 r = evergreen_startup(rdev);
3093 if (r) {
3094 DRM_ERROR("evergreen startup failed on resume\n");
3095 return r;
3096 }
3097
3098 r = r600_ib_test(rdev);
3099 if (r) {
3100 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3101 return r;
3102 }
3103
3104 return r;
3105
3106}
3107
3108int evergreen_suspend(struct radeon_device *rdev)
3109{
3110 int r;
3111
3112 /* FIXME: we should wait for ring to be empty */
3113 r700_cp_stop(rdev);
3114 rdev->cp.ready = false;
3115 evergreen_irq_suspend(rdev);
3116 radeon_wb_disable(rdev);
3117 evergreen_pcie_gart_disable(rdev);
3118
3119 /* unpin shaders bo */
3120 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
3121 if (likely(r == 0)) {
3122 radeon_bo_unpin(rdev->r600_blit.shader_obj);
3123 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
3124 }
3125
3126 return 0;
3127}
3128
3129int evergreen_copy_blit(struct radeon_device *rdev,
3130 uint64_t src_offset,
3131 uint64_t dst_offset,
3132 unsigned num_gpu_pages,
3133 struct radeon_fence *fence)
3134{
3135 int r;
3136
3137 mutex_lock(&rdev->r600_blit.mutex);
3138 rdev->r600_blit.vb_ib = NULL;
3139 r = evergreen_blit_prepare_copy(rdev, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
3140 if (r) {
3141 if (rdev->r600_blit.vb_ib)
3142 radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
3143 mutex_unlock(&rdev->r600_blit.mutex);
3144 return r;
3145 }
3146 evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
3147 evergreen_blit_done_copy(rdev, fence);
3148 mutex_unlock(&rdev->r600_blit.mutex);
3149 return 0;
3150}
3151
3152/* Plan is to move initialization in that function and use
3153 * helper function so that radeon_device_init pretty much
3154 * do nothing more than calling asic specific function. This
3155 * should also allow to remove a bunch of callback function
3156 * like vram_info.
3157 */
3158int evergreen_init(struct radeon_device *rdev)
3159{
3160 int r;
3161
3162 /* This don't do much */
3163 r = radeon_gem_init(rdev);
3164 if (r)
3165 return r;
3166 /* Read BIOS */
3167 if (!radeon_get_bios(rdev)) {
3168 if (ASIC_IS_AVIVO(rdev))
3169 return -EINVAL;
3170 }
3171 /* Must be an ATOMBIOS */
3172 if (!rdev->is_atom_bios) {
3173 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
3174 return -EINVAL;
3175 }
3176 r = radeon_atombios_init(rdev);
3177 if (r)
3178 return r;
3179 /* reset the asic, the gfx blocks are often in a bad state
3180 * after the driver is unloaded or after a resume
3181 */
3182 if (radeon_asic_reset(rdev))
3183 dev_warn(rdev->dev, "GPU reset failed !\n");
3184 /* Post card if necessary */
3185 if (!radeon_card_posted(rdev)) {
3186 if (!rdev->bios) {
3187 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3188 return -EINVAL;
3189 }
3190 DRM_INFO("GPU not posted. posting now...\n");
3191 atom_asic_init(rdev->mode_info.atom_context);
3192 }
3193 /* Initialize scratch registers */
3194 r600_scratch_init(rdev);
3195 /* Initialize surface registers */
3196 radeon_surface_init(rdev);
3197 /* Initialize clocks */
3198 radeon_get_clock_info(rdev->ddev);
3199 /* Fence driver */
3200 r = radeon_fence_driver_init(rdev);
3201 if (r)
3202 return r;
3203 /* initialize AGP */
3204 if (rdev->flags & RADEON_IS_AGP) {
3205 r = radeon_agp_init(rdev);
3206 if (r)
3207 radeon_agp_disable(rdev);
3208 }
3209 /* initialize memory controller */
3210 r = evergreen_mc_init(rdev);
3211 if (r)
3212 return r;
3213 /* Memory manager */
3214 r = radeon_bo_init(rdev);
3215 if (r)
3216 return r;
3217
3218 r = radeon_irq_kms_init(rdev);
3219 if (r)
3220 return r;
3221
3222 rdev->cp.ring_obj = NULL;
3223 r600_ring_init(rdev, 1024 * 1024);
3224
3225 rdev->ih.ring_obj = NULL;
3226 r600_ih_ring_init(rdev, 64 * 1024);
3227
3228 r = r600_pcie_gart_init(rdev);
3229 if (r)
3230 return r;
3231
3232 rdev->accel_working = true;
3233 r = evergreen_startup(rdev);
3234 if (r) {
3235 dev_err(rdev->dev, "disabling GPU acceleration\n");
3236 r700_cp_fini(rdev);
3237 r600_irq_fini(rdev);
3238 radeon_wb_fini(rdev);
3239 radeon_irq_kms_fini(rdev);
3240 evergreen_pcie_gart_fini(rdev);
3241 rdev->accel_working = false;
3242 }
3243 if (rdev->accel_working) {
3244 r = radeon_ib_pool_init(rdev);
3245 if (r) {
3246 DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
3247 rdev->accel_working = false;
3248 }
3249 r = r600_ib_test(rdev);
3250 if (r) {
3251 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3252 rdev->accel_working = false;
3253 }
3254 }
3255 return 0;
3256}
3257
3258void evergreen_fini(struct radeon_device *rdev)
3259{
3260 evergreen_blit_fini(rdev);
3261 r700_cp_fini(rdev);
3262 r600_irq_fini(rdev);
3263 radeon_wb_fini(rdev);
3264 radeon_ib_pool_fini(rdev);
3265 radeon_irq_kms_fini(rdev);
3266 evergreen_pcie_gart_fini(rdev);
3267 radeon_gem_fini(rdev);
3268 radeon_fence_driver_fini(rdev);
3269 radeon_agp_fini(rdev);
3270 radeon_bo_fini(rdev);
3271 radeon_atombios_fini(rdev);
3272 kfree(rdev->bios);
3273 rdev->bios = NULL;
3274}
3275
3276static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
3277{
3278 u32 link_width_cntl, speed_cntl;
3279
3280 if (radeon_pcie_gen2 == 0)
3281 return;
3282
3283 if (rdev->flags & RADEON_IS_IGP)
3284 return;
3285
3286 if (!(rdev->flags & RADEON_IS_PCIE))
3287 return;
3288
3289 /* x2 cards have a special sequence */
3290 if (ASIC_IS_X2(rdev))
3291 return;
3292
3293 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3294 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3295 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3296
3297 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3298 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3299 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3300
3301 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3302 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3303 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3304
3305 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3306 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3307 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3308
3309 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3310 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3311 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3312
3313 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3314 speed_cntl |= LC_GEN2_EN_STRAP;
3315 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3316
3317 } else {
3318 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3319 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3320 if (1)
3321 link_width_cntl |= LC_UPCONFIGURE_DIS;
3322 else
3323 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3324 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3325 }
3326}
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24
25#include <linux/firmware.h>
26#include <linux/pci.h>
27#include <linux/slab.h>
28
29#include <drm/drm_vblank.h>
30#include <drm/radeon_drm.h>
31
32#include "atom.h"
33#include "avivod.h"
34#include "cik.h"
35#include "ni.h"
36#include "rv770.h"
37#include "evergreen.h"
38#include "evergreen_blit_shaders.h"
39#include "evergreen_reg.h"
40#include "evergreend.h"
41#include "radeon.h"
42#include "radeon_asic.h"
43#include "radeon_audio.h"
44#include "radeon_ucode.h"
45#include "si.h"
46
47#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
48#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
49#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
50
51/*
52 * Indirect registers accessor
53 */
54u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
55{
56 unsigned long flags;
57 u32 r;
58
59 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
60 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
61 r = RREG32(EVERGREEN_CG_IND_DATA);
62 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
63 return r;
64}
65
66void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
67{
68 unsigned long flags;
69
70 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
71 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
72 WREG32(EVERGREEN_CG_IND_DATA, (v));
73 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
74}
75
76u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
77{
78 unsigned long flags;
79 u32 r;
80
81 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
82 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
83 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
84 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
85 return r;
86}
87
88void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
89{
90 unsigned long flags;
91
92 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
93 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
94 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
95 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
96}
97
98u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
99{
100 unsigned long flags;
101 u32 r;
102
103 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
104 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
105 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
106 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
107 return r;
108}
109
110void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
111{
112 unsigned long flags;
113
114 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
115 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
116 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
117 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
118}
119
120static const u32 crtc_offsets[6] =
121{
122 EVERGREEN_CRTC0_REGISTER_OFFSET,
123 EVERGREEN_CRTC1_REGISTER_OFFSET,
124 EVERGREEN_CRTC2_REGISTER_OFFSET,
125 EVERGREEN_CRTC3_REGISTER_OFFSET,
126 EVERGREEN_CRTC4_REGISTER_OFFSET,
127 EVERGREEN_CRTC5_REGISTER_OFFSET
128};
129
130#include "clearstate_evergreen.h"
131
132static const u32 sumo_rlc_save_restore_register_list[] =
133{
134 0x98fc,
135 0x9830,
136 0x9834,
137 0x9838,
138 0x9870,
139 0x9874,
140 0x8a14,
141 0x8b24,
142 0x8bcc,
143 0x8b10,
144 0x8d00,
145 0x8d04,
146 0x8c00,
147 0x8c04,
148 0x8c08,
149 0x8c0c,
150 0x8d8c,
151 0x8c20,
152 0x8c24,
153 0x8c28,
154 0x8c18,
155 0x8c1c,
156 0x8cf0,
157 0x8e2c,
158 0x8e38,
159 0x8c30,
160 0x9508,
161 0x9688,
162 0x9608,
163 0x960c,
164 0x9610,
165 0x9614,
166 0x88c4,
167 0x88d4,
168 0xa008,
169 0x900c,
170 0x9100,
171 0x913c,
172 0x98f8,
173 0x98f4,
174 0x9b7c,
175 0x3f8c,
176 0x8950,
177 0x8954,
178 0x8a18,
179 0x8b28,
180 0x9144,
181 0x9148,
182 0x914c,
183 0x3f90,
184 0x3f94,
185 0x915c,
186 0x9160,
187 0x9178,
188 0x917c,
189 0x9180,
190 0x918c,
191 0x9190,
192 0x9194,
193 0x9198,
194 0x919c,
195 0x91a8,
196 0x91ac,
197 0x91b0,
198 0x91b4,
199 0x91b8,
200 0x91c4,
201 0x91c8,
202 0x91cc,
203 0x91d0,
204 0x91d4,
205 0x91e0,
206 0x91e4,
207 0x91ec,
208 0x91f0,
209 0x91f4,
210 0x9200,
211 0x9204,
212 0x929c,
213 0x9150,
214 0x802c,
215};
216
217static void evergreen_gpu_init(struct radeon_device *rdev);
218void evergreen_fini(struct radeon_device *rdev);
219void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
220void evergreen_program_aspm(struct radeon_device *rdev);
221
222static const u32 evergreen_golden_registers[] =
223{
224 0x3f90, 0xffff0000, 0xff000000,
225 0x9148, 0xffff0000, 0xff000000,
226 0x3f94, 0xffff0000, 0xff000000,
227 0x914c, 0xffff0000, 0xff000000,
228 0x9b7c, 0xffffffff, 0x00000000,
229 0x8a14, 0xffffffff, 0x00000007,
230 0x8b10, 0xffffffff, 0x00000000,
231 0x960c, 0xffffffff, 0x54763210,
232 0x88c4, 0xffffffff, 0x000000c2,
233 0x88d4, 0xffffffff, 0x00000010,
234 0x8974, 0xffffffff, 0x00000000,
235 0xc78, 0x00000080, 0x00000080,
236 0x5eb4, 0xffffffff, 0x00000002,
237 0x5e78, 0xffffffff, 0x001000f0,
238 0x6104, 0x01000300, 0x00000000,
239 0x5bc0, 0x00300000, 0x00000000,
240 0x7030, 0xffffffff, 0x00000011,
241 0x7c30, 0xffffffff, 0x00000011,
242 0x10830, 0xffffffff, 0x00000011,
243 0x11430, 0xffffffff, 0x00000011,
244 0x12030, 0xffffffff, 0x00000011,
245 0x12c30, 0xffffffff, 0x00000011,
246 0xd02c, 0xffffffff, 0x08421000,
247 0x240c, 0xffffffff, 0x00000380,
248 0x8b24, 0xffffffff, 0x00ff0fff,
249 0x28a4c, 0x06000000, 0x06000000,
250 0x10c, 0x00000001, 0x00000001,
251 0x8d00, 0xffffffff, 0x100e4848,
252 0x8d04, 0xffffffff, 0x00164745,
253 0x8c00, 0xffffffff, 0xe4000003,
254 0x8c04, 0xffffffff, 0x40600060,
255 0x8c08, 0xffffffff, 0x001c001c,
256 0x8cf0, 0xffffffff, 0x08e00620,
257 0x8c20, 0xffffffff, 0x00800080,
258 0x8c24, 0xffffffff, 0x00800080,
259 0x8c18, 0xffffffff, 0x20202078,
260 0x8c1c, 0xffffffff, 0x00001010,
261 0x28350, 0xffffffff, 0x00000000,
262 0xa008, 0xffffffff, 0x00010000,
263 0x5c4, 0xffffffff, 0x00000001,
264 0x9508, 0xffffffff, 0x00000002,
265 0x913c, 0x0000000f, 0x0000000a
266};
267
268static const u32 evergreen_golden_registers2[] =
269{
270 0x2f4c, 0xffffffff, 0x00000000,
271 0x54f4, 0xffffffff, 0x00000000,
272 0x54f0, 0xffffffff, 0x00000000,
273 0x5498, 0xffffffff, 0x00000000,
274 0x549c, 0xffffffff, 0x00000000,
275 0x5494, 0xffffffff, 0x00000000,
276 0x53cc, 0xffffffff, 0x00000000,
277 0x53c8, 0xffffffff, 0x00000000,
278 0x53c4, 0xffffffff, 0x00000000,
279 0x53c0, 0xffffffff, 0x00000000,
280 0x53bc, 0xffffffff, 0x00000000,
281 0x53b8, 0xffffffff, 0x00000000,
282 0x53b4, 0xffffffff, 0x00000000,
283 0x53b0, 0xffffffff, 0x00000000
284};
285
286static const u32 cypress_mgcg_init[] =
287{
288 0x802c, 0xffffffff, 0xc0000000,
289 0x5448, 0xffffffff, 0x00000100,
290 0x55e4, 0xffffffff, 0x00000100,
291 0x160c, 0xffffffff, 0x00000100,
292 0x5644, 0xffffffff, 0x00000100,
293 0xc164, 0xffffffff, 0x00000100,
294 0x8a18, 0xffffffff, 0x00000100,
295 0x897c, 0xffffffff, 0x06000100,
296 0x8b28, 0xffffffff, 0x00000100,
297 0x9144, 0xffffffff, 0x00000100,
298 0x9a60, 0xffffffff, 0x00000100,
299 0x9868, 0xffffffff, 0x00000100,
300 0x8d58, 0xffffffff, 0x00000100,
301 0x9510, 0xffffffff, 0x00000100,
302 0x949c, 0xffffffff, 0x00000100,
303 0x9654, 0xffffffff, 0x00000100,
304 0x9030, 0xffffffff, 0x00000100,
305 0x9034, 0xffffffff, 0x00000100,
306 0x9038, 0xffffffff, 0x00000100,
307 0x903c, 0xffffffff, 0x00000100,
308 0x9040, 0xffffffff, 0x00000100,
309 0xa200, 0xffffffff, 0x00000100,
310 0xa204, 0xffffffff, 0x00000100,
311 0xa208, 0xffffffff, 0x00000100,
312 0xa20c, 0xffffffff, 0x00000100,
313 0x971c, 0xffffffff, 0x00000100,
314 0x977c, 0xffffffff, 0x00000100,
315 0x3f80, 0xffffffff, 0x00000100,
316 0xa210, 0xffffffff, 0x00000100,
317 0xa214, 0xffffffff, 0x00000100,
318 0x4d8, 0xffffffff, 0x00000100,
319 0x9784, 0xffffffff, 0x00000100,
320 0x9698, 0xffffffff, 0x00000100,
321 0x4d4, 0xffffffff, 0x00000200,
322 0x30cc, 0xffffffff, 0x00000100,
323 0xd0c0, 0xffffffff, 0xff000100,
324 0x802c, 0xffffffff, 0x40000000,
325 0x915c, 0xffffffff, 0x00010000,
326 0x9160, 0xffffffff, 0x00030002,
327 0x9178, 0xffffffff, 0x00070000,
328 0x917c, 0xffffffff, 0x00030002,
329 0x9180, 0xffffffff, 0x00050004,
330 0x918c, 0xffffffff, 0x00010006,
331 0x9190, 0xffffffff, 0x00090008,
332 0x9194, 0xffffffff, 0x00070000,
333 0x9198, 0xffffffff, 0x00030002,
334 0x919c, 0xffffffff, 0x00050004,
335 0x91a8, 0xffffffff, 0x00010006,
336 0x91ac, 0xffffffff, 0x00090008,
337 0x91b0, 0xffffffff, 0x00070000,
338 0x91b4, 0xffffffff, 0x00030002,
339 0x91b8, 0xffffffff, 0x00050004,
340 0x91c4, 0xffffffff, 0x00010006,
341 0x91c8, 0xffffffff, 0x00090008,
342 0x91cc, 0xffffffff, 0x00070000,
343 0x91d0, 0xffffffff, 0x00030002,
344 0x91d4, 0xffffffff, 0x00050004,
345 0x91e0, 0xffffffff, 0x00010006,
346 0x91e4, 0xffffffff, 0x00090008,
347 0x91e8, 0xffffffff, 0x00000000,
348 0x91ec, 0xffffffff, 0x00070000,
349 0x91f0, 0xffffffff, 0x00030002,
350 0x91f4, 0xffffffff, 0x00050004,
351 0x9200, 0xffffffff, 0x00010006,
352 0x9204, 0xffffffff, 0x00090008,
353 0x9208, 0xffffffff, 0x00070000,
354 0x920c, 0xffffffff, 0x00030002,
355 0x9210, 0xffffffff, 0x00050004,
356 0x921c, 0xffffffff, 0x00010006,
357 0x9220, 0xffffffff, 0x00090008,
358 0x9224, 0xffffffff, 0x00070000,
359 0x9228, 0xffffffff, 0x00030002,
360 0x922c, 0xffffffff, 0x00050004,
361 0x9238, 0xffffffff, 0x00010006,
362 0x923c, 0xffffffff, 0x00090008,
363 0x9240, 0xffffffff, 0x00070000,
364 0x9244, 0xffffffff, 0x00030002,
365 0x9248, 0xffffffff, 0x00050004,
366 0x9254, 0xffffffff, 0x00010006,
367 0x9258, 0xffffffff, 0x00090008,
368 0x925c, 0xffffffff, 0x00070000,
369 0x9260, 0xffffffff, 0x00030002,
370 0x9264, 0xffffffff, 0x00050004,
371 0x9270, 0xffffffff, 0x00010006,
372 0x9274, 0xffffffff, 0x00090008,
373 0x9278, 0xffffffff, 0x00070000,
374 0x927c, 0xffffffff, 0x00030002,
375 0x9280, 0xffffffff, 0x00050004,
376 0x928c, 0xffffffff, 0x00010006,
377 0x9290, 0xffffffff, 0x00090008,
378 0x9294, 0xffffffff, 0x00000000,
379 0x929c, 0xffffffff, 0x00000001,
380 0x802c, 0xffffffff, 0x40010000,
381 0x915c, 0xffffffff, 0x00010000,
382 0x9160, 0xffffffff, 0x00030002,
383 0x9178, 0xffffffff, 0x00070000,
384 0x917c, 0xffffffff, 0x00030002,
385 0x9180, 0xffffffff, 0x00050004,
386 0x918c, 0xffffffff, 0x00010006,
387 0x9190, 0xffffffff, 0x00090008,
388 0x9194, 0xffffffff, 0x00070000,
389 0x9198, 0xffffffff, 0x00030002,
390 0x919c, 0xffffffff, 0x00050004,
391 0x91a8, 0xffffffff, 0x00010006,
392 0x91ac, 0xffffffff, 0x00090008,
393 0x91b0, 0xffffffff, 0x00070000,
394 0x91b4, 0xffffffff, 0x00030002,
395 0x91b8, 0xffffffff, 0x00050004,
396 0x91c4, 0xffffffff, 0x00010006,
397 0x91c8, 0xffffffff, 0x00090008,
398 0x91cc, 0xffffffff, 0x00070000,
399 0x91d0, 0xffffffff, 0x00030002,
400 0x91d4, 0xffffffff, 0x00050004,
401 0x91e0, 0xffffffff, 0x00010006,
402 0x91e4, 0xffffffff, 0x00090008,
403 0x91e8, 0xffffffff, 0x00000000,
404 0x91ec, 0xffffffff, 0x00070000,
405 0x91f0, 0xffffffff, 0x00030002,
406 0x91f4, 0xffffffff, 0x00050004,
407 0x9200, 0xffffffff, 0x00010006,
408 0x9204, 0xffffffff, 0x00090008,
409 0x9208, 0xffffffff, 0x00070000,
410 0x920c, 0xffffffff, 0x00030002,
411 0x9210, 0xffffffff, 0x00050004,
412 0x921c, 0xffffffff, 0x00010006,
413 0x9220, 0xffffffff, 0x00090008,
414 0x9224, 0xffffffff, 0x00070000,
415 0x9228, 0xffffffff, 0x00030002,
416 0x922c, 0xffffffff, 0x00050004,
417 0x9238, 0xffffffff, 0x00010006,
418 0x923c, 0xffffffff, 0x00090008,
419 0x9240, 0xffffffff, 0x00070000,
420 0x9244, 0xffffffff, 0x00030002,
421 0x9248, 0xffffffff, 0x00050004,
422 0x9254, 0xffffffff, 0x00010006,
423 0x9258, 0xffffffff, 0x00090008,
424 0x925c, 0xffffffff, 0x00070000,
425 0x9260, 0xffffffff, 0x00030002,
426 0x9264, 0xffffffff, 0x00050004,
427 0x9270, 0xffffffff, 0x00010006,
428 0x9274, 0xffffffff, 0x00090008,
429 0x9278, 0xffffffff, 0x00070000,
430 0x927c, 0xffffffff, 0x00030002,
431 0x9280, 0xffffffff, 0x00050004,
432 0x928c, 0xffffffff, 0x00010006,
433 0x9290, 0xffffffff, 0x00090008,
434 0x9294, 0xffffffff, 0x00000000,
435 0x929c, 0xffffffff, 0x00000001,
436 0x802c, 0xffffffff, 0xc0000000
437};
438
439static const u32 redwood_mgcg_init[] =
440{
441 0x802c, 0xffffffff, 0xc0000000,
442 0x5448, 0xffffffff, 0x00000100,
443 0x55e4, 0xffffffff, 0x00000100,
444 0x160c, 0xffffffff, 0x00000100,
445 0x5644, 0xffffffff, 0x00000100,
446 0xc164, 0xffffffff, 0x00000100,
447 0x8a18, 0xffffffff, 0x00000100,
448 0x897c, 0xffffffff, 0x06000100,
449 0x8b28, 0xffffffff, 0x00000100,
450 0x9144, 0xffffffff, 0x00000100,
451 0x9a60, 0xffffffff, 0x00000100,
452 0x9868, 0xffffffff, 0x00000100,
453 0x8d58, 0xffffffff, 0x00000100,
454 0x9510, 0xffffffff, 0x00000100,
455 0x949c, 0xffffffff, 0x00000100,
456 0x9654, 0xffffffff, 0x00000100,
457 0x9030, 0xffffffff, 0x00000100,
458 0x9034, 0xffffffff, 0x00000100,
459 0x9038, 0xffffffff, 0x00000100,
460 0x903c, 0xffffffff, 0x00000100,
461 0x9040, 0xffffffff, 0x00000100,
462 0xa200, 0xffffffff, 0x00000100,
463 0xa204, 0xffffffff, 0x00000100,
464 0xa208, 0xffffffff, 0x00000100,
465 0xa20c, 0xffffffff, 0x00000100,
466 0x971c, 0xffffffff, 0x00000100,
467 0x977c, 0xffffffff, 0x00000100,
468 0x3f80, 0xffffffff, 0x00000100,
469 0xa210, 0xffffffff, 0x00000100,
470 0xa214, 0xffffffff, 0x00000100,
471 0x4d8, 0xffffffff, 0x00000100,
472 0x9784, 0xffffffff, 0x00000100,
473 0x9698, 0xffffffff, 0x00000100,
474 0x4d4, 0xffffffff, 0x00000200,
475 0x30cc, 0xffffffff, 0x00000100,
476 0xd0c0, 0xffffffff, 0xff000100,
477 0x802c, 0xffffffff, 0x40000000,
478 0x915c, 0xffffffff, 0x00010000,
479 0x9160, 0xffffffff, 0x00030002,
480 0x9178, 0xffffffff, 0x00070000,
481 0x917c, 0xffffffff, 0x00030002,
482 0x9180, 0xffffffff, 0x00050004,
483 0x918c, 0xffffffff, 0x00010006,
484 0x9190, 0xffffffff, 0x00090008,
485 0x9194, 0xffffffff, 0x00070000,
486 0x9198, 0xffffffff, 0x00030002,
487 0x919c, 0xffffffff, 0x00050004,
488 0x91a8, 0xffffffff, 0x00010006,
489 0x91ac, 0xffffffff, 0x00090008,
490 0x91b0, 0xffffffff, 0x00070000,
491 0x91b4, 0xffffffff, 0x00030002,
492 0x91b8, 0xffffffff, 0x00050004,
493 0x91c4, 0xffffffff, 0x00010006,
494 0x91c8, 0xffffffff, 0x00090008,
495 0x91cc, 0xffffffff, 0x00070000,
496 0x91d0, 0xffffffff, 0x00030002,
497 0x91d4, 0xffffffff, 0x00050004,
498 0x91e0, 0xffffffff, 0x00010006,
499 0x91e4, 0xffffffff, 0x00090008,
500 0x91e8, 0xffffffff, 0x00000000,
501 0x91ec, 0xffffffff, 0x00070000,
502 0x91f0, 0xffffffff, 0x00030002,
503 0x91f4, 0xffffffff, 0x00050004,
504 0x9200, 0xffffffff, 0x00010006,
505 0x9204, 0xffffffff, 0x00090008,
506 0x9294, 0xffffffff, 0x00000000,
507 0x929c, 0xffffffff, 0x00000001,
508 0x802c, 0xffffffff, 0xc0000000
509};
510
511static const u32 cedar_golden_registers[] =
512{
513 0x3f90, 0xffff0000, 0xff000000,
514 0x9148, 0xffff0000, 0xff000000,
515 0x3f94, 0xffff0000, 0xff000000,
516 0x914c, 0xffff0000, 0xff000000,
517 0x9b7c, 0xffffffff, 0x00000000,
518 0x8a14, 0xffffffff, 0x00000007,
519 0x8b10, 0xffffffff, 0x00000000,
520 0x960c, 0xffffffff, 0x54763210,
521 0x88c4, 0xffffffff, 0x000000c2,
522 0x88d4, 0xffffffff, 0x00000000,
523 0x8974, 0xffffffff, 0x00000000,
524 0xc78, 0x00000080, 0x00000080,
525 0x5eb4, 0xffffffff, 0x00000002,
526 0x5e78, 0xffffffff, 0x001000f0,
527 0x6104, 0x01000300, 0x00000000,
528 0x5bc0, 0x00300000, 0x00000000,
529 0x7030, 0xffffffff, 0x00000011,
530 0x7c30, 0xffffffff, 0x00000011,
531 0x10830, 0xffffffff, 0x00000011,
532 0x11430, 0xffffffff, 0x00000011,
533 0xd02c, 0xffffffff, 0x08421000,
534 0x240c, 0xffffffff, 0x00000380,
535 0x8b24, 0xffffffff, 0x00ff0fff,
536 0x28a4c, 0x06000000, 0x06000000,
537 0x10c, 0x00000001, 0x00000001,
538 0x8d00, 0xffffffff, 0x100e4848,
539 0x8d04, 0xffffffff, 0x00164745,
540 0x8c00, 0xffffffff, 0xe4000003,
541 0x8c04, 0xffffffff, 0x40600060,
542 0x8c08, 0xffffffff, 0x001c001c,
543 0x8cf0, 0xffffffff, 0x08e00410,
544 0x8c20, 0xffffffff, 0x00800080,
545 0x8c24, 0xffffffff, 0x00800080,
546 0x8c18, 0xffffffff, 0x20202078,
547 0x8c1c, 0xffffffff, 0x00001010,
548 0x28350, 0xffffffff, 0x00000000,
549 0xa008, 0xffffffff, 0x00010000,
550 0x5c4, 0xffffffff, 0x00000001,
551 0x9508, 0xffffffff, 0x00000002
552};
553
554static const u32 cedar_mgcg_init[] =
555{
556 0x802c, 0xffffffff, 0xc0000000,
557 0x5448, 0xffffffff, 0x00000100,
558 0x55e4, 0xffffffff, 0x00000100,
559 0x160c, 0xffffffff, 0x00000100,
560 0x5644, 0xffffffff, 0x00000100,
561 0xc164, 0xffffffff, 0x00000100,
562 0x8a18, 0xffffffff, 0x00000100,
563 0x897c, 0xffffffff, 0x06000100,
564 0x8b28, 0xffffffff, 0x00000100,
565 0x9144, 0xffffffff, 0x00000100,
566 0x9a60, 0xffffffff, 0x00000100,
567 0x9868, 0xffffffff, 0x00000100,
568 0x8d58, 0xffffffff, 0x00000100,
569 0x9510, 0xffffffff, 0x00000100,
570 0x949c, 0xffffffff, 0x00000100,
571 0x9654, 0xffffffff, 0x00000100,
572 0x9030, 0xffffffff, 0x00000100,
573 0x9034, 0xffffffff, 0x00000100,
574 0x9038, 0xffffffff, 0x00000100,
575 0x903c, 0xffffffff, 0x00000100,
576 0x9040, 0xffffffff, 0x00000100,
577 0xa200, 0xffffffff, 0x00000100,
578 0xa204, 0xffffffff, 0x00000100,
579 0xa208, 0xffffffff, 0x00000100,
580 0xa20c, 0xffffffff, 0x00000100,
581 0x971c, 0xffffffff, 0x00000100,
582 0x977c, 0xffffffff, 0x00000100,
583 0x3f80, 0xffffffff, 0x00000100,
584 0xa210, 0xffffffff, 0x00000100,
585 0xa214, 0xffffffff, 0x00000100,
586 0x4d8, 0xffffffff, 0x00000100,
587 0x9784, 0xffffffff, 0x00000100,
588 0x9698, 0xffffffff, 0x00000100,
589 0x4d4, 0xffffffff, 0x00000200,
590 0x30cc, 0xffffffff, 0x00000100,
591 0xd0c0, 0xffffffff, 0xff000100,
592 0x802c, 0xffffffff, 0x40000000,
593 0x915c, 0xffffffff, 0x00010000,
594 0x9178, 0xffffffff, 0x00050000,
595 0x917c, 0xffffffff, 0x00030002,
596 0x918c, 0xffffffff, 0x00010004,
597 0x9190, 0xffffffff, 0x00070006,
598 0x9194, 0xffffffff, 0x00050000,
599 0x9198, 0xffffffff, 0x00030002,
600 0x91a8, 0xffffffff, 0x00010004,
601 0x91ac, 0xffffffff, 0x00070006,
602 0x91e8, 0xffffffff, 0x00000000,
603 0x9294, 0xffffffff, 0x00000000,
604 0x929c, 0xffffffff, 0x00000001,
605 0x802c, 0xffffffff, 0xc0000000
606};
607
608static const u32 juniper_mgcg_init[] =
609{
610 0x802c, 0xffffffff, 0xc0000000,
611 0x5448, 0xffffffff, 0x00000100,
612 0x55e4, 0xffffffff, 0x00000100,
613 0x160c, 0xffffffff, 0x00000100,
614 0x5644, 0xffffffff, 0x00000100,
615 0xc164, 0xffffffff, 0x00000100,
616 0x8a18, 0xffffffff, 0x00000100,
617 0x897c, 0xffffffff, 0x06000100,
618 0x8b28, 0xffffffff, 0x00000100,
619 0x9144, 0xffffffff, 0x00000100,
620 0x9a60, 0xffffffff, 0x00000100,
621 0x9868, 0xffffffff, 0x00000100,
622 0x8d58, 0xffffffff, 0x00000100,
623 0x9510, 0xffffffff, 0x00000100,
624 0x949c, 0xffffffff, 0x00000100,
625 0x9654, 0xffffffff, 0x00000100,
626 0x9030, 0xffffffff, 0x00000100,
627 0x9034, 0xffffffff, 0x00000100,
628 0x9038, 0xffffffff, 0x00000100,
629 0x903c, 0xffffffff, 0x00000100,
630 0x9040, 0xffffffff, 0x00000100,
631 0xa200, 0xffffffff, 0x00000100,
632 0xa204, 0xffffffff, 0x00000100,
633 0xa208, 0xffffffff, 0x00000100,
634 0xa20c, 0xffffffff, 0x00000100,
635 0x971c, 0xffffffff, 0x00000100,
636 0xd0c0, 0xffffffff, 0xff000100,
637 0x802c, 0xffffffff, 0x40000000,
638 0x915c, 0xffffffff, 0x00010000,
639 0x9160, 0xffffffff, 0x00030002,
640 0x9178, 0xffffffff, 0x00070000,
641 0x917c, 0xffffffff, 0x00030002,
642 0x9180, 0xffffffff, 0x00050004,
643 0x918c, 0xffffffff, 0x00010006,
644 0x9190, 0xffffffff, 0x00090008,
645 0x9194, 0xffffffff, 0x00070000,
646 0x9198, 0xffffffff, 0x00030002,
647 0x919c, 0xffffffff, 0x00050004,
648 0x91a8, 0xffffffff, 0x00010006,
649 0x91ac, 0xffffffff, 0x00090008,
650 0x91b0, 0xffffffff, 0x00070000,
651 0x91b4, 0xffffffff, 0x00030002,
652 0x91b8, 0xffffffff, 0x00050004,
653 0x91c4, 0xffffffff, 0x00010006,
654 0x91c8, 0xffffffff, 0x00090008,
655 0x91cc, 0xffffffff, 0x00070000,
656 0x91d0, 0xffffffff, 0x00030002,
657 0x91d4, 0xffffffff, 0x00050004,
658 0x91e0, 0xffffffff, 0x00010006,
659 0x91e4, 0xffffffff, 0x00090008,
660 0x91e8, 0xffffffff, 0x00000000,
661 0x91ec, 0xffffffff, 0x00070000,
662 0x91f0, 0xffffffff, 0x00030002,
663 0x91f4, 0xffffffff, 0x00050004,
664 0x9200, 0xffffffff, 0x00010006,
665 0x9204, 0xffffffff, 0x00090008,
666 0x9208, 0xffffffff, 0x00070000,
667 0x920c, 0xffffffff, 0x00030002,
668 0x9210, 0xffffffff, 0x00050004,
669 0x921c, 0xffffffff, 0x00010006,
670 0x9220, 0xffffffff, 0x00090008,
671 0x9224, 0xffffffff, 0x00070000,
672 0x9228, 0xffffffff, 0x00030002,
673 0x922c, 0xffffffff, 0x00050004,
674 0x9238, 0xffffffff, 0x00010006,
675 0x923c, 0xffffffff, 0x00090008,
676 0x9240, 0xffffffff, 0x00070000,
677 0x9244, 0xffffffff, 0x00030002,
678 0x9248, 0xffffffff, 0x00050004,
679 0x9254, 0xffffffff, 0x00010006,
680 0x9258, 0xffffffff, 0x00090008,
681 0x925c, 0xffffffff, 0x00070000,
682 0x9260, 0xffffffff, 0x00030002,
683 0x9264, 0xffffffff, 0x00050004,
684 0x9270, 0xffffffff, 0x00010006,
685 0x9274, 0xffffffff, 0x00090008,
686 0x9278, 0xffffffff, 0x00070000,
687 0x927c, 0xffffffff, 0x00030002,
688 0x9280, 0xffffffff, 0x00050004,
689 0x928c, 0xffffffff, 0x00010006,
690 0x9290, 0xffffffff, 0x00090008,
691 0x9294, 0xffffffff, 0x00000000,
692 0x929c, 0xffffffff, 0x00000001,
693 0x802c, 0xffffffff, 0xc0000000,
694 0x977c, 0xffffffff, 0x00000100,
695 0x3f80, 0xffffffff, 0x00000100,
696 0xa210, 0xffffffff, 0x00000100,
697 0xa214, 0xffffffff, 0x00000100,
698 0x4d8, 0xffffffff, 0x00000100,
699 0x9784, 0xffffffff, 0x00000100,
700 0x9698, 0xffffffff, 0x00000100,
701 0x4d4, 0xffffffff, 0x00000200,
702 0x30cc, 0xffffffff, 0x00000100,
703 0x802c, 0xffffffff, 0xc0000000
704};
705
706static const u32 supersumo_golden_registers[] =
707{
708 0x5eb4, 0xffffffff, 0x00000002,
709 0x5c4, 0xffffffff, 0x00000001,
710 0x7030, 0xffffffff, 0x00000011,
711 0x7c30, 0xffffffff, 0x00000011,
712 0x6104, 0x01000300, 0x00000000,
713 0x5bc0, 0x00300000, 0x00000000,
714 0x8c04, 0xffffffff, 0x40600060,
715 0x8c08, 0xffffffff, 0x001c001c,
716 0x8c20, 0xffffffff, 0x00800080,
717 0x8c24, 0xffffffff, 0x00800080,
718 0x8c18, 0xffffffff, 0x20202078,
719 0x8c1c, 0xffffffff, 0x00001010,
720 0x918c, 0xffffffff, 0x00010006,
721 0x91a8, 0xffffffff, 0x00010006,
722 0x91c4, 0xffffffff, 0x00010006,
723 0x91e0, 0xffffffff, 0x00010006,
724 0x9200, 0xffffffff, 0x00010006,
725 0x9150, 0xffffffff, 0x6e944040,
726 0x917c, 0xffffffff, 0x00030002,
727 0x9180, 0xffffffff, 0x00050004,
728 0x9198, 0xffffffff, 0x00030002,
729 0x919c, 0xffffffff, 0x00050004,
730 0x91b4, 0xffffffff, 0x00030002,
731 0x91b8, 0xffffffff, 0x00050004,
732 0x91d0, 0xffffffff, 0x00030002,
733 0x91d4, 0xffffffff, 0x00050004,
734 0x91f0, 0xffffffff, 0x00030002,
735 0x91f4, 0xffffffff, 0x00050004,
736 0x915c, 0xffffffff, 0x00010000,
737 0x9160, 0xffffffff, 0x00030002,
738 0x3f90, 0xffff0000, 0xff000000,
739 0x9178, 0xffffffff, 0x00070000,
740 0x9194, 0xffffffff, 0x00070000,
741 0x91b0, 0xffffffff, 0x00070000,
742 0x91cc, 0xffffffff, 0x00070000,
743 0x91ec, 0xffffffff, 0x00070000,
744 0x9148, 0xffff0000, 0xff000000,
745 0x9190, 0xffffffff, 0x00090008,
746 0x91ac, 0xffffffff, 0x00090008,
747 0x91c8, 0xffffffff, 0x00090008,
748 0x91e4, 0xffffffff, 0x00090008,
749 0x9204, 0xffffffff, 0x00090008,
750 0x3f94, 0xffff0000, 0xff000000,
751 0x914c, 0xffff0000, 0xff000000,
752 0x929c, 0xffffffff, 0x00000001,
753 0x8a18, 0xffffffff, 0x00000100,
754 0x8b28, 0xffffffff, 0x00000100,
755 0x9144, 0xffffffff, 0x00000100,
756 0x5644, 0xffffffff, 0x00000100,
757 0x9b7c, 0xffffffff, 0x00000000,
758 0x8030, 0xffffffff, 0x0000100a,
759 0x8a14, 0xffffffff, 0x00000007,
760 0x8b24, 0xffffffff, 0x00ff0fff,
761 0x8b10, 0xffffffff, 0x00000000,
762 0x28a4c, 0x06000000, 0x06000000,
763 0x4d8, 0xffffffff, 0x00000100,
764 0x913c, 0xffff000f, 0x0100000a,
765 0x960c, 0xffffffff, 0x54763210,
766 0x88c4, 0xffffffff, 0x000000c2,
767 0x88d4, 0xffffffff, 0x00000010,
768 0x8974, 0xffffffff, 0x00000000,
769 0xc78, 0x00000080, 0x00000080,
770 0x5e78, 0xffffffff, 0x001000f0,
771 0xd02c, 0xffffffff, 0x08421000,
772 0xa008, 0xffffffff, 0x00010000,
773 0x8d00, 0xffffffff, 0x100e4848,
774 0x8d04, 0xffffffff, 0x00164745,
775 0x8c00, 0xffffffff, 0xe4000003,
776 0x8cf0, 0x1fffffff, 0x08e00620,
777 0x28350, 0xffffffff, 0x00000000,
778 0x9508, 0xffffffff, 0x00000002
779};
780
781static const u32 sumo_golden_registers[] =
782{
783 0x900c, 0x00ffffff, 0x0017071f,
784 0x8c18, 0xffffffff, 0x10101060,
785 0x8c1c, 0xffffffff, 0x00001010,
786 0x8c30, 0x0000000f, 0x00000005,
787 0x9688, 0x0000000f, 0x00000007
788};
789
790static const u32 wrestler_golden_registers[] =
791{
792 0x5eb4, 0xffffffff, 0x00000002,
793 0x5c4, 0xffffffff, 0x00000001,
794 0x7030, 0xffffffff, 0x00000011,
795 0x7c30, 0xffffffff, 0x00000011,
796 0x6104, 0x01000300, 0x00000000,
797 0x5bc0, 0x00300000, 0x00000000,
798 0x918c, 0xffffffff, 0x00010006,
799 0x91a8, 0xffffffff, 0x00010006,
800 0x9150, 0xffffffff, 0x6e944040,
801 0x917c, 0xffffffff, 0x00030002,
802 0x9198, 0xffffffff, 0x00030002,
803 0x915c, 0xffffffff, 0x00010000,
804 0x3f90, 0xffff0000, 0xff000000,
805 0x9178, 0xffffffff, 0x00070000,
806 0x9194, 0xffffffff, 0x00070000,
807 0x9148, 0xffff0000, 0xff000000,
808 0x9190, 0xffffffff, 0x00090008,
809 0x91ac, 0xffffffff, 0x00090008,
810 0x3f94, 0xffff0000, 0xff000000,
811 0x914c, 0xffff0000, 0xff000000,
812 0x929c, 0xffffffff, 0x00000001,
813 0x8a18, 0xffffffff, 0x00000100,
814 0x8b28, 0xffffffff, 0x00000100,
815 0x9144, 0xffffffff, 0x00000100,
816 0x9b7c, 0xffffffff, 0x00000000,
817 0x8030, 0xffffffff, 0x0000100a,
818 0x8a14, 0xffffffff, 0x00000001,
819 0x8b24, 0xffffffff, 0x00ff0fff,
820 0x8b10, 0xffffffff, 0x00000000,
821 0x28a4c, 0x06000000, 0x06000000,
822 0x4d8, 0xffffffff, 0x00000100,
823 0x913c, 0xffff000f, 0x0100000a,
824 0x960c, 0xffffffff, 0x54763210,
825 0x88c4, 0xffffffff, 0x000000c2,
826 0x88d4, 0xffffffff, 0x00000010,
827 0x8974, 0xffffffff, 0x00000000,
828 0xc78, 0x00000080, 0x00000080,
829 0x5e78, 0xffffffff, 0x001000f0,
830 0xd02c, 0xffffffff, 0x08421000,
831 0xa008, 0xffffffff, 0x00010000,
832 0x8d00, 0xffffffff, 0x100e4848,
833 0x8d04, 0xffffffff, 0x00164745,
834 0x8c00, 0xffffffff, 0xe4000003,
835 0x8cf0, 0x1fffffff, 0x08e00410,
836 0x28350, 0xffffffff, 0x00000000,
837 0x9508, 0xffffffff, 0x00000002,
838 0x900c, 0xffffffff, 0x0017071f,
839 0x8c18, 0xffffffff, 0x10101060,
840 0x8c1c, 0xffffffff, 0x00001010
841};
842
843static const u32 barts_golden_registers[] =
844{
845 0x5eb4, 0xffffffff, 0x00000002,
846 0x5e78, 0x8f311ff1, 0x001000f0,
847 0x3f90, 0xffff0000, 0xff000000,
848 0x9148, 0xffff0000, 0xff000000,
849 0x3f94, 0xffff0000, 0xff000000,
850 0x914c, 0xffff0000, 0xff000000,
851 0xc78, 0x00000080, 0x00000080,
852 0xbd4, 0x70073777, 0x00010001,
853 0xd02c, 0xbfffff1f, 0x08421000,
854 0xd0b8, 0x03773777, 0x02011003,
855 0x5bc0, 0x00200000, 0x50100000,
856 0x98f8, 0x33773777, 0x02011003,
857 0x98fc, 0xffffffff, 0x76543210,
858 0x7030, 0x31000311, 0x00000011,
859 0x2f48, 0x00000007, 0x02011003,
860 0x6b28, 0x00000010, 0x00000012,
861 0x7728, 0x00000010, 0x00000012,
862 0x10328, 0x00000010, 0x00000012,
863 0x10f28, 0x00000010, 0x00000012,
864 0x11b28, 0x00000010, 0x00000012,
865 0x12728, 0x00000010, 0x00000012,
866 0x240c, 0x000007ff, 0x00000380,
867 0x8a14, 0xf000001f, 0x00000007,
868 0x8b24, 0x3fff3fff, 0x00ff0fff,
869 0x8b10, 0x0000ff0f, 0x00000000,
870 0x28a4c, 0x07ffffff, 0x06000000,
871 0x10c, 0x00000001, 0x00010003,
872 0xa02c, 0xffffffff, 0x0000009b,
873 0x913c, 0x0000000f, 0x0100000a,
874 0x8d00, 0xffff7f7f, 0x100e4848,
875 0x8d04, 0x00ffffff, 0x00164745,
876 0x8c00, 0xfffc0003, 0xe4000003,
877 0x8c04, 0xf8ff00ff, 0x40600060,
878 0x8c08, 0x00ff00ff, 0x001c001c,
879 0x8cf0, 0x1fff1fff, 0x08e00620,
880 0x8c20, 0x0fff0fff, 0x00800080,
881 0x8c24, 0x0fff0fff, 0x00800080,
882 0x8c18, 0xffffffff, 0x20202078,
883 0x8c1c, 0x0000ffff, 0x00001010,
884 0x28350, 0x00000f01, 0x00000000,
885 0x9508, 0x3700001f, 0x00000002,
886 0x960c, 0xffffffff, 0x54763210,
887 0x88c4, 0x001f3ae3, 0x000000c2,
888 0x88d4, 0x0000001f, 0x00000010,
889 0x8974, 0xffffffff, 0x00000000
890};
891
892static const u32 turks_golden_registers[] =
893{
894 0x5eb4, 0xffffffff, 0x00000002,
895 0x5e78, 0x8f311ff1, 0x001000f0,
896 0x8c8, 0x00003000, 0x00001070,
897 0x8cc, 0x000fffff, 0x00040035,
898 0x3f90, 0xffff0000, 0xfff00000,
899 0x9148, 0xffff0000, 0xfff00000,
900 0x3f94, 0xffff0000, 0xfff00000,
901 0x914c, 0xffff0000, 0xfff00000,
902 0xc78, 0x00000080, 0x00000080,
903 0xbd4, 0x00073007, 0x00010002,
904 0xd02c, 0xbfffff1f, 0x08421000,
905 0xd0b8, 0x03773777, 0x02010002,
906 0x5bc0, 0x00200000, 0x50100000,
907 0x98f8, 0x33773777, 0x00010002,
908 0x98fc, 0xffffffff, 0x33221100,
909 0x7030, 0x31000311, 0x00000011,
910 0x2f48, 0x33773777, 0x00010002,
911 0x6b28, 0x00000010, 0x00000012,
912 0x7728, 0x00000010, 0x00000012,
913 0x10328, 0x00000010, 0x00000012,
914 0x10f28, 0x00000010, 0x00000012,
915 0x11b28, 0x00000010, 0x00000012,
916 0x12728, 0x00000010, 0x00000012,
917 0x240c, 0x000007ff, 0x00000380,
918 0x8a14, 0xf000001f, 0x00000007,
919 0x8b24, 0x3fff3fff, 0x00ff0fff,
920 0x8b10, 0x0000ff0f, 0x00000000,
921 0x28a4c, 0x07ffffff, 0x06000000,
922 0x10c, 0x00000001, 0x00010003,
923 0xa02c, 0xffffffff, 0x0000009b,
924 0x913c, 0x0000000f, 0x0100000a,
925 0x8d00, 0xffff7f7f, 0x100e4848,
926 0x8d04, 0x00ffffff, 0x00164745,
927 0x8c00, 0xfffc0003, 0xe4000003,
928 0x8c04, 0xf8ff00ff, 0x40600060,
929 0x8c08, 0x00ff00ff, 0x001c001c,
930 0x8cf0, 0x1fff1fff, 0x08e00410,
931 0x8c20, 0x0fff0fff, 0x00800080,
932 0x8c24, 0x0fff0fff, 0x00800080,
933 0x8c18, 0xffffffff, 0x20202078,
934 0x8c1c, 0x0000ffff, 0x00001010,
935 0x28350, 0x00000f01, 0x00000000,
936 0x9508, 0x3700001f, 0x00000002,
937 0x960c, 0xffffffff, 0x54763210,
938 0x88c4, 0x001f3ae3, 0x000000c2,
939 0x88d4, 0x0000001f, 0x00000010,
940 0x8974, 0xffffffff, 0x00000000
941};
942
943static const u32 caicos_golden_registers[] =
944{
945 0x5eb4, 0xffffffff, 0x00000002,
946 0x5e78, 0x8f311ff1, 0x001000f0,
947 0x8c8, 0x00003420, 0x00001450,
948 0x8cc, 0x000fffff, 0x00040035,
949 0x3f90, 0xffff0000, 0xfffc0000,
950 0x9148, 0xffff0000, 0xfffc0000,
951 0x3f94, 0xffff0000, 0xfffc0000,
952 0x914c, 0xffff0000, 0xfffc0000,
953 0xc78, 0x00000080, 0x00000080,
954 0xbd4, 0x00073007, 0x00010001,
955 0xd02c, 0xbfffff1f, 0x08421000,
956 0xd0b8, 0x03773777, 0x02010001,
957 0x5bc0, 0x00200000, 0x50100000,
958 0x98f8, 0x33773777, 0x02010001,
959 0x98fc, 0xffffffff, 0x33221100,
960 0x7030, 0x31000311, 0x00000011,
961 0x2f48, 0x33773777, 0x02010001,
962 0x6b28, 0x00000010, 0x00000012,
963 0x7728, 0x00000010, 0x00000012,
964 0x10328, 0x00000010, 0x00000012,
965 0x10f28, 0x00000010, 0x00000012,
966 0x11b28, 0x00000010, 0x00000012,
967 0x12728, 0x00000010, 0x00000012,
968 0x240c, 0x000007ff, 0x00000380,
969 0x8a14, 0xf000001f, 0x00000001,
970 0x8b24, 0x3fff3fff, 0x00ff0fff,
971 0x8b10, 0x0000ff0f, 0x00000000,
972 0x28a4c, 0x07ffffff, 0x06000000,
973 0x10c, 0x00000001, 0x00010003,
974 0xa02c, 0xffffffff, 0x0000009b,
975 0x913c, 0x0000000f, 0x0100000a,
976 0x8d00, 0xffff7f7f, 0x100e4848,
977 0x8d04, 0x00ffffff, 0x00164745,
978 0x8c00, 0xfffc0003, 0xe4000003,
979 0x8c04, 0xf8ff00ff, 0x40600060,
980 0x8c08, 0x00ff00ff, 0x001c001c,
981 0x8cf0, 0x1fff1fff, 0x08e00410,
982 0x8c20, 0x0fff0fff, 0x00800080,
983 0x8c24, 0x0fff0fff, 0x00800080,
984 0x8c18, 0xffffffff, 0x20202078,
985 0x8c1c, 0x0000ffff, 0x00001010,
986 0x28350, 0x00000f01, 0x00000000,
987 0x9508, 0x3700001f, 0x00000002,
988 0x960c, 0xffffffff, 0x54763210,
989 0x88c4, 0x001f3ae3, 0x000000c2,
990 0x88d4, 0x0000001f, 0x00000010,
991 0x8974, 0xffffffff, 0x00000000
992};
993
994static void evergreen_init_golden_registers(struct radeon_device *rdev)
995{
996 switch (rdev->family) {
997 case CHIP_CYPRESS:
998 case CHIP_HEMLOCK:
999 radeon_program_register_sequence(rdev,
1000 evergreen_golden_registers,
1001 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1002 radeon_program_register_sequence(rdev,
1003 evergreen_golden_registers2,
1004 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1005 radeon_program_register_sequence(rdev,
1006 cypress_mgcg_init,
1007 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1008 break;
1009 case CHIP_JUNIPER:
1010 radeon_program_register_sequence(rdev,
1011 evergreen_golden_registers,
1012 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1013 radeon_program_register_sequence(rdev,
1014 evergreen_golden_registers2,
1015 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1016 radeon_program_register_sequence(rdev,
1017 juniper_mgcg_init,
1018 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1019 break;
1020 case CHIP_REDWOOD:
1021 radeon_program_register_sequence(rdev,
1022 evergreen_golden_registers,
1023 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1024 radeon_program_register_sequence(rdev,
1025 evergreen_golden_registers2,
1026 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1027 radeon_program_register_sequence(rdev,
1028 redwood_mgcg_init,
1029 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1030 break;
1031 case CHIP_CEDAR:
1032 radeon_program_register_sequence(rdev,
1033 cedar_golden_registers,
1034 (const u32)ARRAY_SIZE(cedar_golden_registers));
1035 radeon_program_register_sequence(rdev,
1036 evergreen_golden_registers2,
1037 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1038 radeon_program_register_sequence(rdev,
1039 cedar_mgcg_init,
1040 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1041 break;
1042 case CHIP_PALM:
1043 radeon_program_register_sequence(rdev,
1044 wrestler_golden_registers,
1045 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1046 break;
1047 case CHIP_SUMO:
1048 radeon_program_register_sequence(rdev,
1049 supersumo_golden_registers,
1050 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1051 break;
1052 case CHIP_SUMO2:
1053 radeon_program_register_sequence(rdev,
1054 supersumo_golden_registers,
1055 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1056 radeon_program_register_sequence(rdev,
1057 sumo_golden_registers,
1058 (const u32)ARRAY_SIZE(sumo_golden_registers));
1059 break;
1060 case CHIP_BARTS:
1061 radeon_program_register_sequence(rdev,
1062 barts_golden_registers,
1063 (const u32)ARRAY_SIZE(barts_golden_registers));
1064 break;
1065 case CHIP_TURKS:
1066 radeon_program_register_sequence(rdev,
1067 turks_golden_registers,
1068 (const u32)ARRAY_SIZE(turks_golden_registers));
1069 break;
1070 case CHIP_CAICOS:
1071 radeon_program_register_sequence(rdev,
1072 caicos_golden_registers,
1073 (const u32)ARRAY_SIZE(caicos_golden_registers));
1074 break;
1075 default:
1076 break;
1077 }
1078}
1079
1080/**
1081 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1082 *
1083 * @rdev: radeon_device pointer
1084 * @reg: register offset in bytes
1085 * @val: register value
1086 *
1087 * Returns 0 for success or -EINVAL for an invalid register
1088 *
1089 */
1090int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1091 u32 reg, u32 *val)
1092{
1093 switch (reg) {
1094 case GRBM_STATUS:
1095 case GRBM_STATUS_SE0:
1096 case GRBM_STATUS_SE1:
1097 case SRBM_STATUS:
1098 case SRBM_STATUS2:
1099 case DMA_STATUS_REG:
1100 case UVD_STATUS:
1101 *val = RREG32(reg);
1102 return 0;
1103 default:
1104 return -EINVAL;
1105 }
1106}
1107
1108void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1109 unsigned *bankh, unsigned *mtaspect,
1110 unsigned *tile_split)
1111{
1112 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1113 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1114 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1115 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1116 switch (*bankw) {
1117 default:
1118 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1119 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1120 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1121 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1122 }
1123 switch (*bankh) {
1124 default:
1125 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1126 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1127 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1128 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1129 }
1130 switch (*mtaspect) {
1131 default:
1132 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1133 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1134 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1135 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1136 }
1137}
1138
1139static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1140 u32 cntl_reg, u32 status_reg)
1141{
1142 int r, i;
1143 struct atom_clock_dividers dividers;
1144
1145 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1146 clock, false, ÷rs);
1147 if (r)
1148 return r;
1149
1150 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1151
1152 for (i = 0; i < 100; i++) {
1153 if (RREG32(status_reg) & DCLK_STATUS)
1154 break;
1155 mdelay(10);
1156 }
1157 if (i == 100)
1158 return -ETIMEDOUT;
1159
1160 return 0;
1161}
1162
1163int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1164{
1165 int r = 0;
1166 u32 cg_scratch = RREG32(CG_SCRATCH1);
1167
1168 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1169 if (r)
1170 goto done;
1171 cg_scratch &= 0xffff0000;
1172 cg_scratch |= vclk / 100; /* Mhz */
1173
1174 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1175 if (r)
1176 goto done;
1177 cg_scratch &= 0x0000ffff;
1178 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1179
1180done:
1181 WREG32(CG_SCRATCH1, cg_scratch);
1182
1183 return r;
1184}
1185
1186int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1187{
1188 /* start off with something large */
1189 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1190 int r;
1191
1192 /* bypass vclk and dclk with bclk */
1193 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1194 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1195 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1196
1197 /* put PLL in bypass mode */
1198 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1199
1200 if (!vclk || !dclk) {
1201 /* keep the Bypass mode, put PLL to sleep */
1202 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1203 return 0;
1204 }
1205
1206 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1207 16384, 0x03FFFFFF, 0, 128, 5,
1208 &fb_div, &vclk_div, &dclk_div);
1209 if (r)
1210 return r;
1211
1212 /* set VCO_MODE to 1 */
1213 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1214
1215 /* toggle UPLL_SLEEP to 1 then back to 0 */
1216 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1217 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1218
1219 /* deassert UPLL_RESET */
1220 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1221
1222 mdelay(1);
1223
1224 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1225 if (r)
1226 return r;
1227
1228 /* assert UPLL_RESET again */
1229 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1230
1231 /* disable spread spectrum. */
1232 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1233
1234 /* set feedback divider */
1235 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1236
1237 /* set ref divider to 0 */
1238 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1239
1240 if (fb_div < 307200)
1241 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1242 else
1243 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1244
1245 /* set PDIV_A and PDIV_B */
1246 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1247 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1248 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1249
1250 /* give the PLL some time to settle */
1251 mdelay(15);
1252
1253 /* deassert PLL_RESET */
1254 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1255
1256 mdelay(15);
1257
1258 /* switch from bypass mode to normal mode */
1259 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1260
1261 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1262 if (r)
1263 return r;
1264
1265 /* switch VCLK and DCLK selection */
1266 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1267 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1268 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1269
1270 mdelay(100);
1271
1272 return 0;
1273}
1274
1275void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1276{
1277 int readrq;
1278 u16 v;
1279
1280 readrq = pcie_get_readrq(rdev->pdev);
1281 v = ffs(readrq) - 8;
1282 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1283 * to avoid hangs or perfomance issues
1284 */
1285 if ((v == 0) || (v == 6) || (v == 7))
1286 pcie_set_readrq(rdev->pdev, 512);
1287}
1288
1289void dce4_program_fmt(struct drm_encoder *encoder)
1290{
1291 struct drm_device *dev = encoder->dev;
1292 struct radeon_device *rdev = dev->dev_private;
1293 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1294 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1295 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1296 int bpc = 0;
1297 u32 tmp = 0;
1298 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1299
1300 if (connector) {
1301 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1302 bpc = radeon_get_monitor_bpc(connector);
1303 dither = radeon_connector->dither;
1304 }
1305
1306 /* LVDS/eDP FMT is set up by atom */
1307 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1308 return;
1309
1310 /* not needed for analog */
1311 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1312 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1313 return;
1314
1315 if (bpc == 0)
1316 return;
1317
1318 switch (bpc) {
1319 case 6:
1320 if (dither == RADEON_FMT_DITHER_ENABLE)
1321 /* XXX sort out optimal dither settings */
1322 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1323 FMT_SPATIAL_DITHER_EN);
1324 else
1325 tmp |= FMT_TRUNCATE_EN;
1326 break;
1327 case 8:
1328 if (dither == RADEON_FMT_DITHER_ENABLE)
1329 /* XXX sort out optimal dither settings */
1330 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1331 FMT_RGB_RANDOM_ENABLE |
1332 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1333 else
1334 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1335 break;
1336 case 10:
1337 default:
1338 /* not needed */
1339 break;
1340 }
1341
1342 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1343}
1344
1345static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1346{
1347 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1348 return true;
1349 else
1350 return false;
1351}
1352
1353static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1354{
1355 u32 pos1, pos2;
1356
1357 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1358 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1359
1360 if (pos1 != pos2)
1361 return true;
1362 else
1363 return false;
1364}
1365
1366/**
1367 * dce4_wait_for_vblank - vblank wait asic callback.
1368 *
1369 * @rdev: radeon_device pointer
1370 * @crtc: crtc to wait for vblank on
1371 *
1372 * Wait for vblank on the requested crtc (evergreen+).
1373 */
1374void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1375{
1376 unsigned i = 0;
1377
1378 if (crtc >= rdev->num_crtc)
1379 return;
1380
1381 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1382 return;
1383
1384 /* depending on when we hit vblank, we may be close to active; if so,
1385 * wait for another frame.
1386 */
1387 while (dce4_is_in_vblank(rdev, crtc)) {
1388 if (i++ % 100 == 0) {
1389 if (!dce4_is_counter_moving(rdev, crtc))
1390 break;
1391 }
1392 }
1393
1394 while (!dce4_is_in_vblank(rdev, crtc)) {
1395 if (i++ % 100 == 0) {
1396 if (!dce4_is_counter_moving(rdev, crtc))
1397 break;
1398 }
1399 }
1400}
1401
1402/**
1403 * evergreen_page_flip - pageflip callback.
1404 *
1405 * @rdev: radeon_device pointer
1406 * @crtc_id: crtc to cleanup pageflip on
1407 * @crtc_base: new address of the crtc (GPU MC address)
1408 * @async: asynchronous flip
1409 *
1410 * Triggers the actual pageflip by updating the primary
1411 * surface base address (evergreen+).
1412 */
1413void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1414 bool async)
1415{
1416 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1417
1418 /* update the scanout addresses */
1419 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1420 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1421 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1422 upper_32_bits(crtc_base));
1423 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1424 (u32)crtc_base);
1425 /* post the write */
1426 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1427}
1428
1429/**
1430 * evergreen_page_flip_pending - check if page flip is still pending
1431 *
1432 * @rdev: radeon_device pointer
1433 * @crtc_id: crtc to check
1434 *
1435 * Returns the current update pending status.
1436 */
1437bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1438{
1439 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1440
1441 /* Return current update_pending status: */
1442 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1443 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1444}
1445
1446/* get temperature in millidegrees */
1447int evergreen_get_temp(struct radeon_device *rdev)
1448{
1449 u32 temp, toffset;
1450 int actual_temp = 0;
1451
1452 if (rdev->family == CHIP_JUNIPER) {
1453 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1454 TOFFSET_SHIFT;
1455 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1456 TS0_ADC_DOUT_SHIFT;
1457
1458 if (toffset & 0x100)
1459 actual_temp = temp / 2 - (0x200 - toffset);
1460 else
1461 actual_temp = temp / 2 + toffset;
1462
1463 actual_temp = actual_temp * 1000;
1464
1465 } else {
1466 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1467 ASIC_T_SHIFT;
1468
1469 if (temp & 0x400)
1470 actual_temp = -256;
1471 else if (temp & 0x200)
1472 actual_temp = 255;
1473 else if (temp & 0x100) {
1474 actual_temp = temp & 0x1ff;
1475 actual_temp |= ~0x1ff;
1476 } else
1477 actual_temp = temp & 0xff;
1478
1479 actual_temp = (actual_temp * 1000) / 2;
1480 }
1481
1482 return actual_temp;
1483}
1484
1485int sumo_get_temp(struct radeon_device *rdev)
1486{
1487 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1488 int actual_temp = temp - 49;
1489
1490 return actual_temp * 1000;
1491}
1492
1493/**
1494 * sumo_pm_init_profile - Initialize power profiles callback.
1495 *
1496 * @rdev: radeon_device pointer
1497 *
1498 * Initialize the power states used in profile mode
1499 * (sumo, trinity, SI).
1500 * Used for profile mode only.
1501 */
1502void sumo_pm_init_profile(struct radeon_device *rdev)
1503{
1504 int idx;
1505
1506 /* default */
1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1508 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1509 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1510 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1511
1512 /* low,mid sh/mh */
1513 if (rdev->flags & RADEON_IS_MOBILITY)
1514 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1515 else
1516 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1517
1518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1519 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1520 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1521 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1522
1523 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1525 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1526 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1527
1528 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1531 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1532
1533 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1534 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1536 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1537
1538 /* high sh/mh */
1539 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1540 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1544 rdev->pm.power_state[idx].num_clock_modes - 1;
1545
1546 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1547 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1548 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1550 rdev->pm.power_state[idx].num_clock_modes - 1;
1551}
1552
1553/**
1554 * btc_pm_init_profile - Initialize power profiles callback.
1555 *
1556 * @rdev: radeon_device pointer
1557 *
1558 * Initialize the power states used in profile mode
1559 * (BTC, cayman).
1560 * Used for profile mode only.
1561 */
1562void btc_pm_init_profile(struct radeon_device *rdev)
1563{
1564 int idx;
1565
1566 /* default */
1567 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1568 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1569 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1570 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1571 /* starting with BTC, there is one state that is used for both
1572 * MH and SH. Difference is that we always use the high clock index for
1573 * mclk.
1574 */
1575 if (rdev->flags & RADEON_IS_MOBILITY)
1576 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1577 else
1578 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1579 /* low sh */
1580 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1581 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1582 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1583 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1584 /* mid sh */
1585 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1586 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1587 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1588 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1589 /* high sh */
1590 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1591 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1592 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1593 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1594 /* low mh */
1595 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1596 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1597 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1598 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1599 /* mid mh */
1600 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1601 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1602 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1603 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1604 /* high mh */
1605 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1606 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1607 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1608 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1609}
1610
1611/**
1612 * evergreen_pm_misc - set additional pm hw parameters callback.
1613 *
1614 * @rdev: radeon_device pointer
1615 *
1616 * Set non-clock parameters associated with a power state
1617 * (voltage, etc.) (evergreen+).
1618 */
1619void evergreen_pm_misc(struct radeon_device *rdev)
1620{
1621 int req_ps_idx = rdev->pm.requested_power_state_index;
1622 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1623 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1624 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1625
1626 if (voltage->type == VOLTAGE_SW) {
1627 /* 0xff0x are flags rather then an actual voltage */
1628 if ((voltage->voltage & 0xff00) == 0xff00)
1629 return;
1630 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1631 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1632 rdev->pm.current_vddc = voltage->voltage;
1633 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1634 }
1635
1636 /* starting with BTC, there is one state that is used for both
1637 * MH and SH. Difference is that we always use the high clock index for
1638 * mclk and vddci.
1639 */
1640 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1641 (rdev->family >= CHIP_BARTS) &&
1642 rdev->pm.active_crtc_count &&
1643 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1644 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1645 voltage = &rdev->pm.power_state[req_ps_idx].
1646 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1647
1648 /* 0xff0x are flags rather then an actual voltage */
1649 if ((voltage->vddci & 0xff00) == 0xff00)
1650 return;
1651 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1652 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1653 rdev->pm.current_vddci = voltage->vddci;
1654 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1655 }
1656 }
1657}
1658
1659/**
1660 * evergreen_pm_prepare - pre-power state change callback.
1661 *
1662 * @rdev: radeon_device pointer
1663 *
1664 * Prepare for a power state change (evergreen+).
1665 */
1666void evergreen_pm_prepare(struct radeon_device *rdev)
1667{
1668 struct drm_device *ddev = rdev->ddev;
1669 struct drm_crtc *crtc;
1670 struct radeon_crtc *radeon_crtc;
1671 u32 tmp;
1672
1673 /* disable any active CRTCs */
1674 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1675 radeon_crtc = to_radeon_crtc(crtc);
1676 if (radeon_crtc->enabled) {
1677 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1678 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1679 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1680 }
1681 }
1682}
1683
1684/**
1685 * evergreen_pm_finish - post-power state change callback.
1686 *
1687 * @rdev: radeon_device pointer
1688 *
1689 * Clean up after a power state change (evergreen+).
1690 */
1691void evergreen_pm_finish(struct radeon_device *rdev)
1692{
1693 struct drm_device *ddev = rdev->ddev;
1694 struct drm_crtc *crtc;
1695 struct radeon_crtc *radeon_crtc;
1696 u32 tmp;
1697
1698 /* enable any active CRTCs */
1699 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1700 radeon_crtc = to_radeon_crtc(crtc);
1701 if (radeon_crtc->enabled) {
1702 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1703 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1704 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1705 }
1706 }
1707}
1708
1709/**
1710 * evergreen_hpd_sense - hpd sense callback.
1711 *
1712 * @rdev: radeon_device pointer
1713 * @hpd: hpd (hotplug detect) pin
1714 *
1715 * Checks if a digital monitor is connected (evergreen+).
1716 * Returns true if connected, false if not connected.
1717 */
1718bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1719{
1720 if (hpd == RADEON_HPD_NONE)
1721 return false;
1722
1723 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1724}
1725
1726/**
1727 * evergreen_hpd_set_polarity - hpd set polarity callback.
1728 *
1729 * @rdev: radeon_device pointer
1730 * @hpd: hpd (hotplug detect) pin
1731 *
1732 * Set the polarity of the hpd pin (evergreen+).
1733 */
1734void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1735 enum radeon_hpd_id hpd)
1736{
1737 bool connected = evergreen_hpd_sense(rdev, hpd);
1738
1739 if (hpd == RADEON_HPD_NONE)
1740 return;
1741
1742 if (connected)
1743 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1744 else
1745 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1746}
1747
1748/**
1749 * evergreen_hpd_init - hpd setup callback.
1750 *
1751 * @rdev: radeon_device pointer
1752 *
1753 * Setup the hpd pins used by the card (evergreen+).
1754 * Enable the pin, set the polarity, and enable the hpd interrupts.
1755 */
1756void evergreen_hpd_init(struct radeon_device *rdev)
1757{
1758 struct drm_device *dev = rdev->ddev;
1759 struct drm_connector *connector;
1760 unsigned enabled = 0;
1761 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1762 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1763
1764 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1765 enum radeon_hpd_id hpd =
1766 to_radeon_connector(connector)->hpd.hpd;
1767
1768 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1769 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1770 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1771 * aux dp channel on imac and help (but not completely fix)
1772 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1773 * also avoid interrupt storms during dpms.
1774 */
1775 continue;
1776 }
1777
1778 if (hpd == RADEON_HPD_NONE)
1779 continue;
1780
1781 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1782 enabled |= 1 << hpd;
1783
1784 radeon_hpd_set_polarity(rdev, hpd);
1785 }
1786 radeon_irq_kms_enable_hpd(rdev, enabled);
1787}
1788
1789/**
1790 * evergreen_hpd_fini - hpd tear down callback.
1791 *
1792 * @rdev: radeon_device pointer
1793 *
1794 * Tear down the hpd pins used by the card (evergreen+).
1795 * Disable the hpd interrupts.
1796 */
1797void evergreen_hpd_fini(struct radeon_device *rdev)
1798{
1799 struct drm_device *dev = rdev->ddev;
1800 struct drm_connector *connector;
1801 unsigned disabled = 0;
1802
1803 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1804 enum radeon_hpd_id hpd =
1805 to_radeon_connector(connector)->hpd.hpd;
1806
1807 if (hpd == RADEON_HPD_NONE)
1808 continue;
1809
1810 WREG32(DC_HPDx_CONTROL(hpd), 0);
1811 disabled |= 1 << hpd;
1812 }
1813 radeon_irq_kms_disable_hpd(rdev, disabled);
1814}
1815
1816/* watermark setup */
1817
1818static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1819 struct radeon_crtc *radeon_crtc,
1820 struct drm_display_mode *mode,
1821 struct drm_display_mode *other_mode)
1822{
1823 u32 tmp, buffer_alloc, i;
1824 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1825 /*
1826 * Line Buffer Setup
1827 * There are 3 line buffers, each one shared by 2 display controllers.
1828 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1829 * the display controllers. The paritioning is done via one of four
1830 * preset allocations specified in bits 2:0:
1831 * first display controller
1832 * 0 - first half of lb (3840 * 2)
1833 * 1 - first 3/4 of lb (5760 * 2)
1834 * 2 - whole lb (7680 * 2), other crtc must be disabled
1835 * 3 - first 1/4 of lb (1920 * 2)
1836 * second display controller
1837 * 4 - second half of lb (3840 * 2)
1838 * 5 - second 3/4 of lb (5760 * 2)
1839 * 6 - whole lb (7680 * 2), other crtc must be disabled
1840 * 7 - last 1/4 of lb (1920 * 2)
1841 */
1842 /* this can get tricky if we have two large displays on a paired group
1843 * of crtcs. Ideally for multiple large displays we'd assign them to
1844 * non-linked crtcs for maximum line buffer allocation.
1845 */
1846 if (radeon_crtc->base.enabled && mode) {
1847 if (other_mode) {
1848 tmp = 0; /* 1/2 */
1849 buffer_alloc = 1;
1850 } else {
1851 tmp = 2; /* whole */
1852 buffer_alloc = 2;
1853 }
1854 } else {
1855 tmp = 0;
1856 buffer_alloc = 0;
1857 }
1858
1859 /* second controller of the pair uses second half of the lb */
1860 if (radeon_crtc->crtc_id % 2)
1861 tmp += 4;
1862 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1863
1864 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1865 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1866 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1867 for (i = 0; i < rdev->usec_timeout; i++) {
1868 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1869 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1870 break;
1871 udelay(1);
1872 }
1873 }
1874
1875 if (radeon_crtc->base.enabled && mode) {
1876 switch (tmp) {
1877 case 0:
1878 case 4:
1879 default:
1880 if (ASIC_IS_DCE5(rdev))
1881 return 4096 * 2;
1882 else
1883 return 3840 * 2;
1884 case 1:
1885 case 5:
1886 if (ASIC_IS_DCE5(rdev))
1887 return 6144 * 2;
1888 else
1889 return 5760 * 2;
1890 case 2:
1891 case 6:
1892 if (ASIC_IS_DCE5(rdev))
1893 return 8192 * 2;
1894 else
1895 return 7680 * 2;
1896 case 3:
1897 case 7:
1898 if (ASIC_IS_DCE5(rdev))
1899 return 2048 * 2;
1900 else
1901 return 1920 * 2;
1902 }
1903 }
1904
1905 /* controller not enabled, so no lb used */
1906 return 0;
1907}
1908
1909u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1910{
1911 u32 tmp = RREG32(MC_SHARED_CHMAP);
1912
1913 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1914 case 0:
1915 default:
1916 return 1;
1917 case 1:
1918 return 2;
1919 case 2:
1920 return 4;
1921 case 3:
1922 return 8;
1923 }
1924}
1925
1926struct evergreen_wm_params {
1927 u32 dram_channels; /* number of dram channels */
1928 u32 yclk; /* bandwidth per dram data pin in kHz */
1929 u32 sclk; /* engine clock in kHz */
1930 u32 disp_clk; /* display clock in kHz */
1931 u32 src_width; /* viewport width */
1932 u32 active_time; /* active display time in ns */
1933 u32 blank_time; /* blank time in ns */
1934 bool interlaced; /* mode is interlaced */
1935 fixed20_12 vsc; /* vertical scale ratio */
1936 u32 num_heads; /* number of active crtcs */
1937 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1938 u32 lb_size; /* line buffer allocated to pipe */
1939 u32 vtaps; /* vertical scaler taps */
1940};
1941
1942static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1943{
1944 /* Calculate DRAM Bandwidth and the part allocated to display. */
1945 fixed20_12 dram_efficiency; /* 0.7 */
1946 fixed20_12 yclk, dram_channels, bandwidth;
1947 fixed20_12 a;
1948
1949 a.full = dfixed_const(1000);
1950 yclk.full = dfixed_const(wm->yclk);
1951 yclk.full = dfixed_div(yclk, a);
1952 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1953 a.full = dfixed_const(10);
1954 dram_efficiency.full = dfixed_const(7);
1955 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1956 bandwidth.full = dfixed_mul(dram_channels, yclk);
1957 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1958
1959 return dfixed_trunc(bandwidth);
1960}
1961
1962static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1963{
1964 /* Calculate DRAM Bandwidth and the part allocated to display. */
1965 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1966 fixed20_12 yclk, dram_channels, bandwidth;
1967 fixed20_12 a;
1968
1969 a.full = dfixed_const(1000);
1970 yclk.full = dfixed_const(wm->yclk);
1971 yclk.full = dfixed_div(yclk, a);
1972 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1973 a.full = dfixed_const(10);
1974 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1975 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1976 bandwidth.full = dfixed_mul(dram_channels, yclk);
1977 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1978
1979 return dfixed_trunc(bandwidth);
1980}
1981
1982static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1983{
1984 /* Calculate the display Data return Bandwidth */
1985 fixed20_12 return_efficiency; /* 0.8 */
1986 fixed20_12 sclk, bandwidth;
1987 fixed20_12 a;
1988
1989 a.full = dfixed_const(1000);
1990 sclk.full = dfixed_const(wm->sclk);
1991 sclk.full = dfixed_div(sclk, a);
1992 a.full = dfixed_const(10);
1993 return_efficiency.full = dfixed_const(8);
1994 return_efficiency.full = dfixed_div(return_efficiency, a);
1995 a.full = dfixed_const(32);
1996 bandwidth.full = dfixed_mul(a, sclk);
1997 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1998
1999 return dfixed_trunc(bandwidth);
2000}
2001
2002static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2003{
2004 /* Calculate the DMIF Request Bandwidth */
2005 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2006 fixed20_12 disp_clk, bandwidth;
2007 fixed20_12 a;
2008
2009 a.full = dfixed_const(1000);
2010 disp_clk.full = dfixed_const(wm->disp_clk);
2011 disp_clk.full = dfixed_div(disp_clk, a);
2012 a.full = dfixed_const(10);
2013 disp_clk_request_efficiency.full = dfixed_const(8);
2014 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2015 a.full = dfixed_const(32);
2016 bandwidth.full = dfixed_mul(a, disp_clk);
2017 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2018
2019 return dfixed_trunc(bandwidth);
2020}
2021
2022static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2023{
2024 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2025 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2026 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2027 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2028
2029 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2030}
2031
2032static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2033{
2034 /* Calculate the display mode Average Bandwidth
2035 * DisplayMode should contain the source and destination dimensions,
2036 * timing, etc.
2037 */
2038 fixed20_12 bpp;
2039 fixed20_12 line_time;
2040 fixed20_12 src_width;
2041 fixed20_12 bandwidth;
2042 fixed20_12 a;
2043
2044 a.full = dfixed_const(1000);
2045 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2046 line_time.full = dfixed_div(line_time, a);
2047 bpp.full = dfixed_const(wm->bytes_per_pixel);
2048 src_width.full = dfixed_const(wm->src_width);
2049 bandwidth.full = dfixed_mul(src_width, bpp);
2050 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2051 bandwidth.full = dfixed_div(bandwidth, line_time);
2052
2053 return dfixed_trunc(bandwidth);
2054}
2055
2056static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2057{
2058 /* First calcualte the latency in ns */
2059 u32 mc_latency = 2000; /* 2000 ns. */
2060 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2061 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2062 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2063 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2064 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2065 (wm->num_heads * cursor_line_pair_return_time);
2066 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2067 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2068 fixed20_12 a, b, c;
2069
2070 if (wm->num_heads == 0)
2071 return 0;
2072
2073 a.full = dfixed_const(2);
2074 b.full = dfixed_const(1);
2075 if ((wm->vsc.full > a.full) ||
2076 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2077 (wm->vtaps >= 5) ||
2078 ((wm->vsc.full >= a.full) && wm->interlaced))
2079 max_src_lines_per_dst_line = 4;
2080 else
2081 max_src_lines_per_dst_line = 2;
2082
2083 a.full = dfixed_const(available_bandwidth);
2084 b.full = dfixed_const(wm->num_heads);
2085 a.full = dfixed_div(a, b);
2086
2087 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2088
2089 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2090 b.full = dfixed_const(1000);
2091 c.full = dfixed_const(lb_fill_bw);
2092 b.full = dfixed_div(c, b);
2093 a.full = dfixed_div(a, b);
2094 line_fill_time = dfixed_trunc(a);
2095
2096 if (line_fill_time < wm->active_time)
2097 return latency;
2098 else
2099 return latency + (line_fill_time - wm->active_time);
2100
2101}
2102
2103static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2104{
2105 if (evergreen_average_bandwidth(wm) <=
2106 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2107 return true;
2108 else
2109 return false;
2110};
2111
2112static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2113{
2114 if (evergreen_average_bandwidth(wm) <=
2115 (evergreen_available_bandwidth(wm) / wm->num_heads))
2116 return true;
2117 else
2118 return false;
2119};
2120
2121static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2122{
2123 u32 lb_partitions = wm->lb_size / wm->src_width;
2124 u32 line_time = wm->active_time + wm->blank_time;
2125 u32 latency_tolerant_lines;
2126 u32 latency_hiding;
2127 fixed20_12 a;
2128
2129 a.full = dfixed_const(1);
2130 if (wm->vsc.full > a.full)
2131 latency_tolerant_lines = 1;
2132 else {
2133 if (lb_partitions <= (wm->vtaps + 1))
2134 latency_tolerant_lines = 1;
2135 else
2136 latency_tolerant_lines = 2;
2137 }
2138
2139 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2140
2141 if (evergreen_latency_watermark(wm) <= latency_hiding)
2142 return true;
2143 else
2144 return false;
2145}
2146
2147static void evergreen_program_watermarks(struct radeon_device *rdev,
2148 struct radeon_crtc *radeon_crtc,
2149 u32 lb_size, u32 num_heads)
2150{
2151 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2152 struct evergreen_wm_params wm_low, wm_high;
2153 u32 dram_channels;
2154 u32 active_time;
2155 u32 line_time = 0;
2156 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2157 u32 priority_a_mark = 0, priority_b_mark = 0;
2158 u32 priority_a_cnt = PRIORITY_OFF;
2159 u32 priority_b_cnt = PRIORITY_OFF;
2160 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2161 u32 tmp, arb_control3;
2162 fixed20_12 a, b, c;
2163
2164 if (radeon_crtc->base.enabled && num_heads && mode) {
2165 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2166 (u32)mode->clock);
2167 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2168 (u32)mode->clock);
2169 line_time = min(line_time, (u32)65535);
2170 priority_a_cnt = 0;
2171 priority_b_cnt = 0;
2172 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2173
2174 /* watermark for high clocks */
2175 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2176 wm_high.yclk =
2177 radeon_dpm_get_mclk(rdev, false) * 10;
2178 wm_high.sclk =
2179 radeon_dpm_get_sclk(rdev, false) * 10;
2180 } else {
2181 wm_high.yclk = rdev->pm.current_mclk * 10;
2182 wm_high.sclk = rdev->pm.current_sclk * 10;
2183 }
2184
2185 wm_high.disp_clk = mode->clock;
2186 wm_high.src_width = mode->crtc_hdisplay;
2187 wm_high.active_time = active_time;
2188 wm_high.blank_time = line_time - wm_high.active_time;
2189 wm_high.interlaced = false;
2190 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2191 wm_high.interlaced = true;
2192 wm_high.vsc = radeon_crtc->vsc;
2193 wm_high.vtaps = 1;
2194 if (radeon_crtc->rmx_type != RMX_OFF)
2195 wm_high.vtaps = 2;
2196 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2197 wm_high.lb_size = lb_size;
2198 wm_high.dram_channels = dram_channels;
2199 wm_high.num_heads = num_heads;
2200
2201 /* watermark for low clocks */
2202 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2203 wm_low.yclk =
2204 radeon_dpm_get_mclk(rdev, true) * 10;
2205 wm_low.sclk =
2206 radeon_dpm_get_sclk(rdev, true) * 10;
2207 } else {
2208 wm_low.yclk = rdev->pm.current_mclk * 10;
2209 wm_low.sclk = rdev->pm.current_sclk * 10;
2210 }
2211
2212 wm_low.disp_clk = mode->clock;
2213 wm_low.src_width = mode->crtc_hdisplay;
2214 wm_low.active_time = active_time;
2215 wm_low.blank_time = line_time - wm_low.active_time;
2216 wm_low.interlaced = false;
2217 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2218 wm_low.interlaced = true;
2219 wm_low.vsc = radeon_crtc->vsc;
2220 wm_low.vtaps = 1;
2221 if (radeon_crtc->rmx_type != RMX_OFF)
2222 wm_low.vtaps = 2;
2223 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2224 wm_low.lb_size = lb_size;
2225 wm_low.dram_channels = dram_channels;
2226 wm_low.num_heads = num_heads;
2227
2228 /* set for high clocks */
2229 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2230 /* set for low clocks */
2231 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2232
2233 /* possibly force display priority to high */
2234 /* should really do this at mode validation time... */
2235 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2236 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2237 !evergreen_check_latency_hiding(&wm_high) ||
2238 (rdev->disp_priority == 2)) {
2239 DRM_DEBUG_KMS("force priority a to high\n");
2240 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2241 }
2242 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2243 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2244 !evergreen_check_latency_hiding(&wm_low) ||
2245 (rdev->disp_priority == 2)) {
2246 DRM_DEBUG_KMS("force priority b to high\n");
2247 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2248 }
2249
2250 a.full = dfixed_const(1000);
2251 b.full = dfixed_const(mode->clock);
2252 b.full = dfixed_div(b, a);
2253 c.full = dfixed_const(latency_watermark_a);
2254 c.full = dfixed_mul(c, b);
2255 c.full = dfixed_mul(c, radeon_crtc->hsc);
2256 c.full = dfixed_div(c, a);
2257 a.full = dfixed_const(16);
2258 c.full = dfixed_div(c, a);
2259 priority_a_mark = dfixed_trunc(c);
2260 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2261
2262 a.full = dfixed_const(1000);
2263 b.full = dfixed_const(mode->clock);
2264 b.full = dfixed_div(b, a);
2265 c.full = dfixed_const(latency_watermark_b);
2266 c.full = dfixed_mul(c, b);
2267 c.full = dfixed_mul(c, radeon_crtc->hsc);
2268 c.full = dfixed_div(c, a);
2269 a.full = dfixed_const(16);
2270 c.full = dfixed_div(c, a);
2271 priority_b_mark = dfixed_trunc(c);
2272 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2273
2274 /* Save number of lines the linebuffer leads before the scanout */
2275 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2276 }
2277
2278 /* select wm A */
2279 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2280 tmp = arb_control3;
2281 tmp &= ~LATENCY_WATERMARK_MASK(3);
2282 tmp |= LATENCY_WATERMARK_MASK(1);
2283 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2284 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2285 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2286 LATENCY_HIGH_WATERMARK(line_time)));
2287 /* select wm B */
2288 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2289 tmp &= ~LATENCY_WATERMARK_MASK(3);
2290 tmp |= LATENCY_WATERMARK_MASK(2);
2291 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2294 LATENCY_HIGH_WATERMARK(line_time)));
2295 /* restore original selection */
2296 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2297
2298 /* write the priority marks */
2299 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2300 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2301
2302 /* save values for DPM */
2303 radeon_crtc->line_time = line_time;
2304 radeon_crtc->wm_high = latency_watermark_a;
2305 radeon_crtc->wm_low = latency_watermark_b;
2306}
2307
2308/**
2309 * evergreen_bandwidth_update - update display watermarks callback.
2310 *
2311 * @rdev: radeon_device pointer
2312 *
2313 * Update the display watermarks based on the requested mode(s)
2314 * (evergreen+).
2315 */
2316void evergreen_bandwidth_update(struct radeon_device *rdev)
2317{
2318 struct drm_display_mode *mode0 = NULL;
2319 struct drm_display_mode *mode1 = NULL;
2320 u32 num_heads = 0, lb_size;
2321 int i;
2322
2323 if (!rdev->mode_info.mode_config_initialized)
2324 return;
2325
2326 radeon_update_display_priority(rdev);
2327
2328 for (i = 0; i < rdev->num_crtc; i++) {
2329 if (rdev->mode_info.crtcs[i]->base.enabled)
2330 num_heads++;
2331 }
2332 for (i = 0; i < rdev->num_crtc; i += 2) {
2333 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2334 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2335 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2336 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2337 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2338 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2339 }
2340}
2341
2342/**
2343 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2344 *
2345 * @rdev: radeon_device pointer
2346 *
2347 * Wait for the MC (memory controller) to be idle.
2348 * (evergreen+).
2349 * Returns 0 if the MC is idle, -1 if not.
2350 */
2351int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2352{
2353 unsigned i;
2354 u32 tmp;
2355
2356 for (i = 0; i < rdev->usec_timeout; i++) {
2357 /* read MC_STATUS */
2358 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2359 if (!tmp)
2360 return 0;
2361 udelay(1);
2362 }
2363 return -1;
2364}
2365
2366/*
2367 * GART
2368 */
2369void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2370{
2371 unsigned i;
2372 u32 tmp;
2373
2374 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2375
2376 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2377 for (i = 0; i < rdev->usec_timeout; i++) {
2378 /* read MC_STATUS */
2379 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2380 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2381 if (tmp == 2) {
2382 pr_warn("[drm] r600 flush TLB failed\n");
2383 return;
2384 }
2385 if (tmp) {
2386 return;
2387 }
2388 udelay(1);
2389 }
2390}
2391
2392static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2393{
2394 u32 tmp;
2395 int r;
2396
2397 if (rdev->gart.robj == NULL) {
2398 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2399 return -EINVAL;
2400 }
2401 r = radeon_gart_table_vram_pin(rdev);
2402 if (r)
2403 return r;
2404 /* Setup L2 cache */
2405 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2406 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2407 EFFECTIVE_L2_QUEUE_SIZE(7));
2408 WREG32(VM_L2_CNTL2, 0);
2409 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2410 /* Setup TLB control */
2411 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2412 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2413 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2414 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2415 if (rdev->flags & RADEON_IS_IGP) {
2416 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2417 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2418 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2419 } else {
2420 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2421 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2422 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2423 if ((rdev->family == CHIP_JUNIPER) ||
2424 (rdev->family == CHIP_CYPRESS) ||
2425 (rdev->family == CHIP_HEMLOCK) ||
2426 (rdev->family == CHIP_BARTS))
2427 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2428 }
2429 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2430 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2431 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2432 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2433 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2434 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2435 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2436 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2437 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2438 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2439 (u32)(rdev->dummy_page.addr >> 12));
2440 WREG32(VM_CONTEXT1_CNTL, 0);
2441
2442 evergreen_pcie_gart_tlb_flush(rdev);
2443 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2444 (unsigned)(rdev->mc.gtt_size >> 20),
2445 (unsigned long long)rdev->gart.table_addr);
2446 rdev->gart.ready = true;
2447 return 0;
2448}
2449
2450static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2451{
2452 u32 tmp;
2453
2454 /* Disable all tables */
2455 WREG32(VM_CONTEXT0_CNTL, 0);
2456 WREG32(VM_CONTEXT1_CNTL, 0);
2457
2458 /* Setup L2 cache */
2459 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2460 EFFECTIVE_L2_QUEUE_SIZE(7));
2461 WREG32(VM_L2_CNTL2, 0);
2462 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2463 /* Setup TLB control */
2464 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2465 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2466 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2468 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2472 radeon_gart_table_vram_unpin(rdev);
2473}
2474
2475static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2476{
2477 evergreen_pcie_gart_disable(rdev);
2478 radeon_gart_table_vram_free(rdev);
2479 radeon_gart_fini(rdev);
2480}
2481
2482
2483static void evergreen_agp_enable(struct radeon_device *rdev)
2484{
2485 u32 tmp;
2486
2487 /* Setup L2 cache */
2488 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2489 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2490 EFFECTIVE_L2_QUEUE_SIZE(7));
2491 WREG32(VM_L2_CNTL2, 0);
2492 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2493 /* Setup TLB control */
2494 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2495 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2496 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2497 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2498 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2499 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2500 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2501 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2502 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2503 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2504 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2505 WREG32(VM_CONTEXT0_CNTL, 0);
2506 WREG32(VM_CONTEXT1_CNTL, 0);
2507}
2508
2509static const unsigned ni_dig_offsets[] =
2510{
2511 NI_DIG0_REGISTER_OFFSET,
2512 NI_DIG1_REGISTER_OFFSET,
2513 NI_DIG2_REGISTER_OFFSET,
2514 NI_DIG3_REGISTER_OFFSET,
2515 NI_DIG4_REGISTER_OFFSET,
2516 NI_DIG5_REGISTER_OFFSET
2517};
2518
2519static const unsigned ni_tx_offsets[] =
2520{
2521 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2522 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2523 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2524 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2525 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2526 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2527};
2528
2529static const unsigned evergreen_dp_offsets[] =
2530{
2531 EVERGREEN_DP0_REGISTER_OFFSET,
2532 EVERGREEN_DP1_REGISTER_OFFSET,
2533 EVERGREEN_DP2_REGISTER_OFFSET,
2534 EVERGREEN_DP3_REGISTER_OFFSET,
2535 EVERGREEN_DP4_REGISTER_OFFSET,
2536 EVERGREEN_DP5_REGISTER_OFFSET
2537};
2538
2539static const unsigned evergreen_disp_int_status[] =
2540{
2541 DISP_INTERRUPT_STATUS,
2542 DISP_INTERRUPT_STATUS_CONTINUE,
2543 DISP_INTERRUPT_STATUS_CONTINUE2,
2544 DISP_INTERRUPT_STATUS_CONTINUE3,
2545 DISP_INTERRUPT_STATUS_CONTINUE4,
2546 DISP_INTERRUPT_STATUS_CONTINUE5
2547};
2548
2549/*
2550 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2551 * We go from crtc to connector and it is not relible since it
2552 * should be an opposite direction .If crtc is enable then
2553 * find the dig_fe which selects this crtc and insure that it enable.
2554 * if such dig_fe is found then find dig_be which selects found dig_be and
2555 * insure that it enable and in DP_SST mode.
2556 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2557 * from dp symbols clocks .
2558 */
2559static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2560 unsigned crtc_id, unsigned *ret_dig_fe)
2561{
2562 unsigned i;
2563 unsigned dig_fe;
2564 unsigned dig_be;
2565 unsigned dig_en_be;
2566 unsigned uniphy_pll;
2567 unsigned digs_fe_selected;
2568 unsigned dig_be_mode;
2569 unsigned dig_fe_mask;
2570 bool is_enabled = false;
2571 bool found_crtc = false;
2572
2573 /* loop through all running dig_fe to find selected crtc */
2574 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2575 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2576 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2577 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2578 /* found running pipe */
2579 found_crtc = true;
2580 dig_fe_mask = 1 << i;
2581 dig_fe = i;
2582 break;
2583 }
2584 }
2585
2586 if (found_crtc) {
2587 /* loop through all running dig_be to find selected dig_fe */
2588 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2589 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2590 /* if dig_fe_selected by dig_be? */
2591 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2592 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2593 if (dig_fe_mask & digs_fe_selected &&
2594 /* if dig_be in sst mode? */
2595 dig_be_mode == NI_DIG_BE_DPSST) {
2596 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2597 ni_dig_offsets[i]);
2598 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2599 ni_tx_offsets[i]);
2600 /* dig_be enable and tx is running */
2601 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2602 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2603 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2604 is_enabled = true;
2605 *ret_dig_fe = dig_fe;
2606 break;
2607 }
2608 }
2609 }
2610 }
2611
2612 return is_enabled;
2613}
2614
2615/*
2616 * Blank dig when in dp sst mode
2617 * Dig ignores crtc timing
2618 */
2619static void evergreen_blank_dp_output(struct radeon_device *rdev,
2620 unsigned dig_fe)
2621{
2622 unsigned stream_ctrl;
2623 unsigned fifo_ctrl;
2624 unsigned counter = 0;
2625
2626 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2627 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2628 return;
2629 }
2630
2631 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2632 evergreen_dp_offsets[dig_fe]);
2633 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2634 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2635 return;
2636 }
2637
2638 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2639 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2640 evergreen_dp_offsets[dig_fe], stream_ctrl);
2641
2642 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2643 evergreen_dp_offsets[dig_fe]);
2644 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2645 msleep(1);
2646 counter++;
2647 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648 evergreen_dp_offsets[dig_fe]);
2649 }
2650 if (counter >= 32 )
2651 DRM_ERROR("counter exceeds %d\n", counter);
2652
2653 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2654 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2655 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2656
2657}
2658
2659void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2660{
2661 u32 crtc_enabled, tmp, frame_count, blackout;
2662 int i, j;
2663 unsigned dig_fe;
2664
2665 if (!ASIC_IS_NODCE(rdev)) {
2666 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2667 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2668
2669 /* disable VGA render */
2670 WREG32(VGA_RENDER_CONTROL, 0);
2671 }
2672 /* blank the display controllers */
2673 for (i = 0; i < rdev->num_crtc; i++) {
2674 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2675 if (crtc_enabled) {
2676 save->crtc_enabled[i] = true;
2677 if (ASIC_IS_DCE6(rdev)) {
2678 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2679 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2680 radeon_wait_for_vblank(rdev, i);
2681 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2682 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2683 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2684 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2685 }
2686 } else {
2687 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2688 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2689 radeon_wait_for_vblank(rdev, i);
2690 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2691 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2692 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2693 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2694 }
2695 }
2696 /* wait for the next frame */
2697 frame_count = radeon_get_vblank_counter(rdev, i);
2698 for (j = 0; j < rdev->usec_timeout; j++) {
2699 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2700 break;
2701 udelay(1);
2702 }
2703 /*we should disable dig if it drives dp sst*/
2704 /*but we are in radeon_device_init and the topology is unknown*/
2705 /*and it is available after radeon_modeset_init*/
2706 /*the following method radeon_atom_encoder_dpms_dig*/
2707 /*does the job if we initialize it properly*/
2708 /*for now we do it this manually*/
2709 /**/
2710 if (ASIC_IS_DCE5(rdev) &&
2711 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2712 evergreen_blank_dp_output(rdev, dig_fe);
2713 /*we could remove 6 lines below*/
2714 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2715 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2716 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2717 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2718 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2719 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2720 save->crtc_enabled[i] = false;
2721 /* ***** */
2722 } else {
2723 save->crtc_enabled[i] = false;
2724 }
2725 }
2726
2727 radeon_mc_wait_for_idle(rdev);
2728
2729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2730 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2731 /* Block CPU access */
2732 WREG32(BIF_FB_EN, 0);
2733 /* blackout the MC */
2734 blackout &= ~BLACKOUT_MODE_MASK;
2735 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2736 }
2737 /* wait for the MC to settle */
2738 udelay(100);
2739
2740 /* lock double buffered regs */
2741 for (i = 0; i < rdev->num_crtc; i++) {
2742 if (save->crtc_enabled[i]) {
2743 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2744 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2745 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2746 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2747 }
2748 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2749 if (!(tmp & 1)) {
2750 tmp |= 1;
2751 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2752 }
2753 }
2754 }
2755}
2756
2757void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2758{
2759 u32 tmp, frame_count;
2760 int i, j;
2761
2762 /* update crtc base addresses */
2763 for (i = 0; i < rdev->num_crtc; i++) {
2764 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2765 upper_32_bits(rdev->mc.vram_start));
2766 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2767 upper_32_bits(rdev->mc.vram_start));
2768 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2769 (u32)rdev->mc.vram_start);
2770 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2771 (u32)rdev->mc.vram_start);
2772 }
2773
2774 if (!ASIC_IS_NODCE(rdev)) {
2775 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2776 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2777 }
2778
2779 /* unlock regs and wait for update */
2780 for (i = 0; i < rdev->num_crtc; i++) {
2781 if (save->crtc_enabled[i]) {
2782 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2783 if ((tmp & 0x7) != 0) {
2784 tmp &= ~0x7;
2785 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2786 }
2787 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2788 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2789 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2790 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2791 }
2792 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2793 if (tmp & 1) {
2794 tmp &= ~1;
2795 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2796 }
2797 for (j = 0; j < rdev->usec_timeout; j++) {
2798 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2799 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2800 break;
2801 udelay(1);
2802 }
2803 }
2804 }
2805
2806 /* unblackout the MC */
2807 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2808 tmp &= ~BLACKOUT_MODE_MASK;
2809 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2810 /* allow CPU access */
2811 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2812
2813 for (i = 0; i < rdev->num_crtc; i++) {
2814 if (save->crtc_enabled[i]) {
2815 if (ASIC_IS_DCE6(rdev)) {
2816 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2817 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2818 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2819 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2820 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2821 } else {
2822 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2823 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2824 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2825 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2826 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2827 }
2828 /* wait for the next frame */
2829 frame_count = radeon_get_vblank_counter(rdev, i);
2830 for (j = 0; j < rdev->usec_timeout; j++) {
2831 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2832 break;
2833 udelay(1);
2834 }
2835 }
2836 }
2837 if (!ASIC_IS_NODCE(rdev)) {
2838 /* Unlock vga access */
2839 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2840 mdelay(1);
2841 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2842 }
2843}
2844
2845void evergreen_mc_program(struct radeon_device *rdev)
2846{
2847 struct evergreen_mc_save save;
2848 u32 tmp;
2849 int i, j;
2850
2851 /* Initialize HDP */
2852 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2853 WREG32((0x2c14 + j), 0x00000000);
2854 WREG32((0x2c18 + j), 0x00000000);
2855 WREG32((0x2c1c + j), 0x00000000);
2856 WREG32((0x2c20 + j), 0x00000000);
2857 WREG32((0x2c24 + j), 0x00000000);
2858 }
2859 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2860
2861 evergreen_mc_stop(rdev, &save);
2862 if (evergreen_mc_wait_for_idle(rdev)) {
2863 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2864 }
2865 /* Lockout access through VGA aperture*/
2866 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2867 /* Update configuration */
2868 if (rdev->flags & RADEON_IS_AGP) {
2869 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2870 /* VRAM before AGP */
2871 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2872 rdev->mc.vram_start >> 12);
2873 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2874 rdev->mc.gtt_end >> 12);
2875 } else {
2876 /* VRAM after AGP */
2877 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2878 rdev->mc.gtt_start >> 12);
2879 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2880 rdev->mc.vram_end >> 12);
2881 }
2882 } else {
2883 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2884 rdev->mc.vram_start >> 12);
2885 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2886 rdev->mc.vram_end >> 12);
2887 }
2888 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2889 /* llano/ontario only */
2890 if ((rdev->family == CHIP_PALM) ||
2891 (rdev->family == CHIP_SUMO) ||
2892 (rdev->family == CHIP_SUMO2)) {
2893 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2894 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2895 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2896 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2897 }
2898 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2899 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2900 WREG32(MC_VM_FB_LOCATION, tmp);
2901 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2902 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2903 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2904 if (rdev->flags & RADEON_IS_AGP) {
2905 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2906 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2907 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2908 } else {
2909 WREG32(MC_VM_AGP_BASE, 0);
2910 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2911 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2912 }
2913 if (evergreen_mc_wait_for_idle(rdev)) {
2914 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2915 }
2916 evergreen_mc_resume(rdev, &save);
2917 /* we need to own VRAM, so turn off the VGA renderer here
2918 * to stop it overwriting our objects */
2919 rv515_vga_render_disable(rdev);
2920}
2921
2922/*
2923 * CP.
2924 */
2925void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2926{
2927 struct radeon_ring *ring = &rdev->ring[ib->ring];
2928 u32 next_rptr;
2929
2930 /* set to DX10/11 mode */
2931 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2932 radeon_ring_write(ring, 1);
2933
2934 if (ring->rptr_save_reg) {
2935 next_rptr = ring->wptr + 3 + 4;
2936 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2937 radeon_ring_write(ring, ((ring->rptr_save_reg -
2938 PACKET3_SET_CONFIG_REG_START) >> 2));
2939 radeon_ring_write(ring, next_rptr);
2940 } else if (rdev->wb.enabled) {
2941 next_rptr = ring->wptr + 5 + 4;
2942 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2943 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2944 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2945 radeon_ring_write(ring, next_rptr);
2946 radeon_ring_write(ring, 0);
2947 }
2948
2949 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2950 radeon_ring_write(ring,
2951#ifdef __BIG_ENDIAN
2952 (2 << 0) |
2953#endif
2954 (ib->gpu_addr & 0xFFFFFFFC));
2955 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2956 radeon_ring_write(ring, ib->length_dw);
2957}
2958
2959
2960static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2961{
2962 const __be32 *fw_data;
2963 int i;
2964
2965 if (!rdev->me_fw || !rdev->pfp_fw)
2966 return -EINVAL;
2967
2968 r700_cp_stop(rdev);
2969 WREG32(CP_RB_CNTL,
2970#ifdef __BIG_ENDIAN
2971 BUF_SWAP_32BIT |
2972#endif
2973 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2974
2975 fw_data = (const __be32 *)rdev->pfp_fw->data;
2976 WREG32(CP_PFP_UCODE_ADDR, 0);
2977 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2978 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2979 WREG32(CP_PFP_UCODE_ADDR, 0);
2980
2981 fw_data = (const __be32 *)rdev->me_fw->data;
2982 WREG32(CP_ME_RAM_WADDR, 0);
2983 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2984 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2985
2986 WREG32(CP_PFP_UCODE_ADDR, 0);
2987 WREG32(CP_ME_RAM_WADDR, 0);
2988 WREG32(CP_ME_RAM_RADDR, 0);
2989 return 0;
2990}
2991
2992static int evergreen_cp_start(struct radeon_device *rdev)
2993{
2994 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2995 int r, i;
2996 uint32_t cp_me;
2997
2998 r = radeon_ring_lock(rdev, ring, 7);
2999 if (r) {
3000 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3001 return r;
3002 }
3003 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3004 radeon_ring_write(ring, 0x1);
3005 radeon_ring_write(ring, 0x0);
3006 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3007 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3008 radeon_ring_write(ring, 0);
3009 radeon_ring_write(ring, 0);
3010 radeon_ring_unlock_commit(rdev, ring, false);
3011
3012 cp_me = 0xff;
3013 WREG32(CP_ME_CNTL, cp_me);
3014
3015 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3016 if (r) {
3017 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3018 return r;
3019 }
3020
3021 /* setup clear context state */
3022 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3023 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3024
3025 for (i = 0; i < evergreen_default_size; i++)
3026 radeon_ring_write(ring, evergreen_default_state[i]);
3027
3028 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3029 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3030
3031 /* set clear context state */
3032 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3033 radeon_ring_write(ring, 0);
3034
3035 /* SQ_VTX_BASE_VTX_LOC */
3036 radeon_ring_write(ring, 0xc0026f00);
3037 radeon_ring_write(ring, 0x00000000);
3038 radeon_ring_write(ring, 0x00000000);
3039 radeon_ring_write(ring, 0x00000000);
3040
3041 /* Clear consts */
3042 radeon_ring_write(ring, 0xc0036f00);
3043 radeon_ring_write(ring, 0x00000bc4);
3044 radeon_ring_write(ring, 0xffffffff);
3045 radeon_ring_write(ring, 0xffffffff);
3046 radeon_ring_write(ring, 0xffffffff);
3047
3048 radeon_ring_write(ring, 0xc0026900);
3049 radeon_ring_write(ring, 0x00000316);
3050 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3051 radeon_ring_write(ring, 0x00000010); /* */
3052
3053 radeon_ring_unlock_commit(rdev, ring, false);
3054
3055 return 0;
3056}
3057
3058static int evergreen_cp_resume(struct radeon_device *rdev)
3059{
3060 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3061 u32 tmp;
3062 u32 rb_bufsz;
3063 int r;
3064
3065 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3066 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3067 SOFT_RESET_PA |
3068 SOFT_RESET_SH |
3069 SOFT_RESET_VGT |
3070 SOFT_RESET_SPI |
3071 SOFT_RESET_SX));
3072 RREG32(GRBM_SOFT_RESET);
3073 mdelay(15);
3074 WREG32(GRBM_SOFT_RESET, 0);
3075 RREG32(GRBM_SOFT_RESET);
3076
3077 /* Set ring buffer size */
3078 rb_bufsz = order_base_2(ring->ring_size / 8);
3079 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3080#ifdef __BIG_ENDIAN
3081 tmp |= BUF_SWAP_32BIT;
3082#endif
3083 WREG32(CP_RB_CNTL, tmp);
3084 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3085 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3086
3087 /* Set the write pointer delay */
3088 WREG32(CP_RB_WPTR_DELAY, 0);
3089
3090 /* Initialize the ring buffer's read and write pointers */
3091 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3092 WREG32(CP_RB_RPTR_WR, 0);
3093 ring->wptr = 0;
3094 WREG32(CP_RB_WPTR, ring->wptr);
3095
3096 /* set the wb address whether it's enabled or not */
3097 WREG32(CP_RB_RPTR_ADDR,
3098 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3099 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3100 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3101
3102 if (rdev->wb.enabled)
3103 WREG32(SCRATCH_UMSK, 0xff);
3104 else {
3105 tmp |= RB_NO_UPDATE;
3106 WREG32(SCRATCH_UMSK, 0);
3107 }
3108
3109 mdelay(1);
3110 WREG32(CP_RB_CNTL, tmp);
3111
3112 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3113 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3114
3115 evergreen_cp_start(rdev);
3116 ring->ready = true;
3117 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3118 if (r) {
3119 ring->ready = false;
3120 return r;
3121 }
3122 return 0;
3123}
3124
3125/*
3126 * Core functions
3127 */
3128static void evergreen_gpu_init(struct radeon_device *rdev)
3129{
3130 u32 gb_addr_config;
3131 u32 mc_arb_ramcfg;
3132 u32 sx_debug_1;
3133 u32 smx_dc_ctl0;
3134 u32 sq_config;
3135 u32 sq_lds_resource_mgmt;
3136 u32 sq_gpr_resource_mgmt_1;
3137 u32 sq_gpr_resource_mgmt_2;
3138 u32 sq_gpr_resource_mgmt_3;
3139 u32 sq_thread_resource_mgmt;
3140 u32 sq_thread_resource_mgmt_2;
3141 u32 sq_stack_resource_mgmt_1;
3142 u32 sq_stack_resource_mgmt_2;
3143 u32 sq_stack_resource_mgmt_3;
3144 u32 vgt_cache_invalidation;
3145 u32 hdp_host_path_cntl, tmp;
3146 u32 disabled_rb_mask;
3147 int i, j, ps_thread_count;
3148
3149 switch (rdev->family) {
3150 case CHIP_CYPRESS:
3151 case CHIP_HEMLOCK:
3152 rdev->config.evergreen.num_ses = 2;
3153 rdev->config.evergreen.max_pipes = 4;
3154 rdev->config.evergreen.max_tile_pipes = 8;
3155 rdev->config.evergreen.max_simds = 10;
3156 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3157 rdev->config.evergreen.max_gprs = 256;
3158 rdev->config.evergreen.max_threads = 248;
3159 rdev->config.evergreen.max_gs_threads = 32;
3160 rdev->config.evergreen.max_stack_entries = 512;
3161 rdev->config.evergreen.sx_num_of_sets = 4;
3162 rdev->config.evergreen.sx_max_export_size = 256;
3163 rdev->config.evergreen.sx_max_export_pos_size = 64;
3164 rdev->config.evergreen.sx_max_export_smx_size = 192;
3165 rdev->config.evergreen.max_hw_contexts = 8;
3166 rdev->config.evergreen.sq_num_cf_insts = 2;
3167
3168 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3169 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3170 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3171 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3172 break;
3173 case CHIP_JUNIPER:
3174 rdev->config.evergreen.num_ses = 1;
3175 rdev->config.evergreen.max_pipes = 4;
3176 rdev->config.evergreen.max_tile_pipes = 4;
3177 rdev->config.evergreen.max_simds = 10;
3178 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3179 rdev->config.evergreen.max_gprs = 256;
3180 rdev->config.evergreen.max_threads = 248;
3181 rdev->config.evergreen.max_gs_threads = 32;
3182 rdev->config.evergreen.max_stack_entries = 512;
3183 rdev->config.evergreen.sx_num_of_sets = 4;
3184 rdev->config.evergreen.sx_max_export_size = 256;
3185 rdev->config.evergreen.sx_max_export_pos_size = 64;
3186 rdev->config.evergreen.sx_max_export_smx_size = 192;
3187 rdev->config.evergreen.max_hw_contexts = 8;
3188 rdev->config.evergreen.sq_num_cf_insts = 2;
3189
3190 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3191 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3192 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3193 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3194 break;
3195 case CHIP_REDWOOD:
3196 rdev->config.evergreen.num_ses = 1;
3197 rdev->config.evergreen.max_pipes = 4;
3198 rdev->config.evergreen.max_tile_pipes = 4;
3199 rdev->config.evergreen.max_simds = 5;
3200 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3201 rdev->config.evergreen.max_gprs = 256;
3202 rdev->config.evergreen.max_threads = 248;
3203 rdev->config.evergreen.max_gs_threads = 32;
3204 rdev->config.evergreen.max_stack_entries = 256;
3205 rdev->config.evergreen.sx_num_of_sets = 4;
3206 rdev->config.evergreen.sx_max_export_size = 256;
3207 rdev->config.evergreen.sx_max_export_pos_size = 64;
3208 rdev->config.evergreen.sx_max_export_smx_size = 192;
3209 rdev->config.evergreen.max_hw_contexts = 8;
3210 rdev->config.evergreen.sq_num_cf_insts = 2;
3211
3212 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3213 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3214 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3215 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3216 break;
3217 case CHIP_CEDAR:
3218 default:
3219 rdev->config.evergreen.num_ses = 1;
3220 rdev->config.evergreen.max_pipes = 2;
3221 rdev->config.evergreen.max_tile_pipes = 2;
3222 rdev->config.evergreen.max_simds = 2;
3223 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3224 rdev->config.evergreen.max_gprs = 256;
3225 rdev->config.evergreen.max_threads = 192;
3226 rdev->config.evergreen.max_gs_threads = 16;
3227 rdev->config.evergreen.max_stack_entries = 256;
3228 rdev->config.evergreen.sx_num_of_sets = 4;
3229 rdev->config.evergreen.sx_max_export_size = 128;
3230 rdev->config.evergreen.sx_max_export_pos_size = 32;
3231 rdev->config.evergreen.sx_max_export_smx_size = 96;
3232 rdev->config.evergreen.max_hw_contexts = 4;
3233 rdev->config.evergreen.sq_num_cf_insts = 1;
3234
3235 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3236 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3237 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3238 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3239 break;
3240 case CHIP_PALM:
3241 rdev->config.evergreen.num_ses = 1;
3242 rdev->config.evergreen.max_pipes = 2;
3243 rdev->config.evergreen.max_tile_pipes = 2;
3244 rdev->config.evergreen.max_simds = 2;
3245 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3246 rdev->config.evergreen.max_gprs = 256;
3247 rdev->config.evergreen.max_threads = 192;
3248 rdev->config.evergreen.max_gs_threads = 16;
3249 rdev->config.evergreen.max_stack_entries = 256;
3250 rdev->config.evergreen.sx_num_of_sets = 4;
3251 rdev->config.evergreen.sx_max_export_size = 128;
3252 rdev->config.evergreen.sx_max_export_pos_size = 32;
3253 rdev->config.evergreen.sx_max_export_smx_size = 96;
3254 rdev->config.evergreen.max_hw_contexts = 4;
3255 rdev->config.evergreen.sq_num_cf_insts = 1;
3256
3257 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3258 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3259 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3260 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3261 break;
3262 case CHIP_SUMO:
3263 rdev->config.evergreen.num_ses = 1;
3264 rdev->config.evergreen.max_pipes = 4;
3265 rdev->config.evergreen.max_tile_pipes = 4;
3266 if (rdev->pdev->device == 0x9648)
3267 rdev->config.evergreen.max_simds = 3;
3268 else if ((rdev->pdev->device == 0x9647) ||
3269 (rdev->pdev->device == 0x964a))
3270 rdev->config.evergreen.max_simds = 4;
3271 else
3272 rdev->config.evergreen.max_simds = 5;
3273 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3274 rdev->config.evergreen.max_gprs = 256;
3275 rdev->config.evergreen.max_threads = 248;
3276 rdev->config.evergreen.max_gs_threads = 32;
3277 rdev->config.evergreen.max_stack_entries = 256;
3278 rdev->config.evergreen.sx_num_of_sets = 4;
3279 rdev->config.evergreen.sx_max_export_size = 256;
3280 rdev->config.evergreen.sx_max_export_pos_size = 64;
3281 rdev->config.evergreen.sx_max_export_smx_size = 192;
3282 rdev->config.evergreen.max_hw_contexts = 8;
3283 rdev->config.evergreen.sq_num_cf_insts = 2;
3284
3285 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3286 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3287 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3288 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3289 break;
3290 case CHIP_SUMO2:
3291 rdev->config.evergreen.num_ses = 1;
3292 rdev->config.evergreen.max_pipes = 4;
3293 rdev->config.evergreen.max_tile_pipes = 4;
3294 rdev->config.evergreen.max_simds = 2;
3295 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3296 rdev->config.evergreen.max_gprs = 256;
3297 rdev->config.evergreen.max_threads = 248;
3298 rdev->config.evergreen.max_gs_threads = 32;
3299 rdev->config.evergreen.max_stack_entries = 512;
3300 rdev->config.evergreen.sx_num_of_sets = 4;
3301 rdev->config.evergreen.sx_max_export_size = 256;
3302 rdev->config.evergreen.sx_max_export_pos_size = 64;
3303 rdev->config.evergreen.sx_max_export_smx_size = 192;
3304 rdev->config.evergreen.max_hw_contexts = 4;
3305 rdev->config.evergreen.sq_num_cf_insts = 2;
3306
3307 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3308 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3309 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3310 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3311 break;
3312 case CHIP_BARTS:
3313 rdev->config.evergreen.num_ses = 2;
3314 rdev->config.evergreen.max_pipes = 4;
3315 rdev->config.evergreen.max_tile_pipes = 8;
3316 rdev->config.evergreen.max_simds = 7;
3317 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3318 rdev->config.evergreen.max_gprs = 256;
3319 rdev->config.evergreen.max_threads = 248;
3320 rdev->config.evergreen.max_gs_threads = 32;
3321 rdev->config.evergreen.max_stack_entries = 512;
3322 rdev->config.evergreen.sx_num_of_sets = 4;
3323 rdev->config.evergreen.sx_max_export_size = 256;
3324 rdev->config.evergreen.sx_max_export_pos_size = 64;
3325 rdev->config.evergreen.sx_max_export_smx_size = 192;
3326 rdev->config.evergreen.max_hw_contexts = 8;
3327 rdev->config.evergreen.sq_num_cf_insts = 2;
3328
3329 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3330 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3331 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3332 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3333 break;
3334 case CHIP_TURKS:
3335 rdev->config.evergreen.num_ses = 1;
3336 rdev->config.evergreen.max_pipes = 4;
3337 rdev->config.evergreen.max_tile_pipes = 4;
3338 rdev->config.evergreen.max_simds = 6;
3339 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3340 rdev->config.evergreen.max_gprs = 256;
3341 rdev->config.evergreen.max_threads = 248;
3342 rdev->config.evergreen.max_gs_threads = 32;
3343 rdev->config.evergreen.max_stack_entries = 256;
3344 rdev->config.evergreen.sx_num_of_sets = 4;
3345 rdev->config.evergreen.sx_max_export_size = 256;
3346 rdev->config.evergreen.sx_max_export_pos_size = 64;
3347 rdev->config.evergreen.sx_max_export_smx_size = 192;
3348 rdev->config.evergreen.max_hw_contexts = 8;
3349 rdev->config.evergreen.sq_num_cf_insts = 2;
3350
3351 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3352 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3353 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3354 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3355 break;
3356 case CHIP_CAICOS:
3357 rdev->config.evergreen.num_ses = 1;
3358 rdev->config.evergreen.max_pipes = 2;
3359 rdev->config.evergreen.max_tile_pipes = 2;
3360 rdev->config.evergreen.max_simds = 2;
3361 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3362 rdev->config.evergreen.max_gprs = 256;
3363 rdev->config.evergreen.max_threads = 192;
3364 rdev->config.evergreen.max_gs_threads = 16;
3365 rdev->config.evergreen.max_stack_entries = 256;
3366 rdev->config.evergreen.sx_num_of_sets = 4;
3367 rdev->config.evergreen.sx_max_export_size = 128;
3368 rdev->config.evergreen.sx_max_export_pos_size = 32;
3369 rdev->config.evergreen.sx_max_export_smx_size = 96;
3370 rdev->config.evergreen.max_hw_contexts = 4;
3371 rdev->config.evergreen.sq_num_cf_insts = 1;
3372
3373 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3374 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3375 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3376 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3377 break;
3378 }
3379
3380 /* Initialize HDP */
3381 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3382 WREG32((0x2c14 + j), 0x00000000);
3383 WREG32((0x2c18 + j), 0x00000000);
3384 WREG32((0x2c1c + j), 0x00000000);
3385 WREG32((0x2c20 + j), 0x00000000);
3386 WREG32((0x2c24 + j), 0x00000000);
3387 }
3388
3389 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3390 WREG32(SRBM_INT_CNTL, 0x1);
3391 WREG32(SRBM_INT_ACK, 0x1);
3392
3393 evergreen_fix_pci_max_read_req_size(rdev);
3394
3395 RREG32(MC_SHARED_CHMAP);
3396 if ((rdev->family == CHIP_PALM) ||
3397 (rdev->family == CHIP_SUMO) ||
3398 (rdev->family == CHIP_SUMO2))
3399 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3400 else
3401 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3402
3403 /* setup tiling info dword. gb_addr_config is not adequate since it does
3404 * not have bank info, so create a custom tiling dword.
3405 * bits 3:0 num_pipes
3406 * bits 7:4 num_banks
3407 * bits 11:8 group_size
3408 * bits 15:12 row_size
3409 */
3410 rdev->config.evergreen.tile_config = 0;
3411 switch (rdev->config.evergreen.max_tile_pipes) {
3412 case 1:
3413 default:
3414 rdev->config.evergreen.tile_config |= (0 << 0);
3415 break;
3416 case 2:
3417 rdev->config.evergreen.tile_config |= (1 << 0);
3418 break;
3419 case 4:
3420 rdev->config.evergreen.tile_config |= (2 << 0);
3421 break;
3422 case 8:
3423 rdev->config.evergreen.tile_config |= (3 << 0);
3424 break;
3425 }
3426 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3427 if (rdev->flags & RADEON_IS_IGP)
3428 rdev->config.evergreen.tile_config |= 1 << 4;
3429 else {
3430 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3431 case 0: /* four banks */
3432 rdev->config.evergreen.tile_config |= 0 << 4;
3433 break;
3434 case 1: /* eight banks */
3435 rdev->config.evergreen.tile_config |= 1 << 4;
3436 break;
3437 case 2: /* sixteen banks */
3438 default:
3439 rdev->config.evergreen.tile_config |= 2 << 4;
3440 break;
3441 }
3442 }
3443 rdev->config.evergreen.tile_config |= 0 << 8;
3444 rdev->config.evergreen.tile_config |=
3445 ((gb_addr_config & 0x30000000) >> 28) << 12;
3446
3447 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3448 u32 efuse_straps_4;
3449 u32 efuse_straps_3;
3450
3451 efuse_straps_4 = RREG32_RCU(0x204);
3452 efuse_straps_3 = RREG32_RCU(0x203);
3453 tmp = (((efuse_straps_4 & 0xf) << 4) |
3454 ((efuse_straps_3 & 0xf0000000) >> 28));
3455 } else {
3456 tmp = 0;
3457 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3458 u32 rb_disable_bitmap;
3459
3460 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3461 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3462 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3463 tmp <<= 4;
3464 tmp |= rb_disable_bitmap;
3465 }
3466 }
3467 /* enabled rb are just the one not disabled :) */
3468 disabled_rb_mask = tmp;
3469 tmp = 0;
3470 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3471 tmp |= (1 << i);
3472 /* if all the backends are disabled, fix it up here */
3473 if ((disabled_rb_mask & tmp) == tmp) {
3474 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3475 disabled_rb_mask &= ~(1 << i);
3476 }
3477
3478 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3479 u32 simd_disable_bitmap;
3480
3481 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3482 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3483 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3484 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3485 tmp <<= 16;
3486 tmp |= simd_disable_bitmap;
3487 }
3488 rdev->config.evergreen.active_simds = hweight32(~tmp);
3489
3490 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3491 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3492
3493 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3494 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3495 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3496 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3497 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3498 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3499 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3500
3501 if ((rdev->config.evergreen.max_backends == 1) &&
3502 (rdev->flags & RADEON_IS_IGP)) {
3503 if ((disabled_rb_mask & 3) == 1) {
3504 /* RB0 disabled, RB1 enabled */
3505 tmp = 0x11111111;
3506 } else {
3507 /* RB1 disabled, RB0 enabled */
3508 tmp = 0x00000000;
3509 }
3510 } else {
3511 tmp = gb_addr_config & NUM_PIPES_MASK;
3512 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3513 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3514 }
3515 rdev->config.evergreen.backend_map = tmp;
3516 WREG32(GB_BACKEND_MAP, tmp);
3517
3518 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3519 WREG32(CGTS_TCC_DISABLE, 0);
3520 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3521 WREG32(CGTS_USER_TCC_DISABLE, 0);
3522
3523 /* set HW defaults for 3D engine */
3524 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3525 ROQ_IB2_START(0x2b)));
3526
3527 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3528
3529 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3530 SYNC_GRADIENT |
3531 SYNC_WALKER |
3532 SYNC_ALIGNER));
3533
3534 sx_debug_1 = RREG32(SX_DEBUG_1);
3535 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3536 WREG32(SX_DEBUG_1, sx_debug_1);
3537
3538
3539 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3540 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3541 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3542 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3543
3544 if (rdev->family <= CHIP_SUMO2)
3545 WREG32(SMX_SAR_CTL0, 0x00010000);
3546
3547 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3548 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3549 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3550
3551 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3552 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3553 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3554
3555 WREG32(VGT_NUM_INSTANCES, 1);
3556 WREG32(SPI_CONFIG_CNTL, 0);
3557 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3558 WREG32(CP_PERFMON_CNTL, 0);
3559
3560 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3561 FETCH_FIFO_HIWATER(0x4) |
3562 DONE_FIFO_HIWATER(0xe0) |
3563 ALU_UPDATE_FIFO_HIWATER(0x8)));
3564
3565 sq_config = RREG32(SQ_CONFIG);
3566 sq_config &= ~(PS_PRIO(3) |
3567 VS_PRIO(3) |
3568 GS_PRIO(3) |
3569 ES_PRIO(3));
3570 sq_config |= (VC_ENABLE |
3571 EXPORT_SRC_C |
3572 PS_PRIO(0) |
3573 VS_PRIO(1) |
3574 GS_PRIO(2) |
3575 ES_PRIO(3));
3576
3577 switch (rdev->family) {
3578 case CHIP_CEDAR:
3579 case CHIP_PALM:
3580 case CHIP_SUMO:
3581 case CHIP_SUMO2:
3582 case CHIP_CAICOS:
3583 /* no vertex cache */
3584 sq_config &= ~VC_ENABLE;
3585 break;
3586 default:
3587 break;
3588 }
3589
3590 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3591
3592 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3593 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3594 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3595 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3596 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3597 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3598 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3599
3600 switch (rdev->family) {
3601 case CHIP_CEDAR:
3602 case CHIP_PALM:
3603 case CHIP_SUMO:
3604 case CHIP_SUMO2:
3605 ps_thread_count = 96;
3606 break;
3607 default:
3608 ps_thread_count = 128;
3609 break;
3610 }
3611
3612 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3613 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3614 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3615 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3616 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3617 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618
3619 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3620 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3621 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3622 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3623 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625
3626 WREG32(SQ_CONFIG, sq_config);
3627 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3628 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3629 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3630 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3631 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3632 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3633 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3634 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3635 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3636 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3637
3638 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3639 FORCE_EOV_MAX_REZ_CNT(255)));
3640
3641 switch (rdev->family) {
3642 case CHIP_CEDAR:
3643 case CHIP_PALM:
3644 case CHIP_SUMO:
3645 case CHIP_SUMO2:
3646 case CHIP_CAICOS:
3647 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3648 break;
3649 default:
3650 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3651 break;
3652 }
3653 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3654 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3655
3656 WREG32(VGT_GS_VERTEX_REUSE, 16);
3657 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3658 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3659
3660 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3661 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3662
3663 WREG32(CB_PERF_CTR0_SEL_0, 0);
3664 WREG32(CB_PERF_CTR0_SEL_1, 0);
3665 WREG32(CB_PERF_CTR1_SEL_0, 0);
3666 WREG32(CB_PERF_CTR1_SEL_1, 0);
3667 WREG32(CB_PERF_CTR2_SEL_0, 0);
3668 WREG32(CB_PERF_CTR2_SEL_1, 0);
3669 WREG32(CB_PERF_CTR3_SEL_0, 0);
3670 WREG32(CB_PERF_CTR3_SEL_1, 0);
3671
3672 /* clear render buffer base addresses */
3673 WREG32(CB_COLOR0_BASE, 0);
3674 WREG32(CB_COLOR1_BASE, 0);
3675 WREG32(CB_COLOR2_BASE, 0);
3676 WREG32(CB_COLOR3_BASE, 0);
3677 WREG32(CB_COLOR4_BASE, 0);
3678 WREG32(CB_COLOR5_BASE, 0);
3679 WREG32(CB_COLOR6_BASE, 0);
3680 WREG32(CB_COLOR7_BASE, 0);
3681 WREG32(CB_COLOR8_BASE, 0);
3682 WREG32(CB_COLOR9_BASE, 0);
3683 WREG32(CB_COLOR10_BASE, 0);
3684 WREG32(CB_COLOR11_BASE, 0);
3685
3686 /* set the shader const cache sizes to 0 */
3687 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3688 WREG32(i, 0);
3689 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3690 WREG32(i, 0);
3691
3692 tmp = RREG32(HDP_MISC_CNTL);
3693 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3694 WREG32(HDP_MISC_CNTL, tmp);
3695
3696 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3697 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3698
3699 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3700
3701 udelay(50);
3702
3703}
3704
3705int evergreen_mc_init(struct radeon_device *rdev)
3706{
3707 u32 tmp;
3708 int chansize, numchan;
3709
3710 /* Get VRAM informations */
3711 rdev->mc.vram_is_ddr = true;
3712 if ((rdev->family == CHIP_PALM) ||
3713 (rdev->family == CHIP_SUMO) ||
3714 (rdev->family == CHIP_SUMO2))
3715 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3716 else
3717 tmp = RREG32(MC_ARB_RAMCFG);
3718 if (tmp & CHANSIZE_OVERRIDE) {
3719 chansize = 16;
3720 } else if (tmp & CHANSIZE_MASK) {
3721 chansize = 64;
3722 } else {
3723 chansize = 32;
3724 }
3725 tmp = RREG32(MC_SHARED_CHMAP);
3726 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3727 case 0:
3728 default:
3729 numchan = 1;
3730 break;
3731 case 1:
3732 numchan = 2;
3733 break;
3734 case 2:
3735 numchan = 4;
3736 break;
3737 case 3:
3738 numchan = 8;
3739 break;
3740 }
3741 rdev->mc.vram_width = numchan * chansize;
3742 /* Could aper size report 0 ? */
3743 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3744 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3745 /* Setup GPU memory space */
3746 if ((rdev->family == CHIP_PALM) ||
3747 (rdev->family == CHIP_SUMO) ||
3748 (rdev->family == CHIP_SUMO2)) {
3749 /* size in bytes on fusion */
3750 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3751 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3752 } else {
3753 /* size in MB on evergreen/cayman/tn */
3754 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3755 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3756 }
3757 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3758 r700_vram_gtt_location(rdev, &rdev->mc);
3759 radeon_update_bandwidth_info(rdev);
3760
3761 return 0;
3762}
3763
3764void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3765{
3766 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3767 RREG32(GRBM_STATUS));
3768 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3769 RREG32(GRBM_STATUS_SE0));
3770 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3771 RREG32(GRBM_STATUS_SE1));
3772 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3773 RREG32(SRBM_STATUS));
3774 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3775 RREG32(SRBM_STATUS2));
3776 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3777 RREG32(CP_STALLED_STAT1));
3778 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3779 RREG32(CP_STALLED_STAT2));
3780 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3781 RREG32(CP_BUSY_STAT));
3782 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3783 RREG32(CP_STAT));
3784 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3785 RREG32(DMA_STATUS_REG));
3786 if (rdev->family >= CHIP_CAYMAN) {
3787 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3788 RREG32(DMA_STATUS_REG + 0x800));
3789 }
3790}
3791
3792bool evergreen_is_display_hung(struct radeon_device *rdev)
3793{
3794 u32 crtc_hung = 0;
3795 u32 crtc_status[6];
3796 u32 i, j, tmp;
3797
3798 for (i = 0; i < rdev->num_crtc; i++) {
3799 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3800 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3801 crtc_hung |= (1 << i);
3802 }
3803 }
3804
3805 for (j = 0; j < 10; j++) {
3806 for (i = 0; i < rdev->num_crtc; i++) {
3807 if (crtc_hung & (1 << i)) {
3808 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3809 if (tmp != crtc_status[i])
3810 crtc_hung &= ~(1 << i);
3811 }
3812 }
3813 if (crtc_hung == 0)
3814 return false;
3815 udelay(100);
3816 }
3817
3818 return true;
3819}
3820
3821u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3822{
3823 u32 reset_mask = 0;
3824 u32 tmp;
3825
3826 /* GRBM_STATUS */
3827 tmp = RREG32(GRBM_STATUS);
3828 if (tmp & (PA_BUSY | SC_BUSY |
3829 SH_BUSY | SX_BUSY |
3830 TA_BUSY | VGT_BUSY |
3831 DB_BUSY | CB_BUSY |
3832 SPI_BUSY | VGT_BUSY_NO_DMA))
3833 reset_mask |= RADEON_RESET_GFX;
3834
3835 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3836 CP_BUSY | CP_COHERENCY_BUSY))
3837 reset_mask |= RADEON_RESET_CP;
3838
3839 if (tmp & GRBM_EE_BUSY)
3840 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3841
3842 /* DMA_STATUS_REG */
3843 tmp = RREG32(DMA_STATUS_REG);
3844 if (!(tmp & DMA_IDLE))
3845 reset_mask |= RADEON_RESET_DMA;
3846
3847 /* SRBM_STATUS2 */
3848 tmp = RREG32(SRBM_STATUS2);
3849 if (tmp & DMA_BUSY)
3850 reset_mask |= RADEON_RESET_DMA;
3851
3852 /* SRBM_STATUS */
3853 tmp = RREG32(SRBM_STATUS);
3854 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3855 reset_mask |= RADEON_RESET_RLC;
3856
3857 if (tmp & IH_BUSY)
3858 reset_mask |= RADEON_RESET_IH;
3859
3860 if (tmp & SEM_BUSY)
3861 reset_mask |= RADEON_RESET_SEM;
3862
3863 if (tmp & GRBM_RQ_PENDING)
3864 reset_mask |= RADEON_RESET_GRBM;
3865
3866 if (tmp & VMC_BUSY)
3867 reset_mask |= RADEON_RESET_VMC;
3868
3869 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3870 MCC_BUSY | MCD_BUSY))
3871 reset_mask |= RADEON_RESET_MC;
3872
3873 if (evergreen_is_display_hung(rdev))
3874 reset_mask |= RADEON_RESET_DISPLAY;
3875
3876 /* VM_L2_STATUS */
3877 tmp = RREG32(VM_L2_STATUS);
3878 if (tmp & L2_BUSY)
3879 reset_mask |= RADEON_RESET_VMC;
3880
3881 /* Skip MC reset as it's mostly likely not hung, just busy */
3882 if (reset_mask & RADEON_RESET_MC) {
3883 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3884 reset_mask &= ~RADEON_RESET_MC;
3885 }
3886
3887 return reset_mask;
3888}
3889
3890static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3891{
3892 struct evergreen_mc_save save;
3893 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3894 u32 tmp;
3895
3896 if (reset_mask == 0)
3897 return;
3898
3899 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3900
3901 evergreen_print_gpu_status_regs(rdev);
3902
3903 /* Disable CP parsing/prefetching */
3904 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3905
3906 if (reset_mask & RADEON_RESET_DMA) {
3907 /* Disable DMA */
3908 tmp = RREG32(DMA_RB_CNTL);
3909 tmp &= ~DMA_RB_ENABLE;
3910 WREG32(DMA_RB_CNTL, tmp);
3911 }
3912
3913 udelay(50);
3914
3915 evergreen_mc_stop(rdev, &save);
3916 if (evergreen_mc_wait_for_idle(rdev)) {
3917 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3918 }
3919
3920 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3921 grbm_soft_reset |= SOFT_RESET_DB |
3922 SOFT_RESET_CB |
3923 SOFT_RESET_PA |
3924 SOFT_RESET_SC |
3925 SOFT_RESET_SPI |
3926 SOFT_RESET_SX |
3927 SOFT_RESET_SH |
3928 SOFT_RESET_TC |
3929 SOFT_RESET_TA |
3930 SOFT_RESET_VC |
3931 SOFT_RESET_VGT;
3932 }
3933
3934 if (reset_mask & RADEON_RESET_CP) {
3935 grbm_soft_reset |= SOFT_RESET_CP |
3936 SOFT_RESET_VGT;
3937
3938 srbm_soft_reset |= SOFT_RESET_GRBM;
3939 }
3940
3941 if (reset_mask & RADEON_RESET_DMA)
3942 srbm_soft_reset |= SOFT_RESET_DMA;
3943
3944 if (reset_mask & RADEON_RESET_DISPLAY)
3945 srbm_soft_reset |= SOFT_RESET_DC;
3946
3947 if (reset_mask & RADEON_RESET_RLC)
3948 srbm_soft_reset |= SOFT_RESET_RLC;
3949
3950 if (reset_mask & RADEON_RESET_SEM)
3951 srbm_soft_reset |= SOFT_RESET_SEM;
3952
3953 if (reset_mask & RADEON_RESET_IH)
3954 srbm_soft_reset |= SOFT_RESET_IH;
3955
3956 if (reset_mask & RADEON_RESET_GRBM)
3957 srbm_soft_reset |= SOFT_RESET_GRBM;
3958
3959 if (reset_mask & RADEON_RESET_VMC)
3960 srbm_soft_reset |= SOFT_RESET_VMC;
3961
3962 if (!(rdev->flags & RADEON_IS_IGP)) {
3963 if (reset_mask & RADEON_RESET_MC)
3964 srbm_soft_reset |= SOFT_RESET_MC;
3965 }
3966
3967 if (grbm_soft_reset) {
3968 tmp = RREG32(GRBM_SOFT_RESET);
3969 tmp |= grbm_soft_reset;
3970 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3971 WREG32(GRBM_SOFT_RESET, tmp);
3972 tmp = RREG32(GRBM_SOFT_RESET);
3973
3974 udelay(50);
3975
3976 tmp &= ~grbm_soft_reset;
3977 WREG32(GRBM_SOFT_RESET, tmp);
3978 tmp = RREG32(GRBM_SOFT_RESET);
3979 }
3980
3981 if (srbm_soft_reset) {
3982 tmp = RREG32(SRBM_SOFT_RESET);
3983 tmp |= srbm_soft_reset;
3984 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3985 WREG32(SRBM_SOFT_RESET, tmp);
3986 tmp = RREG32(SRBM_SOFT_RESET);
3987
3988 udelay(50);
3989
3990 tmp &= ~srbm_soft_reset;
3991 WREG32(SRBM_SOFT_RESET, tmp);
3992 tmp = RREG32(SRBM_SOFT_RESET);
3993 }
3994
3995 /* Wait a little for things to settle down */
3996 udelay(50);
3997
3998 evergreen_mc_resume(rdev, &save);
3999 udelay(50);
4000
4001 evergreen_print_gpu_status_regs(rdev);
4002}
4003
4004void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4005{
4006 struct evergreen_mc_save save;
4007 u32 tmp, i;
4008
4009 dev_info(rdev->dev, "GPU pci config reset\n");
4010
4011 /* disable dpm? */
4012
4013 /* Disable CP parsing/prefetching */
4014 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4015 udelay(50);
4016 /* Disable DMA */
4017 tmp = RREG32(DMA_RB_CNTL);
4018 tmp &= ~DMA_RB_ENABLE;
4019 WREG32(DMA_RB_CNTL, tmp);
4020 /* XXX other engines? */
4021
4022 /* halt the rlc */
4023 r600_rlc_stop(rdev);
4024
4025 udelay(50);
4026
4027 /* set mclk/sclk to bypass */
4028 rv770_set_clk_bypass_mode(rdev);
4029 /* disable BM */
4030 pci_clear_master(rdev->pdev);
4031 /* disable mem access */
4032 evergreen_mc_stop(rdev, &save);
4033 if (evergreen_mc_wait_for_idle(rdev)) {
4034 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4035 }
4036 /* reset */
4037 radeon_pci_config_reset(rdev);
4038 /* wait for asic to come out of reset */
4039 for (i = 0; i < rdev->usec_timeout; i++) {
4040 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4041 break;
4042 udelay(1);
4043 }
4044}
4045
4046int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4047{
4048 u32 reset_mask;
4049
4050 if (hard) {
4051 evergreen_gpu_pci_config_reset(rdev);
4052 return 0;
4053 }
4054
4055 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4056
4057 if (reset_mask)
4058 r600_set_bios_scratch_engine_hung(rdev, true);
4059
4060 /* try soft reset */
4061 evergreen_gpu_soft_reset(rdev, reset_mask);
4062
4063 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4064
4065 /* try pci config reset */
4066 if (reset_mask && radeon_hard_reset)
4067 evergreen_gpu_pci_config_reset(rdev);
4068
4069 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4070
4071 if (!reset_mask)
4072 r600_set_bios_scratch_engine_hung(rdev, false);
4073
4074 return 0;
4075}
4076
4077/**
4078 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4079 *
4080 * @rdev: radeon_device pointer
4081 * @ring: radeon_ring structure holding ring information
4082 *
4083 * Check if the GFX engine is locked up.
4084 * Returns true if the engine appears to be locked up, false if not.
4085 */
4086bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4087{
4088 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4089
4090 if (!(reset_mask & (RADEON_RESET_GFX |
4091 RADEON_RESET_COMPUTE |
4092 RADEON_RESET_CP))) {
4093 radeon_ring_lockup_update(rdev, ring);
4094 return false;
4095 }
4096 return radeon_ring_test_lockup(rdev, ring);
4097}
4098
4099/*
4100 * RLC
4101 */
4102#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4103#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4104
4105void sumo_rlc_fini(struct radeon_device *rdev)
4106{
4107 int r;
4108
4109 /* save restore block */
4110 if (rdev->rlc.save_restore_obj) {
4111 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4112 if (unlikely(r != 0))
4113 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4114 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4115 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4116
4117 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4118 rdev->rlc.save_restore_obj = NULL;
4119 }
4120
4121 /* clear state block */
4122 if (rdev->rlc.clear_state_obj) {
4123 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4124 if (unlikely(r != 0))
4125 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4126 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4127 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4128
4129 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4130 rdev->rlc.clear_state_obj = NULL;
4131 }
4132
4133 /* clear state block */
4134 if (rdev->rlc.cp_table_obj) {
4135 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4136 if (unlikely(r != 0))
4137 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4138 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4139 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4140
4141 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4142 rdev->rlc.cp_table_obj = NULL;
4143 }
4144}
4145
4146#define CP_ME_TABLE_SIZE 96
4147
4148int sumo_rlc_init(struct radeon_device *rdev)
4149{
4150 const u32 *src_ptr;
4151 volatile u32 *dst_ptr;
4152 u32 dws, data, i, j, k, reg_num;
4153 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4154 u64 reg_list_mc_addr;
4155 const struct cs_section_def *cs_data;
4156 int r;
4157
4158 src_ptr = rdev->rlc.reg_list;
4159 dws = rdev->rlc.reg_list_size;
4160 if (rdev->family >= CHIP_BONAIRE) {
4161 dws += (5 * 16) + 48 + 48 + 64;
4162 }
4163 cs_data = rdev->rlc.cs_data;
4164
4165 if (src_ptr) {
4166 /* save restore block */
4167 if (rdev->rlc.save_restore_obj == NULL) {
4168 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4169 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4170 NULL, &rdev->rlc.save_restore_obj);
4171 if (r) {
4172 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4173 return r;
4174 }
4175 }
4176
4177 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4178 if (unlikely(r != 0)) {
4179 sumo_rlc_fini(rdev);
4180 return r;
4181 }
4182 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4183 &rdev->rlc.save_restore_gpu_addr);
4184 if (r) {
4185 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4186 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4187 sumo_rlc_fini(rdev);
4188 return r;
4189 }
4190
4191 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4192 if (r) {
4193 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4194 sumo_rlc_fini(rdev);
4195 return r;
4196 }
4197 /* write the sr buffer */
4198 dst_ptr = rdev->rlc.sr_ptr;
4199 if (rdev->family >= CHIP_TAHITI) {
4200 /* SI */
4201 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4202 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4203 } else {
4204 /* ON/LN/TN */
4205 /* format:
4206 * dw0: (reg2 << 16) | reg1
4207 * dw1: reg1 save space
4208 * dw2: reg2 save space
4209 */
4210 for (i = 0; i < dws; i++) {
4211 data = src_ptr[i] >> 2;
4212 i++;
4213 if (i < dws)
4214 data |= (src_ptr[i] >> 2) << 16;
4215 j = (((i - 1) * 3) / 2);
4216 dst_ptr[j] = cpu_to_le32(data);
4217 }
4218 j = ((i * 3) / 2);
4219 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4220 }
4221 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4222 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4223 }
4224
4225 if (cs_data) {
4226 /* clear state block */
4227 if (rdev->family >= CHIP_BONAIRE) {
4228 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4229 } else if (rdev->family >= CHIP_TAHITI) {
4230 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4231 dws = rdev->rlc.clear_state_size + (256 / 4);
4232 } else {
4233 reg_list_num = 0;
4234 dws = 0;
4235 for (i = 0; cs_data[i].section != NULL; i++) {
4236 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4237 reg_list_num++;
4238 dws += cs_data[i].section[j].reg_count;
4239 }
4240 }
4241 reg_list_blk_index = (3 * reg_list_num + 2);
4242 dws += reg_list_blk_index;
4243 rdev->rlc.clear_state_size = dws;
4244 }
4245
4246 if (rdev->rlc.clear_state_obj == NULL) {
4247 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4248 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4249 NULL, &rdev->rlc.clear_state_obj);
4250 if (r) {
4251 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4252 sumo_rlc_fini(rdev);
4253 return r;
4254 }
4255 }
4256 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4257 if (unlikely(r != 0)) {
4258 sumo_rlc_fini(rdev);
4259 return r;
4260 }
4261 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4262 &rdev->rlc.clear_state_gpu_addr);
4263 if (r) {
4264 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4265 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4266 sumo_rlc_fini(rdev);
4267 return r;
4268 }
4269
4270 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4271 if (r) {
4272 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4273 sumo_rlc_fini(rdev);
4274 return r;
4275 }
4276 /* set up the cs buffer */
4277 dst_ptr = rdev->rlc.cs_ptr;
4278 if (rdev->family >= CHIP_BONAIRE) {
4279 cik_get_csb_buffer(rdev, dst_ptr);
4280 } else if (rdev->family >= CHIP_TAHITI) {
4281 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4282 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4283 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4284 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4285 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4286 } else {
4287 reg_list_hdr_blk_index = 0;
4288 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4289 data = upper_32_bits(reg_list_mc_addr);
4290 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4291 reg_list_hdr_blk_index++;
4292 for (i = 0; cs_data[i].section != NULL; i++) {
4293 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4294 reg_num = cs_data[i].section[j].reg_count;
4295 data = reg_list_mc_addr & 0xffffffff;
4296 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4297 reg_list_hdr_blk_index++;
4298
4299 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4300 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301 reg_list_hdr_blk_index++;
4302
4303 data = 0x08000000 | (reg_num * 4);
4304 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305 reg_list_hdr_blk_index++;
4306
4307 for (k = 0; k < reg_num; k++) {
4308 data = cs_data[i].section[j].extent[k];
4309 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4310 }
4311 reg_list_mc_addr += reg_num * 4;
4312 reg_list_blk_index += reg_num;
4313 }
4314 }
4315 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4316 }
4317 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4318 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4319 }
4320
4321 if (rdev->rlc.cp_table_size) {
4322 if (rdev->rlc.cp_table_obj == NULL) {
4323 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4324 PAGE_SIZE, true,
4325 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4326 NULL, &rdev->rlc.cp_table_obj);
4327 if (r) {
4328 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4329 sumo_rlc_fini(rdev);
4330 return r;
4331 }
4332 }
4333
4334 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4335 if (unlikely(r != 0)) {
4336 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4337 sumo_rlc_fini(rdev);
4338 return r;
4339 }
4340 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4341 &rdev->rlc.cp_table_gpu_addr);
4342 if (r) {
4343 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4344 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4345 sumo_rlc_fini(rdev);
4346 return r;
4347 }
4348 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4349 if (r) {
4350 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4351 sumo_rlc_fini(rdev);
4352 return r;
4353 }
4354
4355 cik_init_cp_pg_table(rdev);
4356
4357 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4358 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4359
4360 }
4361
4362 return 0;
4363}
4364
4365static void evergreen_rlc_start(struct radeon_device *rdev)
4366{
4367 u32 mask = RLC_ENABLE;
4368
4369 if (rdev->flags & RADEON_IS_IGP) {
4370 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4371 }
4372
4373 WREG32(RLC_CNTL, mask);
4374}
4375
4376int evergreen_rlc_resume(struct radeon_device *rdev)
4377{
4378 u32 i;
4379 const __be32 *fw_data;
4380
4381 if (!rdev->rlc_fw)
4382 return -EINVAL;
4383
4384 r600_rlc_stop(rdev);
4385
4386 WREG32(RLC_HB_CNTL, 0);
4387
4388 if (rdev->flags & RADEON_IS_IGP) {
4389 if (rdev->family == CHIP_ARUBA) {
4390 u32 always_on_bitmap =
4391 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4392 /* find out the number of active simds */
4393 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4394 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4395 tmp = hweight32(~tmp);
4396 if (tmp == rdev->config.cayman.max_simds_per_se) {
4397 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4398 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4399 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4400 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4401 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4402 }
4403 } else {
4404 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4405 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4406 }
4407 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4408 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4409 } else {
4410 WREG32(RLC_HB_BASE, 0);
4411 WREG32(RLC_HB_RPTR, 0);
4412 WREG32(RLC_HB_WPTR, 0);
4413 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4414 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4415 }
4416 WREG32(RLC_MC_CNTL, 0);
4417 WREG32(RLC_UCODE_CNTL, 0);
4418
4419 fw_data = (const __be32 *)rdev->rlc_fw->data;
4420 if (rdev->family >= CHIP_ARUBA) {
4421 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4422 WREG32(RLC_UCODE_ADDR, i);
4423 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4424 }
4425 } else if (rdev->family >= CHIP_CAYMAN) {
4426 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4427 WREG32(RLC_UCODE_ADDR, i);
4428 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429 }
4430 } else {
4431 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4432 WREG32(RLC_UCODE_ADDR, i);
4433 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434 }
4435 }
4436 WREG32(RLC_UCODE_ADDR, 0);
4437
4438 evergreen_rlc_start(rdev);
4439
4440 return 0;
4441}
4442
4443/* Interrupts */
4444
4445u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4446{
4447 if (crtc >= rdev->num_crtc)
4448 return 0;
4449 else
4450 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4451}
4452
4453void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4454{
4455 int i;
4456 u32 tmp;
4457
4458 if (rdev->family >= CHIP_CAYMAN) {
4459 cayman_cp_int_cntl_setup(rdev, 0,
4460 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4461 cayman_cp_int_cntl_setup(rdev, 1, 0);
4462 cayman_cp_int_cntl_setup(rdev, 2, 0);
4463 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4464 WREG32(CAYMAN_DMA1_CNTL, tmp);
4465 } else
4466 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4467 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4468 WREG32(DMA_CNTL, tmp);
4469 WREG32(GRBM_INT_CNTL, 0);
4470 WREG32(SRBM_INT_CNTL, 0);
4471 for (i = 0; i < rdev->num_crtc; i++)
4472 WREG32(INT_MASK + crtc_offsets[i], 0);
4473 for (i = 0; i < rdev->num_crtc; i++)
4474 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4475
4476 /* only one DAC on DCE5 */
4477 if (!ASIC_IS_DCE5(rdev))
4478 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4479 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4480
4481 for (i = 0; i < 6; i++)
4482 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4483}
4484
4485/* Note that the order we write back regs here is important */
4486int evergreen_irq_set(struct radeon_device *rdev)
4487{
4488 int i;
4489 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4490 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4491 u32 grbm_int_cntl = 0;
4492 u32 dma_cntl, dma_cntl1 = 0;
4493 u32 thermal_int = 0;
4494
4495 if (!rdev->irq.installed) {
4496 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4497 return -EINVAL;
4498 }
4499 /* don't enable anything if the ih is disabled */
4500 if (!rdev->ih.enabled) {
4501 r600_disable_interrupts(rdev);
4502 /* force the active interrupt state to all disabled */
4503 evergreen_disable_interrupt_state(rdev);
4504 return 0;
4505 }
4506
4507 if (rdev->family == CHIP_ARUBA)
4508 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4509 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4510 else
4511 thermal_int = RREG32(CG_THERMAL_INT) &
4512 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4513
4514 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4515
4516 if (rdev->family >= CHIP_CAYMAN) {
4517 /* enable CP interrupts on all rings */
4518 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4519 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4520 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4521 }
4522 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4523 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4524 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4525 }
4526 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4527 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4528 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4529 }
4530 } else {
4531 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4532 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4533 cp_int_cntl |= RB_INT_ENABLE;
4534 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4535 }
4536 }
4537
4538 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4539 DRM_DEBUG("r600_irq_set: sw int dma\n");
4540 dma_cntl |= TRAP_ENABLE;
4541 }
4542
4543 if (rdev->family >= CHIP_CAYMAN) {
4544 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4545 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4546 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4547 dma_cntl1 |= TRAP_ENABLE;
4548 }
4549 }
4550
4551 if (rdev->irq.dpm_thermal) {
4552 DRM_DEBUG("dpm thermal\n");
4553 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4554 }
4555
4556 if (rdev->family >= CHIP_CAYMAN) {
4557 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4558 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4559 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4560 } else
4561 WREG32(CP_INT_CNTL, cp_int_cntl);
4562
4563 WREG32(DMA_CNTL, dma_cntl);
4564
4565 if (rdev->family >= CHIP_CAYMAN)
4566 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4567
4568 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4569
4570 for (i = 0; i < rdev->num_crtc; i++) {
4571 radeon_irq_kms_set_irq_n_enabled(
4572 rdev, INT_MASK + crtc_offsets[i],
4573 VBLANK_INT_MASK,
4574 rdev->irq.crtc_vblank_int[i] ||
4575 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4576 }
4577
4578 for (i = 0; i < rdev->num_crtc; i++)
4579 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4580
4581 for (i = 0; i < 6; i++) {
4582 radeon_irq_kms_set_irq_n_enabled(
4583 rdev, DC_HPDx_INT_CONTROL(i),
4584 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4585 rdev->irq.hpd[i], "HPD", i);
4586 }
4587
4588 if (rdev->family == CHIP_ARUBA)
4589 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4590 else
4591 WREG32(CG_THERMAL_INT, thermal_int);
4592
4593 for (i = 0; i < 6; i++) {
4594 radeon_irq_kms_set_irq_n_enabled(
4595 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4596 AFMT_AZ_FORMAT_WTRIG_MASK,
4597 rdev->irq.afmt[i], "HDMI", i);
4598 }
4599
4600 /* posting read */
4601 RREG32(SRBM_STATUS);
4602
4603 return 0;
4604}
4605
4606/* Note that the order we write back regs here is important */
4607static void evergreen_irq_ack(struct radeon_device *rdev)
4608{
4609 int i, j;
4610 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4611 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4612 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4613
4614 for (i = 0; i < 6; i++) {
4615 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4616 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4617 if (i < rdev->num_crtc)
4618 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4619 }
4620
4621 /* We write back each interrupt register in pairs of two */
4622 for (i = 0; i < rdev->num_crtc; i += 2) {
4623 for (j = i; j < (i + 2); j++) {
4624 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4625 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4626 GRPH_PFLIP_INT_CLEAR);
4627 }
4628
4629 for (j = i; j < (i + 2); j++) {
4630 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4631 WREG32(VBLANK_STATUS + crtc_offsets[j],
4632 VBLANK_ACK);
4633 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4634 WREG32(VLINE_STATUS + crtc_offsets[j],
4635 VLINE_ACK);
4636 }
4637 }
4638
4639 for (i = 0; i < 6; i++) {
4640 if (disp_int[i] & DC_HPD1_INTERRUPT)
4641 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4642 }
4643
4644 for (i = 0; i < 6; i++) {
4645 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4646 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4647 }
4648
4649 for (i = 0; i < 6; i++) {
4650 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4651 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4652 AFMT_AZ_FORMAT_WTRIG_ACK);
4653 }
4654}
4655
4656static void evergreen_irq_disable(struct radeon_device *rdev)
4657{
4658 r600_disable_interrupts(rdev);
4659 /* Wait and acknowledge irq */
4660 mdelay(1);
4661 evergreen_irq_ack(rdev);
4662 evergreen_disable_interrupt_state(rdev);
4663}
4664
4665void evergreen_irq_suspend(struct radeon_device *rdev)
4666{
4667 evergreen_irq_disable(rdev);
4668 r600_rlc_stop(rdev);
4669}
4670
4671static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4672{
4673 u32 wptr, tmp;
4674
4675 if (rdev->wb.enabled)
4676 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4677 else
4678 wptr = RREG32(IH_RB_WPTR);
4679
4680 if (wptr & RB_OVERFLOW) {
4681 wptr &= ~RB_OVERFLOW;
4682 /* When a ring buffer overflow happen start parsing interrupt
4683 * from the last not overwritten vector (wptr + 16). Hopefully
4684 * this should allow us to catchup.
4685 */
4686 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4687 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4688 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4689 tmp = RREG32(IH_RB_CNTL);
4690 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4691 WREG32(IH_RB_CNTL, tmp);
4692 }
4693 return (wptr & rdev->ih.ptr_mask);
4694}
4695
4696int evergreen_irq_process(struct radeon_device *rdev)
4697{
4698 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4699 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4700 u32 crtc_idx, hpd_idx, afmt_idx;
4701 u32 mask;
4702 u32 wptr;
4703 u32 rptr;
4704 u32 src_id, src_data;
4705 u32 ring_index;
4706 bool queue_hotplug = false;
4707 bool queue_hdmi = false;
4708 bool queue_dp = false;
4709 bool queue_thermal = false;
4710 u32 status, addr;
4711 const char *event_name;
4712
4713 if (!rdev->ih.enabled || rdev->shutdown)
4714 return IRQ_NONE;
4715
4716 wptr = evergreen_get_ih_wptr(rdev);
4717
4718restart_ih:
4719 /* is somebody else already processing irqs? */
4720 if (atomic_xchg(&rdev->ih.lock, 1))
4721 return IRQ_NONE;
4722
4723 rptr = rdev->ih.rptr;
4724 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4725
4726 /* Order reading of wptr vs. reading of IH ring data */
4727 rmb();
4728
4729 /* display interrupts */
4730 evergreen_irq_ack(rdev);
4731
4732 while (rptr != wptr) {
4733 /* wptr/rptr are in bytes! */
4734 ring_index = rptr / 4;
4735 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4736 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4737
4738 switch (src_id) {
4739 case 1: /* D1 vblank/vline */
4740 case 2: /* D2 vblank/vline */
4741 case 3: /* D3 vblank/vline */
4742 case 4: /* D4 vblank/vline */
4743 case 5: /* D5 vblank/vline */
4744 case 6: /* D6 vblank/vline */
4745 crtc_idx = src_id - 1;
4746
4747 if (src_data == 0) { /* vblank */
4748 mask = LB_D1_VBLANK_INTERRUPT;
4749 event_name = "vblank";
4750
4751 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4752 drm_handle_vblank(rdev->ddev, crtc_idx);
4753 rdev->pm.vblank_sync = true;
4754 wake_up(&rdev->irq.vblank_queue);
4755 }
4756 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4757 radeon_crtc_handle_vblank(rdev,
4758 crtc_idx);
4759 }
4760
4761 } else if (src_data == 1) { /* vline */
4762 mask = LB_D1_VLINE_INTERRUPT;
4763 event_name = "vline";
4764 } else {
4765 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4766 src_id, src_data);
4767 break;
4768 }
4769
4770 if (!(disp_int[crtc_idx] & mask)) {
4771 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4772 crtc_idx + 1, event_name);
4773 }
4774
4775 disp_int[crtc_idx] &= ~mask;
4776 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4777
4778 break;
4779 case 8: /* D1 page flip */
4780 case 10: /* D2 page flip */
4781 case 12: /* D3 page flip */
4782 case 14: /* D4 page flip */
4783 case 16: /* D5 page flip */
4784 case 18: /* D6 page flip */
4785 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4786 if (radeon_use_pflipirq > 0)
4787 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4788 break;
4789 case 42: /* HPD hotplug */
4790 if (src_data <= 5) {
4791 hpd_idx = src_data;
4792 mask = DC_HPD1_INTERRUPT;
4793 queue_hotplug = true;
4794 event_name = "HPD";
4795
4796 } else if (src_data <= 11) {
4797 hpd_idx = src_data - 6;
4798 mask = DC_HPD1_RX_INTERRUPT;
4799 queue_dp = true;
4800 event_name = "HPD_RX";
4801
4802 } else {
4803 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4804 src_id, src_data);
4805 break;
4806 }
4807
4808 if (!(disp_int[hpd_idx] & mask))
4809 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4810
4811 disp_int[hpd_idx] &= ~mask;
4812 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4813
4814 break;
4815 case 44: /* hdmi */
4816 afmt_idx = src_data;
4817 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4818 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4819
4820 if (afmt_idx > 5) {
4821 DRM_ERROR("Unhandled interrupt: %d %d\n",
4822 src_id, src_data);
4823 break;
4824 }
4825 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4826 queue_hdmi = true;
4827 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4828 break;
4829 case 96:
4830 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4831 WREG32(SRBM_INT_ACK, 0x1);
4832 break;
4833 case 124: /* UVD */
4834 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4835 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4836 break;
4837 case 146:
4838 case 147:
4839 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4840 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4841 /* reset addr and status */
4842 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4843 if (addr == 0x0 && status == 0x0)
4844 break;
4845 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4846 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4847 addr);
4848 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4849 status);
4850 cayman_vm_decode_fault(rdev, status, addr);
4851 break;
4852 case 176: /* CP_INT in ring buffer */
4853 case 177: /* CP_INT in IB1 */
4854 case 178: /* CP_INT in IB2 */
4855 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4856 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4857 break;
4858 case 181: /* CP EOP event */
4859 DRM_DEBUG("IH: CP EOP\n");
4860 if (rdev->family >= CHIP_CAYMAN) {
4861 switch (src_data) {
4862 case 0:
4863 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4864 break;
4865 case 1:
4866 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4867 break;
4868 case 2:
4869 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4870 break;
4871 }
4872 } else
4873 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4874 break;
4875 case 224: /* DMA trap event */
4876 DRM_DEBUG("IH: DMA trap\n");
4877 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4878 break;
4879 case 230: /* thermal low to high */
4880 DRM_DEBUG("IH: thermal low to high\n");
4881 rdev->pm.dpm.thermal.high_to_low = false;
4882 queue_thermal = true;
4883 break;
4884 case 231: /* thermal high to low */
4885 DRM_DEBUG("IH: thermal high to low\n");
4886 rdev->pm.dpm.thermal.high_to_low = true;
4887 queue_thermal = true;
4888 break;
4889 case 233: /* GUI IDLE */
4890 DRM_DEBUG("IH: GUI idle\n");
4891 break;
4892 case 244: /* DMA trap event */
4893 if (rdev->family >= CHIP_CAYMAN) {
4894 DRM_DEBUG("IH: DMA1 trap\n");
4895 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4896 }
4897 break;
4898 default:
4899 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4900 break;
4901 }
4902
4903 /* wptr/rptr are in bytes! */
4904 rptr += 16;
4905 rptr &= rdev->ih.ptr_mask;
4906 WREG32(IH_RB_RPTR, rptr);
4907 }
4908 if (queue_dp)
4909 schedule_work(&rdev->dp_work);
4910 if (queue_hotplug)
4911 schedule_delayed_work(&rdev->hotplug_work, 0);
4912 if (queue_hdmi)
4913 schedule_work(&rdev->audio_work);
4914 if (queue_thermal && rdev->pm.dpm_enabled)
4915 schedule_work(&rdev->pm.dpm.thermal.work);
4916 rdev->ih.rptr = rptr;
4917 atomic_set(&rdev->ih.lock, 0);
4918
4919 /* make sure wptr hasn't changed while processing */
4920 wptr = evergreen_get_ih_wptr(rdev);
4921 if (wptr != rptr)
4922 goto restart_ih;
4923
4924 return IRQ_HANDLED;
4925}
4926
4927static void evergreen_uvd_init(struct radeon_device *rdev)
4928{
4929 int r;
4930
4931 if (!rdev->has_uvd)
4932 return;
4933
4934 r = radeon_uvd_init(rdev);
4935 if (r) {
4936 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4937 /*
4938 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4939 * to early fails uvd_v2_2_resume() and thus nothing happens
4940 * there. So it is pointless to try to go through that code
4941 * hence why we disable uvd here.
4942 */
4943 rdev->has_uvd = false;
4944 return;
4945 }
4946 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4947 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4948}
4949
4950static void evergreen_uvd_start(struct radeon_device *rdev)
4951{
4952 int r;
4953
4954 if (!rdev->has_uvd)
4955 return;
4956
4957 r = uvd_v2_2_resume(rdev);
4958 if (r) {
4959 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4960 goto error;
4961 }
4962 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4963 if (r) {
4964 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4965 goto error;
4966 }
4967 return;
4968
4969error:
4970 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4971}
4972
4973static void evergreen_uvd_resume(struct radeon_device *rdev)
4974{
4975 struct radeon_ring *ring;
4976 int r;
4977
4978 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4979 return;
4980
4981 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4982 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4983 if (r) {
4984 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4985 return;
4986 }
4987 r = uvd_v1_0_init(rdev);
4988 if (r) {
4989 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4990 return;
4991 }
4992}
4993
4994static int evergreen_startup(struct radeon_device *rdev)
4995{
4996 struct radeon_ring *ring;
4997 int r;
4998
4999 /* enable pcie gen2 link */
5000 evergreen_pcie_gen2_enable(rdev);
5001 /* enable aspm */
5002 evergreen_program_aspm(rdev);
5003
5004 /* scratch needs to be initialized before MC */
5005 r = r600_vram_scratch_init(rdev);
5006 if (r)
5007 return r;
5008
5009 evergreen_mc_program(rdev);
5010
5011 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5012 r = ni_mc_load_microcode(rdev);
5013 if (r) {
5014 DRM_ERROR("Failed to load MC firmware!\n");
5015 return r;
5016 }
5017 }
5018
5019 if (rdev->flags & RADEON_IS_AGP) {
5020 evergreen_agp_enable(rdev);
5021 } else {
5022 r = evergreen_pcie_gart_enable(rdev);
5023 if (r)
5024 return r;
5025 }
5026 evergreen_gpu_init(rdev);
5027
5028 /* allocate rlc buffers */
5029 if (rdev->flags & RADEON_IS_IGP) {
5030 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5031 rdev->rlc.reg_list_size =
5032 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5033 rdev->rlc.cs_data = evergreen_cs_data;
5034 r = sumo_rlc_init(rdev);
5035 if (r) {
5036 DRM_ERROR("Failed to init rlc BOs!\n");
5037 return r;
5038 }
5039 }
5040
5041 /* allocate wb buffer */
5042 r = radeon_wb_init(rdev);
5043 if (r)
5044 return r;
5045
5046 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5047 if (r) {
5048 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5049 return r;
5050 }
5051
5052 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5053 if (r) {
5054 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5055 return r;
5056 }
5057
5058 evergreen_uvd_start(rdev);
5059
5060 /* Enable IRQ */
5061 if (!rdev->irq.installed) {
5062 r = radeon_irq_kms_init(rdev);
5063 if (r)
5064 return r;
5065 }
5066
5067 r = r600_irq_init(rdev);
5068 if (r) {
5069 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5070 radeon_irq_kms_fini(rdev);
5071 return r;
5072 }
5073 evergreen_irq_set(rdev);
5074
5075 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5076 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5077 RADEON_CP_PACKET2);
5078 if (r)
5079 return r;
5080
5081 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5082 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5083 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5084 if (r)
5085 return r;
5086
5087 r = evergreen_cp_load_microcode(rdev);
5088 if (r)
5089 return r;
5090 r = evergreen_cp_resume(rdev);
5091 if (r)
5092 return r;
5093 r = r600_dma_resume(rdev);
5094 if (r)
5095 return r;
5096
5097 evergreen_uvd_resume(rdev);
5098
5099 r = radeon_ib_pool_init(rdev);
5100 if (r) {
5101 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5102 return r;
5103 }
5104
5105 r = radeon_audio_init(rdev);
5106 if (r) {
5107 DRM_ERROR("radeon: audio init failed\n");
5108 return r;
5109 }
5110
5111 return 0;
5112}
5113
5114int evergreen_resume(struct radeon_device *rdev)
5115{
5116 int r;
5117
5118 /* reset the asic, the gfx blocks are often in a bad state
5119 * after the driver is unloaded or after a resume
5120 */
5121 if (radeon_asic_reset(rdev))
5122 dev_warn(rdev->dev, "GPU reset failed !\n");
5123 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5124 * posting will perform necessary task to bring back GPU into good
5125 * shape.
5126 */
5127 /* post card */
5128 atom_asic_init(rdev->mode_info.atom_context);
5129
5130 /* init golden registers */
5131 evergreen_init_golden_registers(rdev);
5132
5133 if (rdev->pm.pm_method == PM_METHOD_DPM)
5134 radeon_pm_resume(rdev);
5135
5136 rdev->accel_working = true;
5137 r = evergreen_startup(rdev);
5138 if (r) {
5139 DRM_ERROR("evergreen startup failed on resume\n");
5140 rdev->accel_working = false;
5141 return r;
5142 }
5143
5144 return r;
5145
5146}
5147
5148int evergreen_suspend(struct radeon_device *rdev)
5149{
5150 radeon_pm_suspend(rdev);
5151 radeon_audio_fini(rdev);
5152 if (rdev->has_uvd) {
5153 uvd_v1_0_fini(rdev);
5154 radeon_uvd_suspend(rdev);
5155 }
5156 r700_cp_stop(rdev);
5157 r600_dma_stop(rdev);
5158 evergreen_irq_suspend(rdev);
5159 radeon_wb_disable(rdev);
5160 evergreen_pcie_gart_disable(rdev);
5161
5162 return 0;
5163}
5164
5165/* Plan is to move initialization in that function and use
5166 * helper function so that radeon_device_init pretty much
5167 * do nothing more than calling asic specific function. This
5168 * should also allow to remove a bunch of callback function
5169 * like vram_info.
5170 */
5171int evergreen_init(struct radeon_device *rdev)
5172{
5173 int r;
5174
5175 /* Read BIOS */
5176 if (!radeon_get_bios(rdev)) {
5177 if (ASIC_IS_AVIVO(rdev))
5178 return -EINVAL;
5179 }
5180 /* Must be an ATOMBIOS */
5181 if (!rdev->is_atom_bios) {
5182 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5183 return -EINVAL;
5184 }
5185 r = radeon_atombios_init(rdev);
5186 if (r)
5187 return r;
5188 /* reset the asic, the gfx blocks are often in a bad state
5189 * after the driver is unloaded or after a resume
5190 */
5191 if (radeon_asic_reset(rdev))
5192 dev_warn(rdev->dev, "GPU reset failed !\n");
5193 /* Post card if necessary */
5194 if (!radeon_card_posted(rdev)) {
5195 if (!rdev->bios) {
5196 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5197 return -EINVAL;
5198 }
5199 DRM_INFO("GPU not posted. posting now...\n");
5200 atom_asic_init(rdev->mode_info.atom_context);
5201 }
5202 /* init golden registers */
5203 evergreen_init_golden_registers(rdev);
5204 /* Initialize scratch registers */
5205 r600_scratch_init(rdev);
5206 /* Initialize surface registers */
5207 radeon_surface_init(rdev);
5208 /* Initialize clocks */
5209 radeon_get_clock_info(rdev->ddev);
5210 /* Fence driver */
5211 radeon_fence_driver_init(rdev);
5212 /* initialize AGP */
5213 if (rdev->flags & RADEON_IS_AGP) {
5214 r = radeon_agp_init(rdev);
5215 if (r)
5216 radeon_agp_disable(rdev);
5217 }
5218 /* initialize memory controller */
5219 r = evergreen_mc_init(rdev);
5220 if (r)
5221 return r;
5222 /* Memory manager */
5223 r = radeon_bo_init(rdev);
5224 if (r)
5225 return r;
5226
5227 if (ASIC_IS_DCE5(rdev)) {
5228 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5229 r = ni_init_microcode(rdev);
5230 if (r) {
5231 DRM_ERROR("Failed to load firmware!\n");
5232 return r;
5233 }
5234 }
5235 } else {
5236 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5237 r = r600_init_microcode(rdev);
5238 if (r) {
5239 DRM_ERROR("Failed to load firmware!\n");
5240 return r;
5241 }
5242 }
5243 }
5244
5245 /* Initialize power management */
5246 radeon_pm_init(rdev);
5247
5248 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5249 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5250
5251 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5252 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5253
5254 evergreen_uvd_init(rdev);
5255
5256 rdev->ih.ring_obj = NULL;
5257 r600_ih_ring_init(rdev, 64 * 1024);
5258
5259 r = r600_pcie_gart_init(rdev);
5260 if (r)
5261 return r;
5262
5263 rdev->accel_working = true;
5264 r = evergreen_startup(rdev);
5265 if (r) {
5266 dev_err(rdev->dev, "disabling GPU acceleration\n");
5267 r700_cp_fini(rdev);
5268 r600_dma_fini(rdev);
5269 r600_irq_fini(rdev);
5270 if (rdev->flags & RADEON_IS_IGP)
5271 sumo_rlc_fini(rdev);
5272 radeon_wb_fini(rdev);
5273 radeon_ib_pool_fini(rdev);
5274 radeon_irq_kms_fini(rdev);
5275 evergreen_pcie_gart_fini(rdev);
5276 rdev->accel_working = false;
5277 }
5278
5279 /* Don't start up if the MC ucode is missing on BTC parts.
5280 * The default clocks and voltages before the MC ucode
5281 * is loaded are not suffient for advanced operations.
5282 */
5283 if (ASIC_IS_DCE5(rdev)) {
5284 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5285 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5286 return -EINVAL;
5287 }
5288 }
5289
5290 return 0;
5291}
5292
5293void evergreen_fini(struct radeon_device *rdev)
5294{
5295 radeon_pm_fini(rdev);
5296 radeon_audio_fini(rdev);
5297 r700_cp_fini(rdev);
5298 r600_dma_fini(rdev);
5299 r600_irq_fini(rdev);
5300 if (rdev->flags & RADEON_IS_IGP)
5301 sumo_rlc_fini(rdev);
5302 radeon_wb_fini(rdev);
5303 radeon_ib_pool_fini(rdev);
5304 radeon_irq_kms_fini(rdev);
5305 uvd_v1_0_fini(rdev);
5306 radeon_uvd_fini(rdev);
5307 evergreen_pcie_gart_fini(rdev);
5308 r600_vram_scratch_fini(rdev);
5309 radeon_gem_fini(rdev);
5310 radeon_fence_driver_fini(rdev);
5311 radeon_agp_fini(rdev);
5312 radeon_bo_fini(rdev);
5313 radeon_atombios_fini(rdev);
5314 kfree(rdev->bios);
5315 rdev->bios = NULL;
5316}
5317
5318void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5319{
5320 u32 link_width_cntl, speed_cntl;
5321
5322 if (radeon_pcie_gen2 == 0)
5323 return;
5324
5325 if (rdev->flags & RADEON_IS_IGP)
5326 return;
5327
5328 if (!(rdev->flags & RADEON_IS_PCIE))
5329 return;
5330
5331 /* x2 cards have a special sequence */
5332 if (ASIC_IS_X2(rdev))
5333 return;
5334
5335 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5336 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5337 return;
5338
5339 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5340 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5341 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5342 return;
5343 }
5344
5345 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5346
5347 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5348 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5349
5350 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5351 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5352 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5353
5354 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5355 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5356 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5357
5358 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5359 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5360 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5361
5362 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5363 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5364 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5365
5366 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5367 speed_cntl |= LC_GEN2_EN_STRAP;
5368 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5369
5370 } else {
5371 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5372 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5373 if (1)
5374 link_width_cntl |= LC_UPCONFIGURE_DIS;
5375 else
5376 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5377 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5378 }
5379}
5380
5381void evergreen_program_aspm(struct radeon_device *rdev)
5382{
5383 u32 data, orig;
5384 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5385 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5386 /* fusion_platform = true
5387 * if the system is a fusion system
5388 * (APU or DGPU in a fusion system).
5389 * todo: check if the system is a fusion platform.
5390 */
5391 bool fusion_platform = false;
5392
5393 if (radeon_aspm == 0)
5394 return;
5395
5396 if (!(rdev->flags & RADEON_IS_PCIE))
5397 return;
5398
5399 switch (rdev->family) {
5400 case CHIP_CYPRESS:
5401 case CHIP_HEMLOCK:
5402 case CHIP_JUNIPER:
5403 case CHIP_REDWOOD:
5404 case CHIP_CEDAR:
5405 case CHIP_SUMO:
5406 case CHIP_SUMO2:
5407 case CHIP_PALM:
5408 case CHIP_ARUBA:
5409 disable_l0s = true;
5410 break;
5411 default:
5412 disable_l0s = false;
5413 break;
5414 }
5415
5416 if (rdev->flags & RADEON_IS_IGP)
5417 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5418
5419 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5420 if (fusion_platform)
5421 data &= ~MULTI_PIF;
5422 else
5423 data |= MULTI_PIF;
5424 if (data != orig)
5425 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5426
5427 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5428 if (fusion_platform)
5429 data &= ~MULTI_PIF;
5430 else
5431 data |= MULTI_PIF;
5432 if (data != orig)
5433 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5434
5435 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5436 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5437 if (!disable_l0s) {
5438 if (rdev->family >= CHIP_BARTS)
5439 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5440 else
5441 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5442 }
5443
5444 if (!disable_l1) {
5445 if (rdev->family >= CHIP_BARTS)
5446 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5447 else
5448 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5449
5450 if (!disable_plloff_in_l1) {
5451 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5452 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5453 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5454 if (data != orig)
5455 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5456
5457 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5458 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5459 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5460 if (data != orig)
5461 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5462
5463 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5464 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5465 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5466 if (data != orig)
5467 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5468
5469 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5470 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5471 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5472 if (data != orig)
5473 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5474
5475 if (rdev->family >= CHIP_BARTS) {
5476 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5477 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5478 data |= PLL_RAMP_UP_TIME_0(4);
5479 if (data != orig)
5480 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5481
5482 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5483 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5484 data |= PLL_RAMP_UP_TIME_1(4);
5485 if (data != orig)
5486 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5487
5488 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5489 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5490 data |= PLL_RAMP_UP_TIME_0(4);
5491 if (data != orig)
5492 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5493
5494 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5495 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5496 data |= PLL_RAMP_UP_TIME_1(4);
5497 if (data != orig)
5498 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5499 }
5500
5501 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5502 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5503 data |= LC_DYN_LANES_PWR_STATE(3);
5504 if (data != orig)
5505 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5506
5507 if (rdev->family >= CHIP_BARTS) {
5508 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5509 data &= ~LS2_EXIT_TIME_MASK;
5510 data |= LS2_EXIT_TIME(1);
5511 if (data != orig)
5512 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5513
5514 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5515 data &= ~LS2_EXIT_TIME_MASK;
5516 data |= LS2_EXIT_TIME(1);
5517 if (data != orig)
5518 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5519 }
5520 }
5521 }
5522
5523 /* evergreen parts only */
5524 if (rdev->family < CHIP_BARTS)
5525 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5526
5527 if (pcie_lc_cntl != pcie_lc_cntl_old)
5528 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5529}