Loading...
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include "drmP.h"
28#include "radeon.h"
29#include "radeon_asic.h"
30#include "radeon_drm.h"
31#include "evergreend.h"
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
35#include "evergreen_blit_shaders.h"
36
37#define EVERGREEN_PFP_UCODE_SIZE 1120
38#define EVERGREEN_PM4_UCODE_SIZE 1376
39
40static void evergreen_gpu_init(struct radeon_device *rdev);
41void evergreen_fini(struct radeon_device *rdev);
42static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
43
44void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
45{
46 u16 ctl, v;
47 int cap, err;
48
49 cap = pci_pcie_cap(rdev->pdev);
50 if (!cap)
51 return;
52
53 err = pci_read_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, &ctl);
54 if (err)
55 return;
56
57 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
58
59 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
60 * to avoid hangs or perfomance issues
61 */
62 if ((v == 0) || (v == 6) || (v == 7)) {
63 ctl &= ~PCI_EXP_DEVCTL_READRQ;
64 ctl |= (2 << 12);
65 pci_write_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, ctl);
66 }
67}
68
69void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
70{
71 /* enable the pflip int */
72 radeon_irq_kms_pflip_irq_get(rdev, crtc);
73}
74
75void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
76{
77 /* disable the pflip int */
78 radeon_irq_kms_pflip_irq_put(rdev, crtc);
79}
80
81u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
82{
83 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
84 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
85
86 /* Lock the graphics update lock */
87 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
88 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
89
90 /* update the scanout addresses */
91 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
92 upper_32_bits(crtc_base));
93 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
94 (u32)crtc_base);
95
96 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
97 upper_32_bits(crtc_base));
98 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
99 (u32)crtc_base);
100
101 /* Wait for update_pending to go high. */
102 while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
103 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
104
105 /* Unlock the lock, so double-buffering can take place inside vblank */
106 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
107 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
108
109 /* Return current update_pending status: */
110 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
111}
112
113/* get temperature in millidegrees */
114int evergreen_get_temp(struct radeon_device *rdev)
115{
116 u32 temp, toffset;
117 int actual_temp = 0;
118
119 if (rdev->family == CHIP_JUNIPER) {
120 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
121 TOFFSET_SHIFT;
122 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
123 TS0_ADC_DOUT_SHIFT;
124
125 if (toffset & 0x100)
126 actual_temp = temp / 2 - (0x200 - toffset);
127 else
128 actual_temp = temp / 2 + toffset;
129
130 actual_temp = actual_temp * 1000;
131
132 } else {
133 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
134 ASIC_T_SHIFT;
135
136 if (temp & 0x400)
137 actual_temp = -256;
138 else if (temp & 0x200)
139 actual_temp = 255;
140 else if (temp & 0x100) {
141 actual_temp = temp & 0x1ff;
142 actual_temp |= ~0x1ff;
143 } else
144 actual_temp = temp & 0xff;
145
146 actual_temp = (actual_temp * 1000) / 2;
147 }
148
149 return actual_temp;
150}
151
152int sumo_get_temp(struct radeon_device *rdev)
153{
154 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
155 int actual_temp = temp - 49;
156
157 return actual_temp * 1000;
158}
159
160void evergreen_pm_misc(struct radeon_device *rdev)
161{
162 int req_ps_idx = rdev->pm.requested_power_state_index;
163 int req_cm_idx = rdev->pm.requested_clock_mode_index;
164 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
165 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
166
167 if (voltage->type == VOLTAGE_SW) {
168 /* 0xff01 is a flag rather then an actual voltage */
169 if (voltage->voltage == 0xff01)
170 return;
171 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
172 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
173 rdev->pm.current_vddc = voltage->voltage;
174 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
175 }
176 /* 0xff01 is a flag rather then an actual voltage */
177 if (voltage->vddci == 0xff01)
178 return;
179 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
180 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
181 rdev->pm.current_vddci = voltage->vddci;
182 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
183 }
184 }
185}
186
187void evergreen_pm_prepare(struct radeon_device *rdev)
188{
189 struct drm_device *ddev = rdev->ddev;
190 struct drm_crtc *crtc;
191 struct radeon_crtc *radeon_crtc;
192 u32 tmp;
193
194 /* disable any active CRTCs */
195 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
196 radeon_crtc = to_radeon_crtc(crtc);
197 if (radeon_crtc->enabled) {
198 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
199 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
200 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
201 }
202 }
203}
204
205void evergreen_pm_finish(struct radeon_device *rdev)
206{
207 struct drm_device *ddev = rdev->ddev;
208 struct drm_crtc *crtc;
209 struct radeon_crtc *radeon_crtc;
210 u32 tmp;
211
212 /* enable any active CRTCs */
213 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
214 radeon_crtc = to_radeon_crtc(crtc);
215 if (radeon_crtc->enabled) {
216 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
217 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
218 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
219 }
220 }
221}
222
223bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
224{
225 bool connected = false;
226
227 switch (hpd) {
228 case RADEON_HPD_1:
229 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
230 connected = true;
231 break;
232 case RADEON_HPD_2:
233 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
234 connected = true;
235 break;
236 case RADEON_HPD_3:
237 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
238 connected = true;
239 break;
240 case RADEON_HPD_4:
241 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
242 connected = true;
243 break;
244 case RADEON_HPD_5:
245 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
246 connected = true;
247 break;
248 case RADEON_HPD_6:
249 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
250 connected = true;
251 break;
252 default:
253 break;
254 }
255
256 return connected;
257}
258
259void evergreen_hpd_set_polarity(struct radeon_device *rdev,
260 enum radeon_hpd_id hpd)
261{
262 u32 tmp;
263 bool connected = evergreen_hpd_sense(rdev, hpd);
264
265 switch (hpd) {
266 case RADEON_HPD_1:
267 tmp = RREG32(DC_HPD1_INT_CONTROL);
268 if (connected)
269 tmp &= ~DC_HPDx_INT_POLARITY;
270 else
271 tmp |= DC_HPDx_INT_POLARITY;
272 WREG32(DC_HPD1_INT_CONTROL, tmp);
273 break;
274 case RADEON_HPD_2:
275 tmp = RREG32(DC_HPD2_INT_CONTROL);
276 if (connected)
277 tmp &= ~DC_HPDx_INT_POLARITY;
278 else
279 tmp |= DC_HPDx_INT_POLARITY;
280 WREG32(DC_HPD2_INT_CONTROL, tmp);
281 break;
282 case RADEON_HPD_3:
283 tmp = RREG32(DC_HPD3_INT_CONTROL);
284 if (connected)
285 tmp &= ~DC_HPDx_INT_POLARITY;
286 else
287 tmp |= DC_HPDx_INT_POLARITY;
288 WREG32(DC_HPD3_INT_CONTROL, tmp);
289 break;
290 case RADEON_HPD_4:
291 tmp = RREG32(DC_HPD4_INT_CONTROL);
292 if (connected)
293 tmp &= ~DC_HPDx_INT_POLARITY;
294 else
295 tmp |= DC_HPDx_INT_POLARITY;
296 WREG32(DC_HPD4_INT_CONTROL, tmp);
297 break;
298 case RADEON_HPD_5:
299 tmp = RREG32(DC_HPD5_INT_CONTROL);
300 if (connected)
301 tmp &= ~DC_HPDx_INT_POLARITY;
302 else
303 tmp |= DC_HPDx_INT_POLARITY;
304 WREG32(DC_HPD5_INT_CONTROL, tmp);
305 break;
306 case RADEON_HPD_6:
307 tmp = RREG32(DC_HPD6_INT_CONTROL);
308 if (connected)
309 tmp &= ~DC_HPDx_INT_POLARITY;
310 else
311 tmp |= DC_HPDx_INT_POLARITY;
312 WREG32(DC_HPD6_INT_CONTROL, tmp);
313 break;
314 default:
315 break;
316 }
317}
318
319void evergreen_hpd_init(struct radeon_device *rdev)
320{
321 struct drm_device *dev = rdev->ddev;
322 struct drm_connector *connector;
323 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
324 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
325
326 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
327 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
328 switch (radeon_connector->hpd.hpd) {
329 case RADEON_HPD_1:
330 WREG32(DC_HPD1_CONTROL, tmp);
331 rdev->irq.hpd[0] = true;
332 break;
333 case RADEON_HPD_2:
334 WREG32(DC_HPD2_CONTROL, tmp);
335 rdev->irq.hpd[1] = true;
336 break;
337 case RADEON_HPD_3:
338 WREG32(DC_HPD3_CONTROL, tmp);
339 rdev->irq.hpd[2] = true;
340 break;
341 case RADEON_HPD_4:
342 WREG32(DC_HPD4_CONTROL, tmp);
343 rdev->irq.hpd[3] = true;
344 break;
345 case RADEON_HPD_5:
346 WREG32(DC_HPD5_CONTROL, tmp);
347 rdev->irq.hpd[4] = true;
348 break;
349 case RADEON_HPD_6:
350 WREG32(DC_HPD6_CONTROL, tmp);
351 rdev->irq.hpd[5] = true;
352 break;
353 default:
354 break;
355 }
356 }
357 if (rdev->irq.installed)
358 evergreen_irq_set(rdev);
359}
360
361void evergreen_hpd_fini(struct radeon_device *rdev)
362{
363 struct drm_device *dev = rdev->ddev;
364 struct drm_connector *connector;
365
366 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
367 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
368 switch (radeon_connector->hpd.hpd) {
369 case RADEON_HPD_1:
370 WREG32(DC_HPD1_CONTROL, 0);
371 rdev->irq.hpd[0] = false;
372 break;
373 case RADEON_HPD_2:
374 WREG32(DC_HPD2_CONTROL, 0);
375 rdev->irq.hpd[1] = false;
376 break;
377 case RADEON_HPD_3:
378 WREG32(DC_HPD3_CONTROL, 0);
379 rdev->irq.hpd[2] = false;
380 break;
381 case RADEON_HPD_4:
382 WREG32(DC_HPD4_CONTROL, 0);
383 rdev->irq.hpd[3] = false;
384 break;
385 case RADEON_HPD_5:
386 WREG32(DC_HPD5_CONTROL, 0);
387 rdev->irq.hpd[4] = false;
388 break;
389 case RADEON_HPD_6:
390 WREG32(DC_HPD6_CONTROL, 0);
391 rdev->irq.hpd[5] = false;
392 break;
393 default:
394 break;
395 }
396 }
397}
398
399/* watermark setup */
400
401static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
402 struct radeon_crtc *radeon_crtc,
403 struct drm_display_mode *mode,
404 struct drm_display_mode *other_mode)
405{
406 u32 tmp;
407 /*
408 * Line Buffer Setup
409 * There are 3 line buffers, each one shared by 2 display controllers.
410 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
411 * the display controllers. The paritioning is done via one of four
412 * preset allocations specified in bits 2:0:
413 * first display controller
414 * 0 - first half of lb (3840 * 2)
415 * 1 - first 3/4 of lb (5760 * 2)
416 * 2 - whole lb (7680 * 2), other crtc must be disabled
417 * 3 - first 1/4 of lb (1920 * 2)
418 * second display controller
419 * 4 - second half of lb (3840 * 2)
420 * 5 - second 3/4 of lb (5760 * 2)
421 * 6 - whole lb (7680 * 2), other crtc must be disabled
422 * 7 - last 1/4 of lb (1920 * 2)
423 */
424 /* this can get tricky if we have two large displays on a paired group
425 * of crtcs. Ideally for multiple large displays we'd assign them to
426 * non-linked crtcs for maximum line buffer allocation.
427 */
428 if (radeon_crtc->base.enabled && mode) {
429 if (other_mode)
430 tmp = 0; /* 1/2 */
431 else
432 tmp = 2; /* whole */
433 } else
434 tmp = 0;
435
436 /* second controller of the pair uses second half of the lb */
437 if (radeon_crtc->crtc_id % 2)
438 tmp += 4;
439 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
440
441 if (radeon_crtc->base.enabled && mode) {
442 switch (tmp) {
443 case 0:
444 case 4:
445 default:
446 if (ASIC_IS_DCE5(rdev))
447 return 4096 * 2;
448 else
449 return 3840 * 2;
450 case 1:
451 case 5:
452 if (ASIC_IS_DCE5(rdev))
453 return 6144 * 2;
454 else
455 return 5760 * 2;
456 case 2:
457 case 6:
458 if (ASIC_IS_DCE5(rdev))
459 return 8192 * 2;
460 else
461 return 7680 * 2;
462 case 3:
463 case 7:
464 if (ASIC_IS_DCE5(rdev))
465 return 2048 * 2;
466 else
467 return 1920 * 2;
468 }
469 }
470
471 /* controller not enabled, so no lb used */
472 return 0;
473}
474
475static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
476{
477 u32 tmp = RREG32(MC_SHARED_CHMAP);
478
479 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
480 case 0:
481 default:
482 return 1;
483 case 1:
484 return 2;
485 case 2:
486 return 4;
487 case 3:
488 return 8;
489 }
490}
491
492struct evergreen_wm_params {
493 u32 dram_channels; /* number of dram channels */
494 u32 yclk; /* bandwidth per dram data pin in kHz */
495 u32 sclk; /* engine clock in kHz */
496 u32 disp_clk; /* display clock in kHz */
497 u32 src_width; /* viewport width */
498 u32 active_time; /* active display time in ns */
499 u32 blank_time; /* blank time in ns */
500 bool interlaced; /* mode is interlaced */
501 fixed20_12 vsc; /* vertical scale ratio */
502 u32 num_heads; /* number of active crtcs */
503 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
504 u32 lb_size; /* line buffer allocated to pipe */
505 u32 vtaps; /* vertical scaler taps */
506};
507
508static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
509{
510 /* Calculate DRAM Bandwidth and the part allocated to display. */
511 fixed20_12 dram_efficiency; /* 0.7 */
512 fixed20_12 yclk, dram_channels, bandwidth;
513 fixed20_12 a;
514
515 a.full = dfixed_const(1000);
516 yclk.full = dfixed_const(wm->yclk);
517 yclk.full = dfixed_div(yclk, a);
518 dram_channels.full = dfixed_const(wm->dram_channels * 4);
519 a.full = dfixed_const(10);
520 dram_efficiency.full = dfixed_const(7);
521 dram_efficiency.full = dfixed_div(dram_efficiency, a);
522 bandwidth.full = dfixed_mul(dram_channels, yclk);
523 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
524
525 return dfixed_trunc(bandwidth);
526}
527
528static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
529{
530 /* Calculate DRAM Bandwidth and the part allocated to display. */
531 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
532 fixed20_12 yclk, dram_channels, bandwidth;
533 fixed20_12 a;
534
535 a.full = dfixed_const(1000);
536 yclk.full = dfixed_const(wm->yclk);
537 yclk.full = dfixed_div(yclk, a);
538 dram_channels.full = dfixed_const(wm->dram_channels * 4);
539 a.full = dfixed_const(10);
540 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
541 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
542 bandwidth.full = dfixed_mul(dram_channels, yclk);
543 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
544
545 return dfixed_trunc(bandwidth);
546}
547
548static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
549{
550 /* Calculate the display Data return Bandwidth */
551 fixed20_12 return_efficiency; /* 0.8 */
552 fixed20_12 sclk, bandwidth;
553 fixed20_12 a;
554
555 a.full = dfixed_const(1000);
556 sclk.full = dfixed_const(wm->sclk);
557 sclk.full = dfixed_div(sclk, a);
558 a.full = dfixed_const(10);
559 return_efficiency.full = dfixed_const(8);
560 return_efficiency.full = dfixed_div(return_efficiency, a);
561 a.full = dfixed_const(32);
562 bandwidth.full = dfixed_mul(a, sclk);
563 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
564
565 return dfixed_trunc(bandwidth);
566}
567
568static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
569{
570 /* Calculate the DMIF Request Bandwidth */
571 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
572 fixed20_12 disp_clk, bandwidth;
573 fixed20_12 a;
574
575 a.full = dfixed_const(1000);
576 disp_clk.full = dfixed_const(wm->disp_clk);
577 disp_clk.full = dfixed_div(disp_clk, a);
578 a.full = dfixed_const(10);
579 disp_clk_request_efficiency.full = dfixed_const(8);
580 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
581 a.full = dfixed_const(32);
582 bandwidth.full = dfixed_mul(a, disp_clk);
583 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
584
585 return dfixed_trunc(bandwidth);
586}
587
588static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
589{
590 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
591 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
592 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
593 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
594
595 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
596}
597
598static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
599{
600 /* Calculate the display mode Average Bandwidth
601 * DisplayMode should contain the source and destination dimensions,
602 * timing, etc.
603 */
604 fixed20_12 bpp;
605 fixed20_12 line_time;
606 fixed20_12 src_width;
607 fixed20_12 bandwidth;
608 fixed20_12 a;
609
610 a.full = dfixed_const(1000);
611 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
612 line_time.full = dfixed_div(line_time, a);
613 bpp.full = dfixed_const(wm->bytes_per_pixel);
614 src_width.full = dfixed_const(wm->src_width);
615 bandwidth.full = dfixed_mul(src_width, bpp);
616 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
617 bandwidth.full = dfixed_div(bandwidth, line_time);
618
619 return dfixed_trunc(bandwidth);
620}
621
622static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
623{
624 /* First calcualte the latency in ns */
625 u32 mc_latency = 2000; /* 2000 ns. */
626 u32 available_bandwidth = evergreen_available_bandwidth(wm);
627 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
628 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
629 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
630 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
631 (wm->num_heads * cursor_line_pair_return_time);
632 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
633 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
634 fixed20_12 a, b, c;
635
636 if (wm->num_heads == 0)
637 return 0;
638
639 a.full = dfixed_const(2);
640 b.full = dfixed_const(1);
641 if ((wm->vsc.full > a.full) ||
642 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
643 (wm->vtaps >= 5) ||
644 ((wm->vsc.full >= a.full) && wm->interlaced))
645 max_src_lines_per_dst_line = 4;
646 else
647 max_src_lines_per_dst_line = 2;
648
649 a.full = dfixed_const(available_bandwidth);
650 b.full = dfixed_const(wm->num_heads);
651 a.full = dfixed_div(a, b);
652
653 b.full = dfixed_const(1000);
654 c.full = dfixed_const(wm->disp_clk);
655 b.full = dfixed_div(c, b);
656 c.full = dfixed_const(wm->bytes_per_pixel);
657 b.full = dfixed_mul(b, c);
658
659 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
660
661 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
662 b.full = dfixed_const(1000);
663 c.full = dfixed_const(lb_fill_bw);
664 b.full = dfixed_div(c, b);
665 a.full = dfixed_div(a, b);
666 line_fill_time = dfixed_trunc(a);
667
668 if (line_fill_time < wm->active_time)
669 return latency;
670 else
671 return latency + (line_fill_time - wm->active_time);
672
673}
674
675static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
676{
677 if (evergreen_average_bandwidth(wm) <=
678 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
679 return true;
680 else
681 return false;
682};
683
684static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
685{
686 if (evergreen_average_bandwidth(wm) <=
687 (evergreen_available_bandwidth(wm) / wm->num_heads))
688 return true;
689 else
690 return false;
691};
692
693static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
694{
695 u32 lb_partitions = wm->lb_size / wm->src_width;
696 u32 line_time = wm->active_time + wm->blank_time;
697 u32 latency_tolerant_lines;
698 u32 latency_hiding;
699 fixed20_12 a;
700
701 a.full = dfixed_const(1);
702 if (wm->vsc.full > a.full)
703 latency_tolerant_lines = 1;
704 else {
705 if (lb_partitions <= (wm->vtaps + 1))
706 latency_tolerant_lines = 1;
707 else
708 latency_tolerant_lines = 2;
709 }
710
711 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
712
713 if (evergreen_latency_watermark(wm) <= latency_hiding)
714 return true;
715 else
716 return false;
717}
718
719static void evergreen_program_watermarks(struct radeon_device *rdev,
720 struct radeon_crtc *radeon_crtc,
721 u32 lb_size, u32 num_heads)
722{
723 struct drm_display_mode *mode = &radeon_crtc->base.mode;
724 struct evergreen_wm_params wm;
725 u32 pixel_period;
726 u32 line_time = 0;
727 u32 latency_watermark_a = 0, latency_watermark_b = 0;
728 u32 priority_a_mark = 0, priority_b_mark = 0;
729 u32 priority_a_cnt = PRIORITY_OFF;
730 u32 priority_b_cnt = PRIORITY_OFF;
731 u32 pipe_offset = radeon_crtc->crtc_id * 16;
732 u32 tmp, arb_control3;
733 fixed20_12 a, b, c;
734
735 if (radeon_crtc->base.enabled && num_heads && mode) {
736 pixel_period = 1000000 / (u32)mode->clock;
737 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
738 priority_a_cnt = 0;
739 priority_b_cnt = 0;
740
741 wm.yclk = rdev->pm.current_mclk * 10;
742 wm.sclk = rdev->pm.current_sclk * 10;
743 wm.disp_clk = mode->clock;
744 wm.src_width = mode->crtc_hdisplay;
745 wm.active_time = mode->crtc_hdisplay * pixel_period;
746 wm.blank_time = line_time - wm.active_time;
747 wm.interlaced = false;
748 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
749 wm.interlaced = true;
750 wm.vsc = radeon_crtc->vsc;
751 wm.vtaps = 1;
752 if (radeon_crtc->rmx_type != RMX_OFF)
753 wm.vtaps = 2;
754 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
755 wm.lb_size = lb_size;
756 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
757 wm.num_heads = num_heads;
758
759 /* set for high clocks */
760 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
761 /* set for low clocks */
762 /* wm.yclk = low clk; wm.sclk = low clk */
763 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
764
765 /* possibly force display priority to high */
766 /* should really do this at mode validation time... */
767 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
768 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
769 !evergreen_check_latency_hiding(&wm) ||
770 (rdev->disp_priority == 2)) {
771 DRM_DEBUG_KMS("force priority to high\n");
772 priority_a_cnt |= PRIORITY_ALWAYS_ON;
773 priority_b_cnt |= PRIORITY_ALWAYS_ON;
774 }
775
776 a.full = dfixed_const(1000);
777 b.full = dfixed_const(mode->clock);
778 b.full = dfixed_div(b, a);
779 c.full = dfixed_const(latency_watermark_a);
780 c.full = dfixed_mul(c, b);
781 c.full = dfixed_mul(c, radeon_crtc->hsc);
782 c.full = dfixed_div(c, a);
783 a.full = dfixed_const(16);
784 c.full = dfixed_div(c, a);
785 priority_a_mark = dfixed_trunc(c);
786 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
787
788 a.full = dfixed_const(1000);
789 b.full = dfixed_const(mode->clock);
790 b.full = dfixed_div(b, a);
791 c.full = dfixed_const(latency_watermark_b);
792 c.full = dfixed_mul(c, b);
793 c.full = dfixed_mul(c, radeon_crtc->hsc);
794 c.full = dfixed_div(c, a);
795 a.full = dfixed_const(16);
796 c.full = dfixed_div(c, a);
797 priority_b_mark = dfixed_trunc(c);
798 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
799 }
800
801 /* select wm A */
802 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
803 tmp = arb_control3;
804 tmp &= ~LATENCY_WATERMARK_MASK(3);
805 tmp |= LATENCY_WATERMARK_MASK(1);
806 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
807 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
808 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
809 LATENCY_HIGH_WATERMARK(line_time)));
810 /* select wm B */
811 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
812 tmp &= ~LATENCY_WATERMARK_MASK(3);
813 tmp |= LATENCY_WATERMARK_MASK(2);
814 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
815 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
816 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
817 LATENCY_HIGH_WATERMARK(line_time)));
818 /* restore original selection */
819 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
820
821 /* write the priority marks */
822 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
823 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
824
825}
826
827void evergreen_bandwidth_update(struct radeon_device *rdev)
828{
829 struct drm_display_mode *mode0 = NULL;
830 struct drm_display_mode *mode1 = NULL;
831 u32 num_heads = 0, lb_size;
832 int i;
833
834 radeon_update_display_priority(rdev);
835
836 for (i = 0; i < rdev->num_crtc; i++) {
837 if (rdev->mode_info.crtcs[i]->base.enabled)
838 num_heads++;
839 }
840 for (i = 0; i < rdev->num_crtc; i += 2) {
841 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
842 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
843 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
844 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
845 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
846 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
847 }
848}
849
850int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
851{
852 unsigned i;
853 u32 tmp;
854
855 for (i = 0; i < rdev->usec_timeout; i++) {
856 /* read MC_STATUS */
857 tmp = RREG32(SRBM_STATUS) & 0x1F00;
858 if (!tmp)
859 return 0;
860 udelay(1);
861 }
862 return -1;
863}
864
865/*
866 * GART
867 */
868void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
869{
870 unsigned i;
871 u32 tmp;
872
873 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
874
875 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
876 for (i = 0; i < rdev->usec_timeout; i++) {
877 /* read MC_STATUS */
878 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
879 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
880 if (tmp == 2) {
881 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
882 return;
883 }
884 if (tmp) {
885 return;
886 }
887 udelay(1);
888 }
889}
890
891int evergreen_pcie_gart_enable(struct radeon_device *rdev)
892{
893 u32 tmp;
894 int r;
895
896 if (rdev->gart.table.vram.robj == NULL) {
897 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
898 return -EINVAL;
899 }
900 r = radeon_gart_table_vram_pin(rdev);
901 if (r)
902 return r;
903 radeon_gart_restore(rdev);
904 /* Setup L2 cache */
905 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
906 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
907 EFFECTIVE_L2_QUEUE_SIZE(7));
908 WREG32(VM_L2_CNTL2, 0);
909 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
910 /* Setup TLB control */
911 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
912 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
913 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
914 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
915 if (rdev->flags & RADEON_IS_IGP) {
916 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
917 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
918 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
919 } else {
920 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
921 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
922 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
923 }
924 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
925 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
926 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
927 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
928 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
929 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
930 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
931 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
932 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
933 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
934 (u32)(rdev->dummy_page.addr >> 12));
935 WREG32(VM_CONTEXT1_CNTL, 0);
936
937 evergreen_pcie_gart_tlb_flush(rdev);
938 rdev->gart.ready = true;
939 return 0;
940}
941
942void evergreen_pcie_gart_disable(struct radeon_device *rdev)
943{
944 u32 tmp;
945 int r;
946
947 /* Disable all tables */
948 WREG32(VM_CONTEXT0_CNTL, 0);
949 WREG32(VM_CONTEXT1_CNTL, 0);
950
951 /* Setup L2 cache */
952 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
953 EFFECTIVE_L2_QUEUE_SIZE(7));
954 WREG32(VM_L2_CNTL2, 0);
955 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
956 /* Setup TLB control */
957 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
958 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
959 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
960 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
961 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
962 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
963 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
964 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
965 if (rdev->gart.table.vram.robj) {
966 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
967 if (likely(r == 0)) {
968 radeon_bo_kunmap(rdev->gart.table.vram.robj);
969 radeon_bo_unpin(rdev->gart.table.vram.robj);
970 radeon_bo_unreserve(rdev->gart.table.vram.robj);
971 }
972 }
973}
974
975void evergreen_pcie_gart_fini(struct radeon_device *rdev)
976{
977 evergreen_pcie_gart_disable(rdev);
978 radeon_gart_table_vram_free(rdev);
979 radeon_gart_fini(rdev);
980}
981
982
983void evergreen_agp_enable(struct radeon_device *rdev)
984{
985 u32 tmp;
986
987 /* Setup L2 cache */
988 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
989 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
990 EFFECTIVE_L2_QUEUE_SIZE(7));
991 WREG32(VM_L2_CNTL2, 0);
992 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
993 /* Setup TLB control */
994 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
995 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
996 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
997 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
998 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
999 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1000 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1001 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1002 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1003 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1004 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1005 WREG32(VM_CONTEXT0_CNTL, 0);
1006 WREG32(VM_CONTEXT1_CNTL, 0);
1007}
1008
1009void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1010{
1011 save->vga_control[0] = RREG32(D1VGA_CONTROL);
1012 save->vga_control[1] = RREG32(D2VGA_CONTROL);
1013 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
1014 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
1015 save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
1016 save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
1017 if (rdev->num_crtc >= 4) {
1018 save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
1019 save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
1020 save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
1021 save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
1022 }
1023 if (rdev->num_crtc >= 6) {
1024 save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
1025 save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
1026 save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
1027 save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1028 }
1029
1030 /* Stop all video */
1031 WREG32(VGA_RENDER_CONTROL, 0);
1032 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1033 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1034 if (rdev->num_crtc >= 4) {
1035 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1036 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1037 }
1038 if (rdev->num_crtc >= 6) {
1039 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1040 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1041 }
1042 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1043 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1044 if (rdev->num_crtc >= 4) {
1045 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1046 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1047 }
1048 if (rdev->num_crtc >= 6) {
1049 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1050 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1051 }
1052 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1053 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1054 if (rdev->num_crtc >= 4) {
1055 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1056 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1057 }
1058 if (rdev->num_crtc >= 6) {
1059 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1060 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1061 }
1062
1063 WREG32(D1VGA_CONTROL, 0);
1064 WREG32(D2VGA_CONTROL, 0);
1065 if (rdev->num_crtc >= 4) {
1066 WREG32(EVERGREEN_D3VGA_CONTROL, 0);
1067 WREG32(EVERGREEN_D4VGA_CONTROL, 0);
1068 }
1069 if (rdev->num_crtc >= 6) {
1070 WREG32(EVERGREEN_D5VGA_CONTROL, 0);
1071 WREG32(EVERGREEN_D6VGA_CONTROL, 0);
1072 }
1073}
1074
1075void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1076{
1077 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1078 upper_32_bits(rdev->mc.vram_start));
1079 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1080 upper_32_bits(rdev->mc.vram_start));
1081 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1082 (u32)rdev->mc.vram_start);
1083 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1084 (u32)rdev->mc.vram_start);
1085
1086 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1087 upper_32_bits(rdev->mc.vram_start));
1088 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1089 upper_32_bits(rdev->mc.vram_start));
1090 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1091 (u32)rdev->mc.vram_start);
1092 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1093 (u32)rdev->mc.vram_start);
1094
1095 if (rdev->num_crtc >= 4) {
1096 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1097 upper_32_bits(rdev->mc.vram_start));
1098 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1099 upper_32_bits(rdev->mc.vram_start));
1100 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1101 (u32)rdev->mc.vram_start);
1102 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1103 (u32)rdev->mc.vram_start);
1104
1105 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1106 upper_32_bits(rdev->mc.vram_start));
1107 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1108 upper_32_bits(rdev->mc.vram_start));
1109 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1110 (u32)rdev->mc.vram_start);
1111 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1112 (u32)rdev->mc.vram_start);
1113 }
1114 if (rdev->num_crtc >= 6) {
1115 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1116 upper_32_bits(rdev->mc.vram_start));
1117 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1118 upper_32_bits(rdev->mc.vram_start));
1119 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1120 (u32)rdev->mc.vram_start);
1121 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1122 (u32)rdev->mc.vram_start);
1123
1124 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1125 upper_32_bits(rdev->mc.vram_start));
1126 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1127 upper_32_bits(rdev->mc.vram_start));
1128 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1129 (u32)rdev->mc.vram_start);
1130 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1131 (u32)rdev->mc.vram_start);
1132 }
1133
1134 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1135 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1136 /* Unlock host access */
1137 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1138 mdelay(1);
1139 /* Restore video state */
1140 WREG32(D1VGA_CONTROL, save->vga_control[0]);
1141 WREG32(D2VGA_CONTROL, save->vga_control[1]);
1142 if (rdev->num_crtc >= 4) {
1143 WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
1144 WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
1145 }
1146 if (rdev->num_crtc >= 6) {
1147 WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
1148 WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
1149 }
1150 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1151 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1152 if (rdev->num_crtc >= 4) {
1153 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1154 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1155 }
1156 if (rdev->num_crtc >= 6) {
1157 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1158 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1159 }
1160 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
1161 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1162 if (rdev->num_crtc >= 4) {
1163 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
1164 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
1165 }
1166 if (rdev->num_crtc >= 6) {
1167 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
1168 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1169 }
1170 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1171 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1172 if (rdev->num_crtc >= 4) {
1173 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1174 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1175 }
1176 if (rdev->num_crtc >= 6) {
1177 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1178 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1179 }
1180 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1181}
1182
1183void evergreen_mc_program(struct radeon_device *rdev)
1184{
1185 struct evergreen_mc_save save;
1186 u32 tmp;
1187 int i, j;
1188
1189 /* Initialize HDP */
1190 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1191 WREG32((0x2c14 + j), 0x00000000);
1192 WREG32((0x2c18 + j), 0x00000000);
1193 WREG32((0x2c1c + j), 0x00000000);
1194 WREG32((0x2c20 + j), 0x00000000);
1195 WREG32((0x2c24 + j), 0x00000000);
1196 }
1197 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1198
1199 evergreen_mc_stop(rdev, &save);
1200 if (evergreen_mc_wait_for_idle(rdev)) {
1201 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1202 }
1203 /* Lockout access through VGA aperture*/
1204 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1205 /* Update configuration */
1206 if (rdev->flags & RADEON_IS_AGP) {
1207 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1208 /* VRAM before AGP */
1209 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1210 rdev->mc.vram_start >> 12);
1211 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1212 rdev->mc.gtt_end >> 12);
1213 } else {
1214 /* VRAM after AGP */
1215 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1216 rdev->mc.gtt_start >> 12);
1217 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1218 rdev->mc.vram_end >> 12);
1219 }
1220 } else {
1221 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1222 rdev->mc.vram_start >> 12);
1223 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1224 rdev->mc.vram_end >> 12);
1225 }
1226 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1227 if (rdev->flags & RADEON_IS_IGP) {
1228 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1229 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1230 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1231 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1232 }
1233 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1234 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1235 WREG32(MC_VM_FB_LOCATION, tmp);
1236 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1237 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1238 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1239 if (rdev->flags & RADEON_IS_AGP) {
1240 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1241 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1242 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1243 } else {
1244 WREG32(MC_VM_AGP_BASE, 0);
1245 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1246 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1247 }
1248 if (evergreen_mc_wait_for_idle(rdev)) {
1249 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1250 }
1251 evergreen_mc_resume(rdev, &save);
1252 /* we need to own VRAM, so turn off the VGA renderer here
1253 * to stop it overwriting our objects */
1254 rv515_vga_render_disable(rdev);
1255}
1256
1257/*
1258 * CP.
1259 */
1260void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1261{
1262 /* set to DX10/11 mode */
1263 radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
1264 radeon_ring_write(rdev, 1);
1265 /* FIXME: implement */
1266 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1267 radeon_ring_write(rdev,
1268#ifdef __BIG_ENDIAN
1269 (2 << 0) |
1270#endif
1271 (ib->gpu_addr & 0xFFFFFFFC));
1272 radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
1273 radeon_ring_write(rdev, ib->length_dw);
1274}
1275
1276
1277static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1278{
1279 const __be32 *fw_data;
1280 int i;
1281
1282 if (!rdev->me_fw || !rdev->pfp_fw)
1283 return -EINVAL;
1284
1285 r700_cp_stop(rdev);
1286 WREG32(CP_RB_CNTL,
1287#ifdef __BIG_ENDIAN
1288 BUF_SWAP_32BIT |
1289#endif
1290 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1291
1292 fw_data = (const __be32 *)rdev->pfp_fw->data;
1293 WREG32(CP_PFP_UCODE_ADDR, 0);
1294 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1295 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1296 WREG32(CP_PFP_UCODE_ADDR, 0);
1297
1298 fw_data = (const __be32 *)rdev->me_fw->data;
1299 WREG32(CP_ME_RAM_WADDR, 0);
1300 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1301 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1302
1303 WREG32(CP_PFP_UCODE_ADDR, 0);
1304 WREG32(CP_ME_RAM_WADDR, 0);
1305 WREG32(CP_ME_RAM_RADDR, 0);
1306 return 0;
1307}
1308
1309static int evergreen_cp_start(struct radeon_device *rdev)
1310{
1311 int r, i;
1312 uint32_t cp_me;
1313
1314 r = radeon_ring_lock(rdev, 7);
1315 if (r) {
1316 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1317 return r;
1318 }
1319 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1320 radeon_ring_write(rdev, 0x1);
1321 radeon_ring_write(rdev, 0x0);
1322 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1323 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1324 radeon_ring_write(rdev, 0);
1325 radeon_ring_write(rdev, 0);
1326 radeon_ring_unlock_commit(rdev);
1327
1328 cp_me = 0xff;
1329 WREG32(CP_ME_CNTL, cp_me);
1330
1331 r = radeon_ring_lock(rdev, evergreen_default_size + 19);
1332 if (r) {
1333 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1334 return r;
1335 }
1336
1337 /* setup clear context state */
1338 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1339 radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1340
1341 for (i = 0; i < evergreen_default_size; i++)
1342 radeon_ring_write(rdev, evergreen_default_state[i]);
1343
1344 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1345 radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
1346
1347 /* set clear context state */
1348 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
1349 radeon_ring_write(rdev, 0);
1350
1351 /* SQ_VTX_BASE_VTX_LOC */
1352 radeon_ring_write(rdev, 0xc0026f00);
1353 radeon_ring_write(rdev, 0x00000000);
1354 radeon_ring_write(rdev, 0x00000000);
1355 radeon_ring_write(rdev, 0x00000000);
1356
1357 /* Clear consts */
1358 radeon_ring_write(rdev, 0xc0036f00);
1359 radeon_ring_write(rdev, 0x00000bc4);
1360 radeon_ring_write(rdev, 0xffffffff);
1361 radeon_ring_write(rdev, 0xffffffff);
1362 radeon_ring_write(rdev, 0xffffffff);
1363
1364 radeon_ring_write(rdev, 0xc0026900);
1365 radeon_ring_write(rdev, 0x00000316);
1366 radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1367 radeon_ring_write(rdev, 0x00000010); /* */
1368
1369 radeon_ring_unlock_commit(rdev);
1370
1371 return 0;
1372}
1373
1374int evergreen_cp_resume(struct radeon_device *rdev)
1375{
1376 u32 tmp;
1377 u32 rb_bufsz;
1378 int r;
1379
1380 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1381 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1382 SOFT_RESET_PA |
1383 SOFT_RESET_SH |
1384 SOFT_RESET_VGT |
1385 SOFT_RESET_SPI |
1386 SOFT_RESET_SX));
1387 RREG32(GRBM_SOFT_RESET);
1388 mdelay(15);
1389 WREG32(GRBM_SOFT_RESET, 0);
1390 RREG32(GRBM_SOFT_RESET);
1391
1392 /* Set ring buffer size */
1393 rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1394 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1395#ifdef __BIG_ENDIAN
1396 tmp |= BUF_SWAP_32BIT;
1397#endif
1398 WREG32(CP_RB_CNTL, tmp);
1399 WREG32(CP_SEM_WAIT_TIMER, 0x4);
1400
1401 /* Set the write pointer delay */
1402 WREG32(CP_RB_WPTR_DELAY, 0);
1403
1404 /* Initialize the ring buffer's read and write pointers */
1405 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1406 WREG32(CP_RB_RPTR_WR, 0);
1407 rdev->cp.wptr = 0;
1408 WREG32(CP_RB_WPTR, rdev->cp.wptr);
1409
1410 /* set the wb address wether it's enabled or not */
1411 WREG32(CP_RB_RPTR_ADDR,
1412 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1413 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1414 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1415
1416 if (rdev->wb.enabled)
1417 WREG32(SCRATCH_UMSK, 0xff);
1418 else {
1419 tmp |= RB_NO_UPDATE;
1420 WREG32(SCRATCH_UMSK, 0);
1421 }
1422
1423 mdelay(1);
1424 WREG32(CP_RB_CNTL, tmp);
1425
1426 WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1427 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1428
1429 rdev->cp.rptr = RREG32(CP_RB_RPTR);
1430
1431 evergreen_cp_start(rdev);
1432 rdev->cp.ready = true;
1433 r = radeon_ring_test(rdev);
1434 if (r) {
1435 rdev->cp.ready = false;
1436 return r;
1437 }
1438 return 0;
1439}
1440
1441/*
1442 * Core functions
1443 */
1444static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1445 u32 num_tile_pipes,
1446 u32 num_backends,
1447 u32 backend_disable_mask)
1448{
1449 u32 backend_map = 0;
1450 u32 enabled_backends_mask = 0;
1451 u32 enabled_backends_count = 0;
1452 u32 cur_pipe;
1453 u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
1454 u32 cur_backend = 0;
1455 u32 i;
1456 bool force_no_swizzle;
1457
1458 if (num_tile_pipes > EVERGREEN_MAX_PIPES)
1459 num_tile_pipes = EVERGREEN_MAX_PIPES;
1460 if (num_tile_pipes < 1)
1461 num_tile_pipes = 1;
1462 if (num_backends > EVERGREEN_MAX_BACKENDS)
1463 num_backends = EVERGREEN_MAX_BACKENDS;
1464 if (num_backends < 1)
1465 num_backends = 1;
1466
1467 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1468 if (((backend_disable_mask >> i) & 1) == 0) {
1469 enabled_backends_mask |= (1 << i);
1470 ++enabled_backends_count;
1471 }
1472 if (enabled_backends_count == num_backends)
1473 break;
1474 }
1475
1476 if (enabled_backends_count == 0) {
1477 enabled_backends_mask = 1;
1478 enabled_backends_count = 1;
1479 }
1480
1481 if (enabled_backends_count != num_backends)
1482 num_backends = enabled_backends_count;
1483
1484 memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * EVERGREEN_MAX_PIPES);
1485 switch (rdev->family) {
1486 case CHIP_CEDAR:
1487 case CHIP_REDWOOD:
1488 case CHIP_PALM:
1489 case CHIP_SUMO:
1490 case CHIP_SUMO2:
1491 case CHIP_TURKS:
1492 case CHIP_CAICOS:
1493 force_no_swizzle = false;
1494 break;
1495 case CHIP_CYPRESS:
1496 case CHIP_HEMLOCK:
1497 case CHIP_JUNIPER:
1498 case CHIP_BARTS:
1499 default:
1500 force_no_swizzle = true;
1501 break;
1502 }
1503 if (force_no_swizzle) {
1504 bool last_backend_enabled = false;
1505
1506 force_no_swizzle = false;
1507 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1508 if (((enabled_backends_mask >> i) & 1) == 1) {
1509 if (last_backend_enabled)
1510 force_no_swizzle = true;
1511 last_backend_enabled = true;
1512 } else
1513 last_backend_enabled = false;
1514 }
1515 }
1516
1517 switch (num_tile_pipes) {
1518 case 1:
1519 case 3:
1520 case 5:
1521 case 7:
1522 DRM_ERROR("odd number of pipes!\n");
1523 break;
1524 case 2:
1525 swizzle_pipe[0] = 0;
1526 swizzle_pipe[1] = 1;
1527 break;
1528 case 4:
1529 if (force_no_swizzle) {
1530 swizzle_pipe[0] = 0;
1531 swizzle_pipe[1] = 1;
1532 swizzle_pipe[2] = 2;
1533 swizzle_pipe[3] = 3;
1534 } else {
1535 swizzle_pipe[0] = 0;
1536 swizzle_pipe[1] = 2;
1537 swizzle_pipe[2] = 1;
1538 swizzle_pipe[3] = 3;
1539 }
1540 break;
1541 case 6:
1542 if (force_no_swizzle) {
1543 swizzle_pipe[0] = 0;
1544 swizzle_pipe[1] = 1;
1545 swizzle_pipe[2] = 2;
1546 swizzle_pipe[3] = 3;
1547 swizzle_pipe[4] = 4;
1548 swizzle_pipe[5] = 5;
1549 } else {
1550 swizzle_pipe[0] = 0;
1551 swizzle_pipe[1] = 2;
1552 swizzle_pipe[2] = 4;
1553 swizzle_pipe[3] = 1;
1554 swizzle_pipe[4] = 3;
1555 swizzle_pipe[5] = 5;
1556 }
1557 break;
1558 case 8:
1559 if (force_no_swizzle) {
1560 swizzle_pipe[0] = 0;
1561 swizzle_pipe[1] = 1;
1562 swizzle_pipe[2] = 2;
1563 swizzle_pipe[3] = 3;
1564 swizzle_pipe[4] = 4;
1565 swizzle_pipe[5] = 5;
1566 swizzle_pipe[6] = 6;
1567 swizzle_pipe[7] = 7;
1568 } else {
1569 swizzle_pipe[0] = 0;
1570 swizzle_pipe[1] = 2;
1571 swizzle_pipe[2] = 4;
1572 swizzle_pipe[3] = 6;
1573 swizzle_pipe[4] = 1;
1574 swizzle_pipe[5] = 3;
1575 swizzle_pipe[6] = 5;
1576 swizzle_pipe[7] = 7;
1577 }
1578 break;
1579 }
1580
1581 for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1582 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1583 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1584
1585 backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
1586
1587 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1588 }
1589
1590 return backend_map;
1591}
1592
1593static void evergreen_gpu_init(struct radeon_device *rdev)
1594{
1595 u32 cc_rb_backend_disable = 0;
1596 u32 cc_gc_shader_pipe_config;
1597 u32 gb_addr_config = 0;
1598 u32 mc_shared_chmap, mc_arb_ramcfg;
1599 u32 gb_backend_map;
1600 u32 grbm_gfx_index;
1601 u32 sx_debug_1;
1602 u32 smx_dc_ctl0;
1603 u32 sq_config;
1604 u32 sq_lds_resource_mgmt;
1605 u32 sq_gpr_resource_mgmt_1;
1606 u32 sq_gpr_resource_mgmt_2;
1607 u32 sq_gpr_resource_mgmt_3;
1608 u32 sq_thread_resource_mgmt;
1609 u32 sq_thread_resource_mgmt_2;
1610 u32 sq_stack_resource_mgmt_1;
1611 u32 sq_stack_resource_mgmt_2;
1612 u32 sq_stack_resource_mgmt_3;
1613 u32 vgt_cache_invalidation;
1614 u32 hdp_host_path_cntl, tmp;
1615 int i, j, num_shader_engines, ps_thread_count;
1616
1617 switch (rdev->family) {
1618 case CHIP_CYPRESS:
1619 case CHIP_HEMLOCK:
1620 rdev->config.evergreen.num_ses = 2;
1621 rdev->config.evergreen.max_pipes = 4;
1622 rdev->config.evergreen.max_tile_pipes = 8;
1623 rdev->config.evergreen.max_simds = 10;
1624 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1625 rdev->config.evergreen.max_gprs = 256;
1626 rdev->config.evergreen.max_threads = 248;
1627 rdev->config.evergreen.max_gs_threads = 32;
1628 rdev->config.evergreen.max_stack_entries = 512;
1629 rdev->config.evergreen.sx_num_of_sets = 4;
1630 rdev->config.evergreen.sx_max_export_size = 256;
1631 rdev->config.evergreen.sx_max_export_pos_size = 64;
1632 rdev->config.evergreen.sx_max_export_smx_size = 192;
1633 rdev->config.evergreen.max_hw_contexts = 8;
1634 rdev->config.evergreen.sq_num_cf_insts = 2;
1635
1636 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1637 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1638 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1639 break;
1640 case CHIP_JUNIPER:
1641 rdev->config.evergreen.num_ses = 1;
1642 rdev->config.evergreen.max_pipes = 4;
1643 rdev->config.evergreen.max_tile_pipes = 4;
1644 rdev->config.evergreen.max_simds = 10;
1645 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1646 rdev->config.evergreen.max_gprs = 256;
1647 rdev->config.evergreen.max_threads = 248;
1648 rdev->config.evergreen.max_gs_threads = 32;
1649 rdev->config.evergreen.max_stack_entries = 512;
1650 rdev->config.evergreen.sx_num_of_sets = 4;
1651 rdev->config.evergreen.sx_max_export_size = 256;
1652 rdev->config.evergreen.sx_max_export_pos_size = 64;
1653 rdev->config.evergreen.sx_max_export_smx_size = 192;
1654 rdev->config.evergreen.max_hw_contexts = 8;
1655 rdev->config.evergreen.sq_num_cf_insts = 2;
1656
1657 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1658 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1659 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1660 break;
1661 case CHIP_REDWOOD:
1662 rdev->config.evergreen.num_ses = 1;
1663 rdev->config.evergreen.max_pipes = 4;
1664 rdev->config.evergreen.max_tile_pipes = 4;
1665 rdev->config.evergreen.max_simds = 5;
1666 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1667 rdev->config.evergreen.max_gprs = 256;
1668 rdev->config.evergreen.max_threads = 248;
1669 rdev->config.evergreen.max_gs_threads = 32;
1670 rdev->config.evergreen.max_stack_entries = 256;
1671 rdev->config.evergreen.sx_num_of_sets = 4;
1672 rdev->config.evergreen.sx_max_export_size = 256;
1673 rdev->config.evergreen.sx_max_export_pos_size = 64;
1674 rdev->config.evergreen.sx_max_export_smx_size = 192;
1675 rdev->config.evergreen.max_hw_contexts = 8;
1676 rdev->config.evergreen.sq_num_cf_insts = 2;
1677
1678 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1679 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1680 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1681 break;
1682 case CHIP_CEDAR:
1683 default:
1684 rdev->config.evergreen.num_ses = 1;
1685 rdev->config.evergreen.max_pipes = 2;
1686 rdev->config.evergreen.max_tile_pipes = 2;
1687 rdev->config.evergreen.max_simds = 2;
1688 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1689 rdev->config.evergreen.max_gprs = 256;
1690 rdev->config.evergreen.max_threads = 192;
1691 rdev->config.evergreen.max_gs_threads = 16;
1692 rdev->config.evergreen.max_stack_entries = 256;
1693 rdev->config.evergreen.sx_num_of_sets = 4;
1694 rdev->config.evergreen.sx_max_export_size = 128;
1695 rdev->config.evergreen.sx_max_export_pos_size = 32;
1696 rdev->config.evergreen.sx_max_export_smx_size = 96;
1697 rdev->config.evergreen.max_hw_contexts = 4;
1698 rdev->config.evergreen.sq_num_cf_insts = 1;
1699
1700 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1701 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1702 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1703 break;
1704 case CHIP_PALM:
1705 rdev->config.evergreen.num_ses = 1;
1706 rdev->config.evergreen.max_pipes = 2;
1707 rdev->config.evergreen.max_tile_pipes = 2;
1708 rdev->config.evergreen.max_simds = 2;
1709 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1710 rdev->config.evergreen.max_gprs = 256;
1711 rdev->config.evergreen.max_threads = 192;
1712 rdev->config.evergreen.max_gs_threads = 16;
1713 rdev->config.evergreen.max_stack_entries = 256;
1714 rdev->config.evergreen.sx_num_of_sets = 4;
1715 rdev->config.evergreen.sx_max_export_size = 128;
1716 rdev->config.evergreen.sx_max_export_pos_size = 32;
1717 rdev->config.evergreen.sx_max_export_smx_size = 96;
1718 rdev->config.evergreen.max_hw_contexts = 4;
1719 rdev->config.evergreen.sq_num_cf_insts = 1;
1720
1721 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1722 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1723 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1724 break;
1725 case CHIP_SUMO:
1726 rdev->config.evergreen.num_ses = 1;
1727 rdev->config.evergreen.max_pipes = 4;
1728 rdev->config.evergreen.max_tile_pipes = 2;
1729 if (rdev->pdev->device == 0x9648)
1730 rdev->config.evergreen.max_simds = 3;
1731 else if ((rdev->pdev->device == 0x9647) ||
1732 (rdev->pdev->device == 0x964a))
1733 rdev->config.evergreen.max_simds = 4;
1734 else
1735 rdev->config.evergreen.max_simds = 5;
1736 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1737 rdev->config.evergreen.max_gprs = 256;
1738 rdev->config.evergreen.max_threads = 248;
1739 rdev->config.evergreen.max_gs_threads = 32;
1740 rdev->config.evergreen.max_stack_entries = 256;
1741 rdev->config.evergreen.sx_num_of_sets = 4;
1742 rdev->config.evergreen.sx_max_export_size = 256;
1743 rdev->config.evergreen.sx_max_export_pos_size = 64;
1744 rdev->config.evergreen.sx_max_export_smx_size = 192;
1745 rdev->config.evergreen.max_hw_contexts = 8;
1746 rdev->config.evergreen.sq_num_cf_insts = 2;
1747
1748 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1749 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1750 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1751 break;
1752 case CHIP_SUMO2:
1753 rdev->config.evergreen.num_ses = 1;
1754 rdev->config.evergreen.max_pipes = 4;
1755 rdev->config.evergreen.max_tile_pipes = 4;
1756 rdev->config.evergreen.max_simds = 2;
1757 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1758 rdev->config.evergreen.max_gprs = 256;
1759 rdev->config.evergreen.max_threads = 248;
1760 rdev->config.evergreen.max_gs_threads = 32;
1761 rdev->config.evergreen.max_stack_entries = 512;
1762 rdev->config.evergreen.sx_num_of_sets = 4;
1763 rdev->config.evergreen.sx_max_export_size = 256;
1764 rdev->config.evergreen.sx_max_export_pos_size = 64;
1765 rdev->config.evergreen.sx_max_export_smx_size = 192;
1766 rdev->config.evergreen.max_hw_contexts = 8;
1767 rdev->config.evergreen.sq_num_cf_insts = 2;
1768
1769 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1770 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1771 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1772 break;
1773 case CHIP_BARTS:
1774 rdev->config.evergreen.num_ses = 2;
1775 rdev->config.evergreen.max_pipes = 4;
1776 rdev->config.evergreen.max_tile_pipes = 8;
1777 rdev->config.evergreen.max_simds = 7;
1778 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1779 rdev->config.evergreen.max_gprs = 256;
1780 rdev->config.evergreen.max_threads = 248;
1781 rdev->config.evergreen.max_gs_threads = 32;
1782 rdev->config.evergreen.max_stack_entries = 512;
1783 rdev->config.evergreen.sx_num_of_sets = 4;
1784 rdev->config.evergreen.sx_max_export_size = 256;
1785 rdev->config.evergreen.sx_max_export_pos_size = 64;
1786 rdev->config.evergreen.sx_max_export_smx_size = 192;
1787 rdev->config.evergreen.max_hw_contexts = 8;
1788 rdev->config.evergreen.sq_num_cf_insts = 2;
1789
1790 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1791 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1792 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1793 break;
1794 case CHIP_TURKS:
1795 rdev->config.evergreen.num_ses = 1;
1796 rdev->config.evergreen.max_pipes = 4;
1797 rdev->config.evergreen.max_tile_pipes = 4;
1798 rdev->config.evergreen.max_simds = 6;
1799 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1800 rdev->config.evergreen.max_gprs = 256;
1801 rdev->config.evergreen.max_threads = 248;
1802 rdev->config.evergreen.max_gs_threads = 32;
1803 rdev->config.evergreen.max_stack_entries = 256;
1804 rdev->config.evergreen.sx_num_of_sets = 4;
1805 rdev->config.evergreen.sx_max_export_size = 256;
1806 rdev->config.evergreen.sx_max_export_pos_size = 64;
1807 rdev->config.evergreen.sx_max_export_smx_size = 192;
1808 rdev->config.evergreen.max_hw_contexts = 8;
1809 rdev->config.evergreen.sq_num_cf_insts = 2;
1810
1811 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1812 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1813 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1814 break;
1815 case CHIP_CAICOS:
1816 rdev->config.evergreen.num_ses = 1;
1817 rdev->config.evergreen.max_pipes = 4;
1818 rdev->config.evergreen.max_tile_pipes = 2;
1819 rdev->config.evergreen.max_simds = 2;
1820 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1821 rdev->config.evergreen.max_gprs = 256;
1822 rdev->config.evergreen.max_threads = 192;
1823 rdev->config.evergreen.max_gs_threads = 16;
1824 rdev->config.evergreen.max_stack_entries = 256;
1825 rdev->config.evergreen.sx_num_of_sets = 4;
1826 rdev->config.evergreen.sx_max_export_size = 128;
1827 rdev->config.evergreen.sx_max_export_pos_size = 32;
1828 rdev->config.evergreen.sx_max_export_smx_size = 96;
1829 rdev->config.evergreen.max_hw_contexts = 4;
1830 rdev->config.evergreen.sq_num_cf_insts = 1;
1831
1832 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1833 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1834 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1835 break;
1836 }
1837
1838 /* Initialize HDP */
1839 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1840 WREG32((0x2c14 + j), 0x00000000);
1841 WREG32((0x2c18 + j), 0x00000000);
1842 WREG32((0x2c1c + j), 0x00000000);
1843 WREG32((0x2c20 + j), 0x00000000);
1844 WREG32((0x2c24 + j), 0x00000000);
1845 }
1846
1847 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1848
1849 evergreen_fix_pci_max_read_req_size(rdev);
1850
1851 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
1852
1853 cc_gc_shader_pipe_config |=
1854 INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
1855 & EVERGREEN_MAX_PIPES_MASK);
1856 cc_gc_shader_pipe_config |=
1857 INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
1858 & EVERGREEN_MAX_SIMDS_MASK);
1859
1860 cc_rb_backend_disable =
1861 BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
1862 & EVERGREEN_MAX_BACKENDS_MASK);
1863
1864
1865 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1866 if (rdev->flags & RADEON_IS_IGP)
1867 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
1868 else
1869 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1870
1871 switch (rdev->config.evergreen.max_tile_pipes) {
1872 case 1:
1873 default:
1874 gb_addr_config |= NUM_PIPES(0);
1875 break;
1876 case 2:
1877 gb_addr_config |= NUM_PIPES(1);
1878 break;
1879 case 4:
1880 gb_addr_config |= NUM_PIPES(2);
1881 break;
1882 case 8:
1883 gb_addr_config |= NUM_PIPES(3);
1884 break;
1885 }
1886
1887 gb_addr_config |= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1888 gb_addr_config |= BANK_INTERLEAVE_SIZE(0);
1889 gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
1890 gb_addr_config |= SHADER_ENGINE_TILE_SIZE(1);
1891 gb_addr_config |= NUM_GPUS(0); /* Hemlock? */
1892 gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
1893
1894 if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
1895 gb_addr_config |= ROW_SIZE(2);
1896 else
1897 gb_addr_config |= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
1898
1899 if (rdev->ddev->pdev->device == 0x689e) {
1900 u32 efuse_straps_4;
1901 u32 efuse_straps_3;
1902 u8 efuse_box_bit_131_124;
1903
1904 WREG32(RCU_IND_INDEX, 0x204);
1905 efuse_straps_4 = RREG32(RCU_IND_DATA);
1906 WREG32(RCU_IND_INDEX, 0x203);
1907 efuse_straps_3 = RREG32(RCU_IND_DATA);
1908 efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28));
1909
1910 switch(efuse_box_bit_131_124) {
1911 case 0x00:
1912 gb_backend_map = 0x76543210;
1913 break;
1914 case 0x55:
1915 gb_backend_map = 0x77553311;
1916 break;
1917 case 0x56:
1918 gb_backend_map = 0x77553300;
1919 break;
1920 case 0x59:
1921 gb_backend_map = 0x77552211;
1922 break;
1923 case 0x66:
1924 gb_backend_map = 0x77443300;
1925 break;
1926 case 0x99:
1927 gb_backend_map = 0x66552211;
1928 break;
1929 case 0x5a:
1930 gb_backend_map = 0x77552200;
1931 break;
1932 case 0xaa:
1933 gb_backend_map = 0x66442200;
1934 break;
1935 case 0x95:
1936 gb_backend_map = 0x66553311;
1937 break;
1938 default:
1939 DRM_ERROR("bad backend map, using default\n");
1940 gb_backend_map =
1941 evergreen_get_tile_pipe_to_backend_map(rdev,
1942 rdev->config.evergreen.max_tile_pipes,
1943 rdev->config.evergreen.max_backends,
1944 ((EVERGREEN_MAX_BACKENDS_MASK <<
1945 rdev->config.evergreen.max_backends) &
1946 EVERGREEN_MAX_BACKENDS_MASK));
1947 break;
1948 }
1949 } else if (rdev->ddev->pdev->device == 0x68b9) {
1950 u32 efuse_straps_3;
1951 u8 efuse_box_bit_127_124;
1952
1953 WREG32(RCU_IND_INDEX, 0x203);
1954 efuse_straps_3 = RREG32(RCU_IND_DATA);
1955 efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
1956
1957 switch(efuse_box_bit_127_124) {
1958 case 0x0:
1959 gb_backend_map = 0x00003210;
1960 break;
1961 case 0x5:
1962 case 0x6:
1963 case 0x9:
1964 case 0xa:
1965 gb_backend_map = 0x00003311;
1966 break;
1967 default:
1968 DRM_ERROR("bad backend map, using default\n");
1969 gb_backend_map =
1970 evergreen_get_tile_pipe_to_backend_map(rdev,
1971 rdev->config.evergreen.max_tile_pipes,
1972 rdev->config.evergreen.max_backends,
1973 ((EVERGREEN_MAX_BACKENDS_MASK <<
1974 rdev->config.evergreen.max_backends) &
1975 EVERGREEN_MAX_BACKENDS_MASK));
1976 break;
1977 }
1978 } else {
1979 switch (rdev->family) {
1980 case CHIP_CYPRESS:
1981 case CHIP_HEMLOCK:
1982 case CHIP_BARTS:
1983 gb_backend_map = 0x66442200;
1984 break;
1985 case CHIP_JUNIPER:
1986 gb_backend_map = 0x00002200;
1987 break;
1988 default:
1989 gb_backend_map =
1990 evergreen_get_tile_pipe_to_backend_map(rdev,
1991 rdev->config.evergreen.max_tile_pipes,
1992 rdev->config.evergreen.max_backends,
1993 ((EVERGREEN_MAX_BACKENDS_MASK <<
1994 rdev->config.evergreen.max_backends) &
1995 EVERGREEN_MAX_BACKENDS_MASK));
1996 }
1997 }
1998
1999 /* setup tiling info dword. gb_addr_config is not adequate since it does
2000 * not have bank info, so create a custom tiling dword.
2001 * bits 3:0 num_pipes
2002 * bits 7:4 num_banks
2003 * bits 11:8 group_size
2004 * bits 15:12 row_size
2005 */
2006 rdev->config.evergreen.tile_config = 0;
2007 switch (rdev->config.evergreen.max_tile_pipes) {
2008 case 1:
2009 default:
2010 rdev->config.evergreen.tile_config |= (0 << 0);
2011 break;
2012 case 2:
2013 rdev->config.evergreen.tile_config |= (1 << 0);
2014 break;
2015 case 4:
2016 rdev->config.evergreen.tile_config |= (2 << 0);
2017 break;
2018 case 8:
2019 rdev->config.evergreen.tile_config |= (3 << 0);
2020 break;
2021 }
2022 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
2023 if (rdev->flags & RADEON_IS_IGP)
2024 rdev->config.evergreen.tile_config |= 1 << 4;
2025 else
2026 rdev->config.evergreen.tile_config |=
2027 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
2028 rdev->config.evergreen.tile_config |=
2029 ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
2030 rdev->config.evergreen.tile_config |=
2031 ((gb_addr_config & 0x30000000) >> 28) << 12;
2032
2033 rdev->config.evergreen.backend_map = gb_backend_map;
2034 WREG32(GB_BACKEND_MAP, gb_backend_map);
2035 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2036 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2037 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2038
2039 num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
2040 grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
2041
2042 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
2043 u32 rb = cc_rb_backend_disable | (0xf0 << 16);
2044 u32 sp = cc_gc_shader_pipe_config;
2045 u32 gfx = grbm_gfx_index | SE_INDEX(i);
2046
2047 if (i == num_shader_engines) {
2048 rb |= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
2049 sp |= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
2050 }
2051
2052 WREG32(GRBM_GFX_INDEX, gfx);
2053 WREG32(RLC_GFX_INDEX, gfx);
2054
2055 WREG32(CC_RB_BACKEND_DISABLE, rb);
2056 WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
2057 WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
2058 WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
2059 }
2060
2061 grbm_gfx_index |= SE_BROADCAST_WRITES;
2062 WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
2063 WREG32(RLC_GFX_INDEX, grbm_gfx_index);
2064
2065 WREG32(CGTS_SYS_TCC_DISABLE, 0);
2066 WREG32(CGTS_TCC_DISABLE, 0);
2067 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2068 WREG32(CGTS_USER_TCC_DISABLE, 0);
2069
2070 /* set HW defaults for 3D engine */
2071 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2072 ROQ_IB2_START(0x2b)));
2073
2074 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2075
2076 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2077 SYNC_GRADIENT |
2078 SYNC_WALKER |
2079 SYNC_ALIGNER));
2080
2081 sx_debug_1 = RREG32(SX_DEBUG_1);
2082 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2083 WREG32(SX_DEBUG_1, sx_debug_1);
2084
2085
2086 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2087 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2088 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2089 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2090
2091 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2092 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2093 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2094
2095 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2096 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2097 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2098
2099 WREG32(VGT_NUM_INSTANCES, 1);
2100 WREG32(SPI_CONFIG_CNTL, 0);
2101 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2102 WREG32(CP_PERFMON_CNTL, 0);
2103
2104 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2105 FETCH_FIFO_HIWATER(0x4) |
2106 DONE_FIFO_HIWATER(0xe0) |
2107 ALU_UPDATE_FIFO_HIWATER(0x8)));
2108
2109 sq_config = RREG32(SQ_CONFIG);
2110 sq_config &= ~(PS_PRIO(3) |
2111 VS_PRIO(3) |
2112 GS_PRIO(3) |
2113 ES_PRIO(3));
2114 sq_config |= (VC_ENABLE |
2115 EXPORT_SRC_C |
2116 PS_PRIO(0) |
2117 VS_PRIO(1) |
2118 GS_PRIO(2) |
2119 ES_PRIO(3));
2120
2121 switch (rdev->family) {
2122 case CHIP_CEDAR:
2123 case CHIP_PALM:
2124 case CHIP_SUMO:
2125 case CHIP_SUMO2:
2126 case CHIP_CAICOS:
2127 /* no vertex cache */
2128 sq_config &= ~VC_ENABLE;
2129 break;
2130 default:
2131 break;
2132 }
2133
2134 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2135
2136 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2137 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2138 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2139 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2140 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2141 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2142 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2143
2144 switch (rdev->family) {
2145 case CHIP_CEDAR:
2146 case CHIP_PALM:
2147 case CHIP_SUMO:
2148 case CHIP_SUMO2:
2149 ps_thread_count = 96;
2150 break;
2151 default:
2152 ps_thread_count = 128;
2153 break;
2154 }
2155
2156 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2157 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2158 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2159 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2160 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2161 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2162
2163 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2164 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2165 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2166 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2167 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2168 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2169
2170 WREG32(SQ_CONFIG, sq_config);
2171 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2172 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2173 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2174 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2175 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2176 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2177 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2178 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2179 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2180 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2181
2182 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2183 FORCE_EOV_MAX_REZ_CNT(255)));
2184
2185 switch (rdev->family) {
2186 case CHIP_CEDAR:
2187 case CHIP_PALM:
2188 case CHIP_SUMO:
2189 case CHIP_SUMO2:
2190 case CHIP_CAICOS:
2191 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2192 break;
2193 default:
2194 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2195 break;
2196 }
2197 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2198 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2199
2200 WREG32(VGT_GS_VERTEX_REUSE, 16);
2201 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2202 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2203
2204 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2205 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2206
2207 WREG32(CB_PERF_CTR0_SEL_0, 0);
2208 WREG32(CB_PERF_CTR0_SEL_1, 0);
2209 WREG32(CB_PERF_CTR1_SEL_0, 0);
2210 WREG32(CB_PERF_CTR1_SEL_1, 0);
2211 WREG32(CB_PERF_CTR2_SEL_0, 0);
2212 WREG32(CB_PERF_CTR2_SEL_1, 0);
2213 WREG32(CB_PERF_CTR3_SEL_0, 0);
2214 WREG32(CB_PERF_CTR3_SEL_1, 0);
2215
2216 /* clear render buffer base addresses */
2217 WREG32(CB_COLOR0_BASE, 0);
2218 WREG32(CB_COLOR1_BASE, 0);
2219 WREG32(CB_COLOR2_BASE, 0);
2220 WREG32(CB_COLOR3_BASE, 0);
2221 WREG32(CB_COLOR4_BASE, 0);
2222 WREG32(CB_COLOR5_BASE, 0);
2223 WREG32(CB_COLOR6_BASE, 0);
2224 WREG32(CB_COLOR7_BASE, 0);
2225 WREG32(CB_COLOR8_BASE, 0);
2226 WREG32(CB_COLOR9_BASE, 0);
2227 WREG32(CB_COLOR10_BASE, 0);
2228 WREG32(CB_COLOR11_BASE, 0);
2229
2230 /* set the shader const cache sizes to 0 */
2231 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2232 WREG32(i, 0);
2233 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2234 WREG32(i, 0);
2235
2236 tmp = RREG32(HDP_MISC_CNTL);
2237 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2238 WREG32(HDP_MISC_CNTL, tmp);
2239
2240 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2241 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2242
2243 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2244
2245 udelay(50);
2246
2247}
2248
2249int evergreen_mc_init(struct radeon_device *rdev)
2250{
2251 u32 tmp;
2252 int chansize, numchan;
2253
2254 /* Get VRAM informations */
2255 rdev->mc.vram_is_ddr = true;
2256 if (rdev->flags & RADEON_IS_IGP)
2257 tmp = RREG32(FUS_MC_ARB_RAMCFG);
2258 else
2259 tmp = RREG32(MC_ARB_RAMCFG);
2260 if (tmp & CHANSIZE_OVERRIDE) {
2261 chansize = 16;
2262 } else if (tmp & CHANSIZE_MASK) {
2263 chansize = 64;
2264 } else {
2265 chansize = 32;
2266 }
2267 tmp = RREG32(MC_SHARED_CHMAP);
2268 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2269 case 0:
2270 default:
2271 numchan = 1;
2272 break;
2273 case 1:
2274 numchan = 2;
2275 break;
2276 case 2:
2277 numchan = 4;
2278 break;
2279 case 3:
2280 numchan = 8;
2281 break;
2282 }
2283 rdev->mc.vram_width = numchan * chansize;
2284 /* Could aper size report 0 ? */
2285 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2286 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
2287 /* Setup GPU memory space */
2288 if (rdev->flags & RADEON_IS_IGP) {
2289 /* size in bytes on fusion */
2290 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2291 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2292 } else {
2293 /* size in MB on evergreen */
2294 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2295 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2296 }
2297 rdev->mc.visible_vram_size = rdev->mc.aper_size;
2298 r700_vram_gtt_location(rdev, &rdev->mc);
2299 radeon_update_bandwidth_info(rdev);
2300
2301 return 0;
2302}
2303
2304bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
2305{
2306 u32 srbm_status;
2307 u32 grbm_status;
2308 u32 grbm_status_se0, grbm_status_se1;
2309 struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
2310 int r;
2311
2312 srbm_status = RREG32(SRBM_STATUS);
2313 grbm_status = RREG32(GRBM_STATUS);
2314 grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2315 grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2316 if (!(grbm_status & GUI_ACTIVE)) {
2317 r100_gpu_lockup_update(lockup, &rdev->cp);
2318 return false;
2319 }
2320 /* force CP activities */
2321 r = radeon_ring_lock(rdev, 2);
2322 if (!r) {
2323 /* PACKET2 NOP */
2324 radeon_ring_write(rdev, 0x80000000);
2325 radeon_ring_write(rdev, 0x80000000);
2326 radeon_ring_unlock_commit(rdev);
2327 }
2328 rdev->cp.rptr = RREG32(CP_RB_RPTR);
2329 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
2330}
2331
2332static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
2333{
2334 struct evergreen_mc_save save;
2335 u32 grbm_reset = 0;
2336
2337 if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
2338 return 0;
2339
2340 dev_info(rdev->dev, "GPU softreset \n");
2341 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
2342 RREG32(GRBM_STATUS));
2343 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2344 RREG32(GRBM_STATUS_SE0));
2345 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2346 RREG32(GRBM_STATUS_SE1));
2347 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
2348 RREG32(SRBM_STATUS));
2349 evergreen_mc_stop(rdev, &save);
2350 if (evergreen_mc_wait_for_idle(rdev)) {
2351 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2352 }
2353 /* Disable CP parsing/prefetching */
2354 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2355
2356 /* reset all the gfx blocks */
2357 grbm_reset = (SOFT_RESET_CP |
2358 SOFT_RESET_CB |
2359 SOFT_RESET_DB |
2360 SOFT_RESET_PA |
2361 SOFT_RESET_SC |
2362 SOFT_RESET_SPI |
2363 SOFT_RESET_SH |
2364 SOFT_RESET_SX |
2365 SOFT_RESET_TC |
2366 SOFT_RESET_TA |
2367 SOFT_RESET_VC |
2368 SOFT_RESET_VGT);
2369
2370 dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
2371 WREG32(GRBM_SOFT_RESET, grbm_reset);
2372 (void)RREG32(GRBM_SOFT_RESET);
2373 udelay(50);
2374 WREG32(GRBM_SOFT_RESET, 0);
2375 (void)RREG32(GRBM_SOFT_RESET);
2376 /* Wait a little for things to settle down */
2377 udelay(50);
2378 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
2379 RREG32(GRBM_STATUS));
2380 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2381 RREG32(GRBM_STATUS_SE0));
2382 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2383 RREG32(GRBM_STATUS_SE1));
2384 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
2385 RREG32(SRBM_STATUS));
2386 evergreen_mc_resume(rdev, &save);
2387 return 0;
2388}
2389
2390int evergreen_asic_reset(struct radeon_device *rdev)
2391{
2392 return evergreen_gpu_soft_reset(rdev);
2393}
2394
2395/* Interrupts */
2396
2397u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2398{
2399 switch (crtc) {
2400 case 0:
2401 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
2402 case 1:
2403 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
2404 case 2:
2405 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
2406 case 3:
2407 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
2408 case 4:
2409 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
2410 case 5:
2411 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
2412 default:
2413 return 0;
2414 }
2415}
2416
2417void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2418{
2419 u32 tmp;
2420
2421 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2422 WREG32(GRBM_INT_CNTL, 0);
2423 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2424 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2425 if (rdev->num_crtc >= 4) {
2426 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2427 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2428 }
2429 if (rdev->num_crtc >= 6) {
2430 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2431 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2432 }
2433
2434 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2435 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2436 if (rdev->num_crtc >= 4) {
2437 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2438 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2439 }
2440 if (rdev->num_crtc >= 6) {
2441 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2442 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2443 }
2444
2445 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2446 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2447
2448 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2449 WREG32(DC_HPD1_INT_CONTROL, tmp);
2450 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2451 WREG32(DC_HPD2_INT_CONTROL, tmp);
2452 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2453 WREG32(DC_HPD3_INT_CONTROL, tmp);
2454 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2455 WREG32(DC_HPD4_INT_CONTROL, tmp);
2456 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2457 WREG32(DC_HPD5_INT_CONTROL, tmp);
2458 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2459 WREG32(DC_HPD6_INT_CONTROL, tmp);
2460
2461}
2462
2463int evergreen_irq_set(struct radeon_device *rdev)
2464{
2465 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2466 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2467 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2468 u32 grbm_int_cntl = 0;
2469 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2470
2471 if (!rdev->irq.installed) {
2472 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2473 return -EINVAL;
2474 }
2475 /* don't enable anything if the ih is disabled */
2476 if (!rdev->ih.enabled) {
2477 r600_disable_interrupts(rdev);
2478 /* force the active interrupt state to all disabled */
2479 evergreen_disable_interrupt_state(rdev);
2480 return 0;
2481 }
2482
2483 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2484 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2485 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2486 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2487 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2488 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2489
2490 if (rdev->irq.sw_int) {
2491 DRM_DEBUG("evergreen_irq_set: sw int\n");
2492 cp_int_cntl |= RB_INT_ENABLE;
2493 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2494 }
2495 if (rdev->irq.crtc_vblank_int[0] ||
2496 rdev->irq.pflip[0]) {
2497 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2498 crtc1 |= VBLANK_INT_MASK;
2499 }
2500 if (rdev->irq.crtc_vblank_int[1] ||
2501 rdev->irq.pflip[1]) {
2502 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2503 crtc2 |= VBLANK_INT_MASK;
2504 }
2505 if (rdev->irq.crtc_vblank_int[2] ||
2506 rdev->irq.pflip[2]) {
2507 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2508 crtc3 |= VBLANK_INT_MASK;
2509 }
2510 if (rdev->irq.crtc_vblank_int[3] ||
2511 rdev->irq.pflip[3]) {
2512 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2513 crtc4 |= VBLANK_INT_MASK;
2514 }
2515 if (rdev->irq.crtc_vblank_int[4] ||
2516 rdev->irq.pflip[4]) {
2517 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2518 crtc5 |= VBLANK_INT_MASK;
2519 }
2520 if (rdev->irq.crtc_vblank_int[5] ||
2521 rdev->irq.pflip[5]) {
2522 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2523 crtc6 |= VBLANK_INT_MASK;
2524 }
2525 if (rdev->irq.hpd[0]) {
2526 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2527 hpd1 |= DC_HPDx_INT_EN;
2528 }
2529 if (rdev->irq.hpd[1]) {
2530 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2531 hpd2 |= DC_HPDx_INT_EN;
2532 }
2533 if (rdev->irq.hpd[2]) {
2534 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2535 hpd3 |= DC_HPDx_INT_EN;
2536 }
2537 if (rdev->irq.hpd[3]) {
2538 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2539 hpd4 |= DC_HPDx_INT_EN;
2540 }
2541 if (rdev->irq.hpd[4]) {
2542 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2543 hpd5 |= DC_HPDx_INT_EN;
2544 }
2545 if (rdev->irq.hpd[5]) {
2546 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2547 hpd6 |= DC_HPDx_INT_EN;
2548 }
2549 if (rdev->irq.gui_idle) {
2550 DRM_DEBUG("gui idle\n");
2551 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2552 }
2553
2554 WREG32(CP_INT_CNTL, cp_int_cntl);
2555 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2556
2557 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2558 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2559 if (rdev->num_crtc >= 4) {
2560 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2561 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2562 }
2563 if (rdev->num_crtc >= 6) {
2564 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2565 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2566 }
2567
2568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2570 if (rdev->num_crtc >= 4) {
2571 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2573 }
2574 if (rdev->num_crtc >= 6) {
2575 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2576 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2577 }
2578
2579 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2580 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2581 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2582 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2583 WREG32(DC_HPD5_INT_CONTROL, hpd5);
2584 WREG32(DC_HPD6_INT_CONTROL, hpd6);
2585
2586 return 0;
2587}
2588
2589static inline void evergreen_irq_ack(struct radeon_device *rdev)
2590{
2591 u32 tmp;
2592
2593 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2594 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2595 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2596 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2597 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2598 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2599 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2600 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2601 if (rdev->num_crtc >= 4) {
2602 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2603 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2604 }
2605 if (rdev->num_crtc >= 6) {
2606 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2607 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2608 }
2609
2610 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2611 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2612 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2613 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2614 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2615 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2616 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2617 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2618 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2619 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2620 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2621 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2622
2623 if (rdev->num_crtc >= 4) {
2624 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2625 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2626 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2627 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2628 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2629 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2630 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2631 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2632 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2633 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2634 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2635 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2636 }
2637
2638 if (rdev->num_crtc >= 6) {
2639 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2640 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2641 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2642 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2643 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2644 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2645 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2646 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2647 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2648 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2649 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2650 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2651 }
2652
2653 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2654 tmp = RREG32(DC_HPD1_INT_CONTROL);
2655 tmp |= DC_HPDx_INT_ACK;
2656 WREG32(DC_HPD1_INT_CONTROL, tmp);
2657 }
2658 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2659 tmp = RREG32(DC_HPD2_INT_CONTROL);
2660 tmp |= DC_HPDx_INT_ACK;
2661 WREG32(DC_HPD2_INT_CONTROL, tmp);
2662 }
2663 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2664 tmp = RREG32(DC_HPD3_INT_CONTROL);
2665 tmp |= DC_HPDx_INT_ACK;
2666 WREG32(DC_HPD3_INT_CONTROL, tmp);
2667 }
2668 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2669 tmp = RREG32(DC_HPD4_INT_CONTROL);
2670 tmp |= DC_HPDx_INT_ACK;
2671 WREG32(DC_HPD4_INT_CONTROL, tmp);
2672 }
2673 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2674 tmp = RREG32(DC_HPD5_INT_CONTROL);
2675 tmp |= DC_HPDx_INT_ACK;
2676 WREG32(DC_HPD5_INT_CONTROL, tmp);
2677 }
2678 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2679 tmp = RREG32(DC_HPD5_INT_CONTROL);
2680 tmp |= DC_HPDx_INT_ACK;
2681 WREG32(DC_HPD6_INT_CONTROL, tmp);
2682 }
2683}
2684
2685void evergreen_irq_disable(struct radeon_device *rdev)
2686{
2687 r600_disable_interrupts(rdev);
2688 /* Wait and acknowledge irq */
2689 mdelay(1);
2690 evergreen_irq_ack(rdev);
2691 evergreen_disable_interrupt_state(rdev);
2692}
2693
2694void evergreen_irq_suspend(struct radeon_device *rdev)
2695{
2696 evergreen_irq_disable(rdev);
2697 r600_rlc_stop(rdev);
2698}
2699
2700static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
2701{
2702 u32 wptr, tmp;
2703
2704 if (rdev->wb.enabled)
2705 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
2706 else
2707 wptr = RREG32(IH_RB_WPTR);
2708
2709 if (wptr & RB_OVERFLOW) {
2710 /* When a ring buffer overflow happen start parsing interrupt
2711 * from the last not overwritten vector (wptr + 16). Hopefully
2712 * this should allow us to catchup.
2713 */
2714 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2715 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2716 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2717 tmp = RREG32(IH_RB_CNTL);
2718 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2719 WREG32(IH_RB_CNTL, tmp);
2720 }
2721 return (wptr & rdev->ih.ptr_mask);
2722}
2723
2724int evergreen_irq_process(struct radeon_device *rdev)
2725{
2726 u32 wptr;
2727 u32 rptr;
2728 u32 src_id, src_data;
2729 u32 ring_index;
2730 unsigned long flags;
2731 bool queue_hotplug = false;
2732
2733 if (!rdev->ih.enabled || rdev->shutdown)
2734 return IRQ_NONE;
2735
2736 wptr = evergreen_get_ih_wptr(rdev);
2737 rptr = rdev->ih.rptr;
2738 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2739
2740 spin_lock_irqsave(&rdev->ih.lock, flags);
2741 if (rptr == wptr) {
2742 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2743 return IRQ_NONE;
2744 }
2745restart_ih:
2746 /* Order reading of wptr vs. reading of IH ring data */
2747 rmb();
2748
2749 /* display interrupts */
2750 evergreen_irq_ack(rdev);
2751
2752 rdev->ih.wptr = wptr;
2753 while (rptr != wptr) {
2754 /* wptr/rptr are in bytes! */
2755 ring_index = rptr / 4;
2756 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
2757 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
2758
2759 switch (src_id) {
2760 case 1: /* D1 vblank/vline */
2761 switch (src_data) {
2762 case 0: /* D1 vblank */
2763 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
2764 if (rdev->irq.crtc_vblank_int[0]) {
2765 drm_handle_vblank(rdev->ddev, 0);
2766 rdev->pm.vblank_sync = true;
2767 wake_up(&rdev->irq.vblank_queue);
2768 }
2769 if (rdev->irq.pflip[0])
2770 radeon_crtc_handle_flip(rdev, 0);
2771 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2772 DRM_DEBUG("IH: D1 vblank\n");
2773 }
2774 break;
2775 case 1: /* D1 vline */
2776 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
2777 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
2778 DRM_DEBUG("IH: D1 vline\n");
2779 }
2780 break;
2781 default:
2782 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2783 break;
2784 }
2785 break;
2786 case 2: /* D2 vblank/vline */
2787 switch (src_data) {
2788 case 0: /* D2 vblank */
2789 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
2790 if (rdev->irq.crtc_vblank_int[1]) {
2791 drm_handle_vblank(rdev->ddev, 1);
2792 rdev->pm.vblank_sync = true;
2793 wake_up(&rdev->irq.vblank_queue);
2794 }
2795 if (rdev->irq.pflip[1])
2796 radeon_crtc_handle_flip(rdev, 1);
2797 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
2798 DRM_DEBUG("IH: D2 vblank\n");
2799 }
2800 break;
2801 case 1: /* D2 vline */
2802 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
2803 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
2804 DRM_DEBUG("IH: D2 vline\n");
2805 }
2806 break;
2807 default:
2808 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2809 break;
2810 }
2811 break;
2812 case 3: /* D3 vblank/vline */
2813 switch (src_data) {
2814 case 0: /* D3 vblank */
2815 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
2816 if (rdev->irq.crtc_vblank_int[2]) {
2817 drm_handle_vblank(rdev->ddev, 2);
2818 rdev->pm.vblank_sync = true;
2819 wake_up(&rdev->irq.vblank_queue);
2820 }
2821 if (rdev->irq.pflip[2])
2822 radeon_crtc_handle_flip(rdev, 2);
2823 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
2824 DRM_DEBUG("IH: D3 vblank\n");
2825 }
2826 break;
2827 case 1: /* D3 vline */
2828 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
2829 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
2830 DRM_DEBUG("IH: D3 vline\n");
2831 }
2832 break;
2833 default:
2834 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2835 break;
2836 }
2837 break;
2838 case 4: /* D4 vblank/vline */
2839 switch (src_data) {
2840 case 0: /* D4 vblank */
2841 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
2842 if (rdev->irq.crtc_vblank_int[3]) {
2843 drm_handle_vblank(rdev->ddev, 3);
2844 rdev->pm.vblank_sync = true;
2845 wake_up(&rdev->irq.vblank_queue);
2846 }
2847 if (rdev->irq.pflip[3])
2848 radeon_crtc_handle_flip(rdev, 3);
2849 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
2850 DRM_DEBUG("IH: D4 vblank\n");
2851 }
2852 break;
2853 case 1: /* D4 vline */
2854 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
2855 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
2856 DRM_DEBUG("IH: D4 vline\n");
2857 }
2858 break;
2859 default:
2860 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2861 break;
2862 }
2863 break;
2864 case 5: /* D5 vblank/vline */
2865 switch (src_data) {
2866 case 0: /* D5 vblank */
2867 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
2868 if (rdev->irq.crtc_vblank_int[4]) {
2869 drm_handle_vblank(rdev->ddev, 4);
2870 rdev->pm.vblank_sync = true;
2871 wake_up(&rdev->irq.vblank_queue);
2872 }
2873 if (rdev->irq.pflip[4])
2874 radeon_crtc_handle_flip(rdev, 4);
2875 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
2876 DRM_DEBUG("IH: D5 vblank\n");
2877 }
2878 break;
2879 case 1: /* D5 vline */
2880 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
2881 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
2882 DRM_DEBUG("IH: D5 vline\n");
2883 }
2884 break;
2885 default:
2886 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2887 break;
2888 }
2889 break;
2890 case 6: /* D6 vblank/vline */
2891 switch (src_data) {
2892 case 0: /* D6 vblank */
2893 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
2894 if (rdev->irq.crtc_vblank_int[5]) {
2895 drm_handle_vblank(rdev->ddev, 5);
2896 rdev->pm.vblank_sync = true;
2897 wake_up(&rdev->irq.vblank_queue);
2898 }
2899 if (rdev->irq.pflip[5])
2900 radeon_crtc_handle_flip(rdev, 5);
2901 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
2902 DRM_DEBUG("IH: D6 vblank\n");
2903 }
2904 break;
2905 case 1: /* D6 vline */
2906 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
2907 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
2908 DRM_DEBUG("IH: D6 vline\n");
2909 }
2910 break;
2911 default:
2912 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2913 break;
2914 }
2915 break;
2916 case 42: /* HPD hotplug */
2917 switch (src_data) {
2918 case 0:
2919 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2920 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
2921 queue_hotplug = true;
2922 DRM_DEBUG("IH: HPD1\n");
2923 }
2924 break;
2925 case 1:
2926 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2927 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
2928 queue_hotplug = true;
2929 DRM_DEBUG("IH: HPD2\n");
2930 }
2931 break;
2932 case 2:
2933 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2934 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
2935 queue_hotplug = true;
2936 DRM_DEBUG("IH: HPD3\n");
2937 }
2938 break;
2939 case 3:
2940 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2941 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
2942 queue_hotplug = true;
2943 DRM_DEBUG("IH: HPD4\n");
2944 }
2945 break;
2946 case 4:
2947 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2948 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
2949 queue_hotplug = true;
2950 DRM_DEBUG("IH: HPD5\n");
2951 }
2952 break;
2953 case 5:
2954 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2955 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
2956 queue_hotplug = true;
2957 DRM_DEBUG("IH: HPD6\n");
2958 }
2959 break;
2960 default:
2961 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2962 break;
2963 }
2964 break;
2965 case 176: /* CP_INT in ring buffer */
2966 case 177: /* CP_INT in IB1 */
2967 case 178: /* CP_INT in IB2 */
2968 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2969 radeon_fence_process(rdev);
2970 break;
2971 case 181: /* CP EOP event */
2972 DRM_DEBUG("IH: CP EOP\n");
2973 radeon_fence_process(rdev);
2974 break;
2975 case 233: /* GUI IDLE */
2976 DRM_DEBUG("IH: GUI idle\n");
2977 rdev->pm.gui_idle = true;
2978 wake_up(&rdev->irq.idle_queue);
2979 break;
2980 default:
2981 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2982 break;
2983 }
2984
2985 /* wptr/rptr are in bytes! */
2986 rptr += 16;
2987 rptr &= rdev->ih.ptr_mask;
2988 }
2989 /* make sure wptr hasn't changed while processing */
2990 wptr = evergreen_get_ih_wptr(rdev);
2991 if (wptr != rdev->ih.wptr)
2992 goto restart_ih;
2993 if (queue_hotplug)
2994 schedule_work(&rdev->hotplug_work);
2995 rdev->ih.rptr = rptr;
2996 WREG32(IH_RB_RPTR, rdev->ih.rptr);
2997 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2998 return IRQ_HANDLED;
2999}
3000
3001static int evergreen_startup(struct radeon_device *rdev)
3002{
3003 int r;
3004
3005 /* enable pcie gen2 link */
3006 if (!ASIC_IS_DCE5(rdev))
3007 evergreen_pcie_gen2_enable(rdev);
3008
3009 if (ASIC_IS_DCE5(rdev)) {
3010 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3011 r = ni_init_microcode(rdev);
3012 if (r) {
3013 DRM_ERROR("Failed to load firmware!\n");
3014 return r;
3015 }
3016 }
3017 r = ni_mc_load_microcode(rdev);
3018 if (r) {
3019 DRM_ERROR("Failed to load MC firmware!\n");
3020 return r;
3021 }
3022 } else {
3023 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3024 r = r600_init_microcode(rdev);
3025 if (r) {
3026 DRM_ERROR("Failed to load firmware!\n");
3027 return r;
3028 }
3029 }
3030 }
3031
3032 evergreen_mc_program(rdev);
3033 if (rdev->flags & RADEON_IS_AGP) {
3034 evergreen_agp_enable(rdev);
3035 } else {
3036 r = evergreen_pcie_gart_enable(rdev);
3037 if (r)
3038 return r;
3039 }
3040 evergreen_gpu_init(rdev);
3041
3042 r = evergreen_blit_init(rdev);
3043 if (r) {
3044 evergreen_blit_fini(rdev);
3045 rdev->asic->copy = NULL;
3046 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3047 }
3048
3049 /* allocate wb buffer */
3050 r = radeon_wb_init(rdev);
3051 if (r)
3052 return r;
3053
3054 /* Enable IRQ */
3055 r = r600_irq_init(rdev);
3056 if (r) {
3057 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3058 radeon_irq_kms_fini(rdev);
3059 return r;
3060 }
3061 evergreen_irq_set(rdev);
3062
3063 r = radeon_ring_init(rdev, rdev->cp.ring_size);
3064 if (r)
3065 return r;
3066 r = evergreen_cp_load_microcode(rdev);
3067 if (r)
3068 return r;
3069 r = evergreen_cp_resume(rdev);
3070 if (r)
3071 return r;
3072
3073 return 0;
3074}
3075
3076int evergreen_resume(struct radeon_device *rdev)
3077{
3078 int r;
3079
3080 /* reset the asic, the gfx blocks are often in a bad state
3081 * after the driver is unloaded or after a resume
3082 */
3083 if (radeon_asic_reset(rdev))
3084 dev_warn(rdev->dev, "GPU reset failed !\n");
3085 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3086 * posting will perform necessary task to bring back GPU into good
3087 * shape.
3088 */
3089 /* post card */
3090 atom_asic_init(rdev->mode_info.atom_context);
3091
3092 r = evergreen_startup(rdev);
3093 if (r) {
3094 DRM_ERROR("evergreen startup failed on resume\n");
3095 return r;
3096 }
3097
3098 r = r600_ib_test(rdev);
3099 if (r) {
3100 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3101 return r;
3102 }
3103
3104 return r;
3105
3106}
3107
3108int evergreen_suspend(struct radeon_device *rdev)
3109{
3110 int r;
3111
3112 /* FIXME: we should wait for ring to be empty */
3113 r700_cp_stop(rdev);
3114 rdev->cp.ready = false;
3115 evergreen_irq_suspend(rdev);
3116 radeon_wb_disable(rdev);
3117 evergreen_pcie_gart_disable(rdev);
3118
3119 /* unpin shaders bo */
3120 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
3121 if (likely(r == 0)) {
3122 radeon_bo_unpin(rdev->r600_blit.shader_obj);
3123 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
3124 }
3125
3126 return 0;
3127}
3128
3129int evergreen_copy_blit(struct radeon_device *rdev,
3130 uint64_t src_offset,
3131 uint64_t dst_offset,
3132 unsigned num_gpu_pages,
3133 struct radeon_fence *fence)
3134{
3135 int r;
3136
3137 mutex_lock(&rdev->r600_blit.mutex);
3138 rdev->r600_blit.vb_ib = NULL;
3139 r = evergreen_blit_prepare_copy(rdev, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
3140 if (r) {
3141 if (rdev->r600_blit.vb_ib)
3142 radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
3143 mutex_unlock(&rdev->r600_blit.mutex);
3144 return r;
3145 }
3146 evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
3147 evergreen_blit_done_copy(rdev, fence);
3148 mutex_unlock(&rdev->r600_blit.mutex);
3149 return 0;
3150}
3151
3152/* Plan is to move initialization in that function and use
3153 * helper function so that radeon_device_init pretty much
3154 * do nothing more than calling asic specific function. This
3155 * should also allow to remove a bunch of callback function
3156 * like vram_info.
3157 */
3158int evergreen_init(struct radeon_device *rdev)
3159{
3160 int r;
3161
3162 /* This don't do much */
3163 r = radeon_gem_init(rdev);
3164 if (r)
3165 return r;
3166 /* Read BIOS */
3167 if (!radeon_get_bios(rdev)) {
3168 if (ASIC_IS_AVIVO(rdev))
3169 return -EINVAL;
3170 }
3171 /* Must be an ATOMBIOS */
3172 if (!rdev->is_atom_bios) {
3173 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
3174 return -EINVAL;
3175 }
3176 r = radeon_atombios_init(rdev);
3177 if (r)
3178 return r;
3179 /* reset the asic, the gfx blocks are often in a bad state
3180 * after the driver is unloaded or after a resume
3181 */
3182 if (radeon_asic_reset(rdev))
3183 dev_warn(rdev->dev, "GPU reset failed !\n");
3184 /* Post card if necessary */
3185 if (!radeon_card_posted(rdev)) {
3186 if (!rdev->bios) {
3187 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3188 return -EINVAL;
3189 }
3190 DRM_INFO("GPU not posted. posting now...\n");
3191 atom_asic_init(rdev->mode_info.atom_context);
3192 }
3193 /* Initialize scratch registers */
3194 r600_scratch_init(rdev);
3195 /* Initialize surface registers */
3196 radeon_surface_init(rdev);
3197 /* Initialize clocks */
3198 radeon_get_clock_info(rdev->ddev);
3199 /* Fence driver */
3200 r = radeon_fence_driver_init(rdev);
3201 if (r)
3202 return r;
3203 /* initialize AGP */
3204 if (rdev->flags & RADEON_IS_AGP) {
3205 r = radeon_agp_init(rdev);
3206 if (r)
3207 radeon_agp_disable(rdev);
3208 }
3209 /* initialize memory controller */
3210 r = evergreen_mc_init(rdev);
3211 if (r)
3212 return r;
3213 /* Memory manager */
3214 r = radeon_bo_init(rdev);
3215 if (r)
3216 return r;
3217
3218 r = radeon_irq_kms_init(rdev);
3219 if (r)
3220 return r;
3221
3222 rdev->cp.ring_obj = NULL;
3223 r600_ring_init(rdev, 1024 * 1024);
3224
3225 rdev->ih.ring_obj = NULL;
3226 r600_ih_ring_init(rdev, 64 * 1024);
3227
3228 r = r600_pcie_gart_init(rdev);
3229 if (r)
3230 return r;
3231
3232 rdev->accel_working = true;
3233 r = evergreen_startup(rdev);
3234 if (r) {
3235 dev_err(rdev->dev, "disabling GPU acceleration\n");
3236 r700_cp_fini(rdev);
3237 r600_irq_fini(rdev);
3238 radeon_wb_fini(rdev);
3239 radeon_irq_kms_fini(rdev);
3240 evergreen_pcie_gart_fini(rdev);
3241 rdev->accel_working = false;
3242 }
3243 if (rdev->accel_working) {
3244 r = radeon_ib_pool_init(rdev);
3245 if (r) {
3246 DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
3247 rdev->accel_working = false;
3248 }
3249 r = r600_ib_test(rdev);
3250 if (r) {
3251 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3252 rdev->accel_working = false;
3253 }
3254 }
3255 return 0;
3256}
3257
3258void evergreen_fini(struct radeon_device *rdev)
3259{
3260 evergreen_blit_fini(rdev);
3261 r700_cp_fini(rdev);
3262 r600_irq_fini(rdev);
3263 radeon_wb_fini(rdev);
3264 radeon_ib_pool_fini(rdev);
3265 radeon_irq_kms_fini(rdev);
3266 evergreen_pcie_gart_fini(rdev);
3267 radeon_gem_fini(rdev);
3268 radeon_fence_driver_fini(rdev);
3269 radeon_agp_fini(rdev);
3270 radeon_bo_fini(rdev);
3271 radeon_atombios_fini(rdev);
3272 kfree(rdev->bios);
3273 rdev->bios = NULL;
3274}
3275
3276static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
3277{
3278 u32 link_width_cntl, speed_cntl;
3279
3280 if (radeon_pcie_gen2 == 0)
3281 return;
3282
3283 if (rdev->flags & RADEON_IS_IGP)
3284 return;
3285
3286 if (!(rdev->flags & RADEON_IS_PCIE))
3287 return;
3288
3289 /* x2 cards have a special sequence */
3290 if (ASIC_IS_X2(rdev))
3291 return;
3292
3293 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3294 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3295 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3296
3297 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3298 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3299 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3300
3301 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3302 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3303 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3304
3305 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3306 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3307 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3308
3309 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3310 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3311 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3312
3313 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3314 speed_cntl |= LC_GEN2_EN_STRAP;
3315 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3316
3317 } else {
3318 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3319 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3320 if (1)
3321 link_width_cntl |= LC_UPCONFIGURE_DIS;
3322 else
3323 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3324 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3325 }
3326}
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/slab.h>
26#include <drm/drmP.h>
27#include "radeon.h"
28#include "radeon_asic.h"
29#include "radeon_audio.h"
30#include <drm/radeon_drm.h>
31#include "evergreend.h"
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
35#include "evergreen_blit_shaders.h"
36#include "radeon_ucode.h"
37
38#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
39#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
40#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
41
42/*
43 * Indirect registers accessor
44 */
45u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
46{
47 unsigned long flags;
48 u32 r;
49
50 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
51 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
52 r = RREG32(EVERGREEN_CG_IND_DATA);
53 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
54 return r;
55}
56
57void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
58{
59 unsigned long flags;
60
61 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
62 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
63 WREG32(EVERGREEN_CG_IND_DATA, (v));
64 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
65}
66
67u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
68{
69 unsigned long flags;
70 u32 r;
71
72 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
73 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
74 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
75 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
76 return r;
77}
78
79void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
80{
81 unsigned long flags;
82
83 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
84 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
85 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
86 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
87}
88
89u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
90{
91 unsigned long flags;
92 u32 r;
93
94 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
95 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
96 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
97 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
98 return r;
99}
100
101void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
102{
103 unsigned long flags;
104
105 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
106 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
107 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
108 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
109}
110
111static const u32 crtc_offsets[6] =
112{
113 EVERGREEN_CRTC0_REGISTER_OFFSET,
114 EVERGREEN_CRTC1_REGISTER_OFFSET,
115 EVERGREEN_CRTC2_REGISTER_OFFSET,
116 EVERGREEN_CRTC3_REGISTER_OFFSET,
117 EVERGREEN_CRTC4_REGISTER_OFFSET,
118 EVERGREEN_CRTC5_REGISTER_OFFSET
119};
120
121#include "clearstate_evergreen.h"
122
123static const u32 sumo_rlc_save_restore_register_list[] =
124{
125 0x98fc,
126 0x9830,
127 0x9834,
128 0x9838,
129 0x9870,
130 0x9874,
131 0x8a14,
132 0x8b24,
133 0x8bcc,
134 0x8b10,
135 0x8d00,
136 0x8d04,
137 0x8c00,
138 0x8c04,
139 0x8c08,
140 0x8c0c,
141 0x8d8c,
142 0x8c20,
143 0x8c24,
144 0x8c28,
145 0x8c18,
146 0x8c1c,
147 0x8cf0,
148 0x8e2c,
149 0x8e38,
150 0x8c30,
151 0x9508,
152 0x9688,
153 0x9608,
154 0x960c,
155 0x9610,
156 0x9614,
157 0x88c4,
158 0x88d4,
159 0xa008,
160 0x900c,
161 0x9100,
162 0x913c,
163 0x98f8,
164 0x98f4,
165 0x9b7c,
166 0x3f8c,
167 0x8950,
168 0x8954,
169 0x8a18,
170 0x8b28,
171 0x9144,
172 0x9148,
173 0x914c,
174 0x3f90,
175 0x3f94,
176 0x915c,
177 0x9160,
178 0x9178,
179 0x917c,
180 0x9180,
181 0x918c,
182 0x9190,
183 0x9194,
184 0x9198,
185 0x919c,
186 0x91a8,
187 0x91ac,
188 0x91b0,
189 0x91b4,
190 0x91b8,
191 0x91c4,
192 0x91c8,
193 0x91cc,
194 0x91d0,
195 0x91d4,
196 0x91e0,
197 0x91e4,
198 0x91ec,
199 0x91f0,
200 0x91f4,
201 0x9200,
202 0x9204,
203 0x929c,
204 0x9150,
205 0x802c,
206};
207
208static void evergreen_gpu_init(struct radeon_device *rdev);
209void evergreen_fini(struct radeon_device *rdev);
210void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
211void evergreen_program_aspm(struct radeon_device *rdev);
212extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
213 int ring, u32 cp_int_cntl);
214extern void cayman_vm_decode_fault(struct radeon_device *rdev,
215 u32 status, u32 addr);
216void cik_init_cp_pg_table(struct radeon_device *rdev);
217
218extern u32 si_get_csb_size(struct radeon_device *rdev);
219extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
220extern u32 cik_get_csb_size(struct radeon_device *rdev);
221extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
222extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
223
224static const u32 evergreen_golden_registers[] =
225{
226 0x3f90, 0xffff0000, 0xff000000,
227 0x9148, 0xffff0000, 0xff000000,
228 0x3f94, 0xffff0000, 0xff000000,
229 0x914c, 0xffff0000, 0xff000000,
230 0x9b7c, 0xffffffff, 0x00000000,
231 0x8a14, 0xffffffff, 0x00000007,
232 0x8b10, 0xffffffff, 0x00000000,
233 0x960c, 0xffffffff, 0x54763210,
234 0x88c4, 0xffffffff, 0x000000c2,
235 0x88d4, 0xffffffff, 0x00000010,
236 0x8974, 0xffffffff, 0x00000000,
237 0xc78, 0x00000080, 0x00000080,
238 0x5eb4, 0xffffffff, 0x00000002,
239 0x5e78, 0xffffffff, 0x001000f0,
240 0x6104, 0x01000300, 0x00000000,
241 0x5bc0, 0x00300000, 0x00000000,
242 0x7030, 0xffffffff, 0x00000011,
243 0x7c30, 0xffffffff, 0x00000011,
244 0x10830, 0xffffffff, 0x00000011,
245 0x11430, 0xffffffff, 0x00000011,
246 0x12030, 0xffffffff, 0x00000011,
247 0x12c30, 0xffffffff, 0x00000011,
248 0xd02c, 0xffffffff, 0x08421000,
249 0x240c, 0xffffffff, 0x00000380,
250 0x8b24, 0xffffffff, 0x00ff0fff,
251 0x28a4c, 0x06000000, 0x06000000,
252 0x10c, 0x00000001, 0x00000001,
253 0x8d00, 0xffffffff, 0x100e4848,
254 0x8d04, 0xffffffff, 0x00164745,
255 0x8c00, 0xffffffff, 0xe4000003,
256 0x8c04, 0xffffffff, 0x40600060,
257 0x8c08, 0xffffffff, 0x001c001c,
258 0x8cf0, 0xffffffff, 0x08e00620,
259 0x8c20, 0xffffffff, 0x00800080,
260 0x8c24, 0xffffffff, 0x00800080,
261 0x8c18, 0xffffffff, 0x20202078,
262 0x8c1c, 0xffffffff, 0x00001010,
263 0x28350, 0xffffffff, 0x00000000,
264 0xa008, 0xffffffff, 0x00010000,
265 0x5c4, 0xffffffff, 0x00000001,
266 0x9508, 0xffffffff, 0x00000002,
267 0x913c, 0x0000000f, 0x0000000a
268};
269
270static const u32 evergreen_golden_registers2[] =
271{
272 0x2f4c, 0xffffffff, 0x00000000,
273 0x54f4, 0xffffffff, 0x00000000,
274 0x54f0, 0xffffffff, 0x00000000,
275 0x5498, 0xffffffff, 0x00000000,
276 0x549c, 0xffffffff, 0x00000000,
277 0x5494, 0xffffffff, 0x00000000,
278 0x53cc, 0xffffffff, 0x00000000,
279 0x53c8, 0xffffffff, 0x00000000,
280 0x53c4, 0xffffffff, 0x00000000,
281 0x53c0, 0xffffffff, 0x00000000,
282 0x53bc, 0xffffffff, 0x00000000,
283 0x53b8, 0xffffffff, 0x00000000,
284 0x53b4, 0xffffffff, 0x00000000,
285 0x53b0, 0xffffffff, 0x00000000
286};
287
288static const u32 cypress_mgcg_init[] =
289{
290 0x802c, 0xffffffff, 0xc0000000,
291 0x5448, 0xffffffff, 0x00000100,
292 0x55e4, 0xffffffff, 0x00000100,
293 0x160c, 0xffffffff, 0x00000100,
294 0x5644, 0xffffffff, 0x00000100,
295 0xc164, 0xffffffff, 0x00000100,
296 0x8a18, 0xffffffff, 0x00000100,
297 0x897c, 0xffffffff, 0x06000100,
298 0x8b28, 0xffffffff, 0x00000100,
299 0x9144, 0xffffffff, 0x00000100,
300 0x9a60, 0xffffffff, 0x00000100,
301 0x9868, 0xffffffff, 0x00000100,
302 0x8d58, 0xffffffff, 0x00000100,
303 0x9510, 0xffffffff, 0x00000100,
304 0x949c, 0xffffffff, 0x00000100,
305 0x9654, 0xffffffff, 0x00000100,
306 0x9030, 0xffffffff, 0x00000100,
307 0x9034, 0xffffffff, 0x00000100,
308 0x9038, 0xffffffff, 0x00000100,
309 0x903c, 0xffffffff, 0x00000100,
310 0x9040, 0xffffffff, 0x00000100,
311 0xa200, 0xffffffff, 0x00000100,
312 0xa204, 0xffffffff, 0x00000100,
313 0xa208, 0xffffffff, 0x00000100,
314 0xa20c, 0xffffffff, 0x00000100,
315 0x971c, 0xffffffff, 0x00000100,
316 0x977c, 0xffffffff, 0x00000100,
317 0x3f80, 0xffffffff, 0x00000100,
318 0xa210, 0xffffffff, 0x00000100,
319 0xa214, 0xffffffff, 0x00000100,
320 0x4d8, 0xffffffff, 0x00000100,
321 0x9784, 0xffffffff, 0x00000100,
322 0x9698, 0xffffffff, 0x00000100,
323 0x4d4, 0xffffffff, 0x00000200,
324 0x30cc, 0xffffffff, 0x00000100,
325 0xd0c0, 0xffffffff, 0xff000100,
326 0x802c, 0xffffffff, 0x40000000,
327 0x915c, 0xffffffff, 0x00010000,
328 0x9160, 0xffffffff, 0x00030002,
329 0x9178, 0xffffffff, 0x00070000,
330 0x917c, 0xffffffff, 0x00030002,
331 0x9180, 0xffffffff, 0x00050004,
332 0x918c, 0xffffffff, 0x00010006,
333 0x9190, 0xffffffff, 0x00090008,
334 0x9194, 0xffffffff, 0x00070000,
335 0x9198, 0xffffffff, 0x00030002,
336 0x919c, 0xffffffff, 0x00050004,
337 0x91a8, 0xffffffff, 0x00010006,
338 0x91ac, 0xffffffff, 0x00090008,
339 0x91b0, 0xffffffff, 0x00070000,
340 0x91b4, 0xffffffff, 0x00030002,
341 0x91b8, 0xffffffff, 0x00050004,
342 0x91c4, 0xffffffff, 0x00010006,
343 0x91c8, 0xffffffff, 0x00090008,
344 0x91cc, 0xffffffff, 0x00070000,
345 0x91d0, 0xffffffff, 0x00030002,
346 0x91d4, 0xffffffff, 0x00050004,
347 0x91e0, 0xffffffff, 0x00010006,
348 0x91e4, 0xffffffff, 0x00090008,
349 0x91e8, 0xffffffff, 0x00000000,
350 0x91ec, 0xffffffff, 0x00070000,
351 0x91f0, 0xffffffff, 0x00030002,
352 0x91f4, 0xffffffff, 0x00050004,
353 0x9200, 0xffffffff, 0x00010006,
354 0x9204, 0xffffffff, 0x00090008,
355 0x9208, 0xffffffff, 0x00070000,
356 0x920c, 0xffffffff, 0x00030002,
357 0x9210, 0xffffffff, 0x00050004,
358 0x921c, 0xffffffff, 0x00010006,
359 0x9220, 0xffffffff, 0x00090008,
360 0x9224, 0xffffffff, 0x00070000,
361 0x9228, 0xffffffff, 0x00030002,
362 0x922c, 0xffffffff, 0x00050004,
363 0x9238, 0xffffffff, 0x00010006,
364 0x923c, 0xffffffff, 0x00090008,
365 0x9240, 0xffffffff, 0x00070000,
366 0x9244, 0xffffffff, 0x00030002,
367 0x9248, 0xffffffff, 0x00050004,
368 0x9254, 0xffffffff, 0x00010006,
369 0x9258, 0xffffffff, 0x00090008,
370 0x925c, 0xffffffff, 0x00070000,
371 0x9260, 0xffffffff, 0x00030002,
372 0x9264, 0xffffffff, 0x00050004,
373 0x9270, 0xffffffff, 0x00010006,
374 0x9274, 0xffffffff, 0x00090008,
375 0x9278, 0xffffffff, 0x00070000,
376 0x927c, 0xffffffff, 0x00030002,
377 0x9280, 0xffffffff, 0x00050004,
378 0x928c, 0xffffffff, 0x00010006,
379 0x9290, 0xffffffff, 0x00090008,
380 0x9294, 0xffffffff, 0x00000000,
381 0x929c, 0xffffffff, 0x00000001,
382 0x802c, 0xffffffff, 0x40010000,
383 0x915c, 0xffffffff, 0x00010000,
384 0x9160, 0xffffffff, 0x00030002,
385 0x9178, 0xffffffff, 0x00070000,
386 0x917c, 0xffffffff, 0x00030002,
387 0x9180, 0xffffffff, 0x00050004,
388 0x918c, 0xffffffff, 0x00010006,
389 0x9190, 0xffffffff, 0x00090008,
390 0x9194, 0xffffffff, 0x00070000,
391 0x9198, 0xffffffff, 0x00030002,
392 0x919c, 0xffffffff, 0x00050004,
393 0x91a8, 0xffffffff, 0x00010006,
394 0x91ac, 0xffffffff, 0x00090008,
395 0x91b0, 0xffffffff, 0x00070000,
396 0x91b4, 0xffffffff, 0x00030002,
397 0x91b8, 0xffffffff, 0x00050004,
398 0x91c4, 0xffffffff, 0x00010006,
399 0x91c8, 0xffffffff, 0x00090008,
400 0x91cc, 0xffffffff, 0x00070000,
401 0x91d0, 0xffffffff, 0x00030002,
402 0x91d4, 0xffffffff, 0x00050004,
403 0x91e0, 0xffffffff, 0x00010006,
404 0x91e4, 0xffffffff, 0x00090008,
405 0x91e8, 0xffffffff, 0x00000000,
406 0x91ec, 0xffffffff, 0x00070000,
407 0x91f0, 0xffffffff, 0x00030002,
408 0x91f4, 0xffffffff, 0x00050004,
409 0x9200, 0xffffffff, 0x00010006,
410 0x9204, 0xffffffff, 0x00090008,
411 0x9208, 0xffffffff, 0x00070000,
412 0x920c, 0xffffffff, 0x00030002,
413 0x9210, 0xffffffff, 0x00050004,
414 0x921c, 0xffffffff, 0x00010006,
415 0x9220, 0xffffffff, 0x00090008,
416 0x9224, 0xffffffff, 0x00070000,
417 0x9228, 0xffffffff, 0x00030002,
418 0x922c, 0xffffffff, 0x00050004,
419 0x9238, 0xffffffff, 0x00010006,
420 0x923c, 0xffffffff, 0x00090008,
421 0x9240, 0xffffffff, 0x00070000,
422 0x9244, 0xffffffff, 0x00030002,
423 0x9248, 0xffffffff, 0x00050004,
424 0x9254, 0xffffffff, 0x00010006,
425 0x9258, 0xffffffff, 0x00090008,
426 0x925c, 0xffffffff, 0x00070000,
427 0x9260, 0xffffffff, 0x00030002,
428 0x9264, 0xffffffff, 0x00050004,
429 0x9270, 0xffffffff, 0x00010006,
430 0x9274, 0xffffffff, 0x00090008,
431 0x9278, 0xffffffff, 0x00070000,
432 0x927c, 0xffffffff, 0x00030002,
433 0x9280, 0xffffffff, 0x00050004,
434 0x928c, 0xffffffff, 0x00010006,
435 0x9290, 0xffffffff, 0x00090008,
436 0x9294, 0xffffffff, 0x00000000,
437 0x929c, 0xffffffff, 0x00000001,
438 0x802c, 0xffffffff, 0xc0000000
439};
440
441static const u32 redwood_mgcg_init[] =
442{
443 0x802c, 0xffffffff, 0xc0000000,
444 0x5448, 0xffffffff, 0x00000100,
445 0x55e4, 0xffffffff, 0x00000100,
446 0x160c, 0xffffffff, 0x00000100,
447 0x5644, 0xffffffff, 0x00000100,
448 0xc164, 0xffffffff, 0x00000100,
449 0x8a18, 0xffffffff, 0x00000100,
450 0x897c, 0xffffffff, 0x06000100,
451 0x8b28, 0xffffffff, 0x00000100,
452 0x9144, 0xffffffff, 0x00000100,
453 0x9a60, 0xffffffff, 0x00000100,
454 0x9868, 0xffffffff, 0x00000100,
455 0x8d58, 0xffffffff, 0x00000100,
456 0x9510, 0xffffffff, 0x00000100,
457 0x949c, 0xffffffff, 0x00000100,
458 0x9654, 0xffffffff, 0x00000100,
459 0x9030, 0xffffffff, 0x00000100,
460 0x9034, 0xffffffff, 0x00000100,
461 0x9038, 0xffffffff, 0x00000100,
462 0x903c, 0xffffffff, 0x00000100,
463 0x9040, 0xffffffff, 0x00000100,
464 0xa200, 0xffffffff, 0x00000100,
465 0xa204, 0xffffffff, 0x00000100,
466 0xa208, 0xffffffff, 0x00000100,
467 0xa20c, 0xffffffff, 0x00000100,
468 0x971c, 0xffffffff, 0x00000100,
469 0x977c, 0xffffffff, 0x00000100,
470 0x3f80, 0xffffffff, 0x00000100,
471 0xa210, 0xffffffff, 0x00000100,
472 0xa214, 0xffffffff, 0x00000100,
473 0x4d8, 0xffffffff, 0x00000100,
474 0x9784, 0xffffffff, 0x00000100,
475 0x9698, 0xffffffff, 0x00000100,
476 0x4d4, 0xffffffff, 0x00000200,
477 0x30cc, 0xffffffff, 0x00000100,
478 0xd0c0, 0xffffffff, 0xff000100,
479 0x802c, 0xffffffff, 0x40000000,
480 0x915c, 0xffffffff, 0x00010000,
481 0x9160, 0xffffffff, 0x00030002,
482 0x9178, 0xffffffff, 0x00070000,
483 0x917c, 0xffffffff, 0x00030002,
484 0x9180, 0xffffffff, 0x00050004,
485 0x918c, 0xffffffff, 0x00010006,
486 0x9190, 0xffffffff, 0x00090008,
487 0x9194, 0xffffffff, 0x00070000,
488 0x9198, 0xffffffff, 0x00030002,
489 0x919c, 0xffffffff, 0x00050004,
490 0x91a8, 0xffffffff, 0x00010006,
491 0x91ac, 0xffffffff, 0x00090008,
492 0x91b0, 0xffffffff, 0x00070000,
493 0x91b4, 0xffffffff, 0x00030002,
494 0x91b8, 0xffffffff, 0x00050004,
495 0x91c4, 0xffffffff, 0x00010006,
496 0x91c8, 0xffffffff, 0x00090008,
497 0x91cc, 0xffffffff, 0x00070000,
498 0x91d0, 0xffffffff, 0x00030002,
499 0x91d4, 0xffffffff, 0x00050004,
500 0x91e0, 0xffffffff, 0x00010006,
501 0x91e4, 0xffffffff, 0x00090008,
502 0x91e8, 0xffffffff, 0x00000000,
503 0x91ec, 0xffffffff, 0x00070000,
504 0x91f0, 0xffffffff, 0x00030002,
505 0x91f4, 0xffffffff, 0x00050004,
506 0x9200, 0xffffffff, 0x00010006,
507 0x9204, 0xffffffff, 0x00090008,
508 0x9294, 0xffffffff, 0x00000000,
509 0x929c, 0xffffffff, 0x00000001,
510 0x802c, 0xffffffff, 0xc0000000
511};
512
513static const u32 cedar_golden_registers[] =
514{
515 0x3f90, 0xffff0000, 0xff000000,
516 0x9148, 0xffff0000, 0xff000000,
517 0x3f94, 0xffff0000, 0xff000000,
518 0x914c, 0xffff0000, 0xff000000,
519 0x9b7c, 0xffffffff, 0x00000000,
520 0x8a14, 0xffffffff, 0x00000007,
521 0x8b10, 0xffffffff, 0x00000000,
522 0x960c, 0xffffffff, 0x54763210,
523 0x88c4, 0xffffffff, 0x000000c2,
524 0x88d4, 0xffffffff, 0x00000000,
525 0x8974, 0xffffffff, 0x00000000,
526 0xc78, 0x00000080, 0x00000080,
527 0x5eb4, 0xffffffff, 0x00000002,
528 0x5e78, 0xffffffff, 0x001000f0,
529 0x6104, 0x01000300, 0x00000000,
530 0x5bc0, 0x00300000, 0x00000000,
531 0x7030, 0xffffffff, 0x00000011,
532 0x7c30, 0xffffffff, 0x00000011,
533 0x10830, 0xffffffff, 0x00000011,
534 0x11430, 0xffffffff, 0x00000011,
535 0xd02c, 0xffffffff, 0x08421000,
536 0x240c, 0xffffffff, 0x00000380,
537 0x8b24, 0xffffffff, 0x00ff0fff,
538 0x28a4c, 0x06000000, 0x06000000,
539 0x10c, 0x00000001, 0x00000001,
540 0x8d00, 0xffffffff, 0x100e4848,
541 0x8d04, 0xffffffff, 0x00164745,
542 0x8c00, 0xffffffff, 0xe4000003,
543 0x8c04, 0xffffffff, 0x40600060,
544 0x8c08, 0xffffffff, 0x001c001c,
545 0x8cf0, 0xffffffff, 0x08e00410,
546 0x8c20, 0xffffffff, 0x00800080,
547 0x8c24, 0xffffffff, 0x00800080,
548 0x8c18, 0xffffffff, 0x20202078,
549 0x8c1c, 0xffffffff, 0x00001010,
550 0x28350, 0xffffffff, 0x00000000,
551 0xa008, 0xffffffff, 0x00010000,
552 0x5c4, 0xffffffff, 0x00000001,
553 0x9508, 0xffffffff, 0x00000002
554};
555
556static const u32 cedar_mgcg_init[] =
557{
558 0x802c, 0xffffffff, 0xc0000000,
559 0x5448, 0xffffffff, 0x00000100,
560 0x55e4, 0xffffffff, 0x00000100,
561 0x160c, 0xffffffff, 0x00000100,
562 0x5644, 0xffffffff, 0x00000100,
563 0xc164, 0xffffffff, 0x00000100,
564 0x8a18, 0xffffffff, 0x00000100,
565 0x897c, 0xffffffff, 0x06000100,
566 0x8b28, 0xffffffff, 0x00000100,
567 0x9144, 0xffffffff, 0x00000100,
568 0x9a60, 0xffffffff, 0x00000100,
569 0x9868, 0xffffffff, 0x00000100,
570 0x8d58, 0xffffffff, 0x00000100,
571 0x9510, 0xffffffff, 0x00000100,
572 0x949c, 0xffffffff, 0x00000100,
573 0x9654, 0xffffffff, 0x00000100,
574 0x9030, 0xffffffff, 0x00000100,
575 0x9034, 0xffffffff, 0x00000100,
576 0x9038, 0xffffffff, 0x00000100,
577 0x903c, 0xffffffff, 0x00000100,
578 0x9040, 0xffffffff, 0x00000100,
579 0xa200, 0xffffffff, 0x00000100,
580 0xa204, 0xffffffff, 0x00000100,
581 0xa208, 0xffffffff, 0x00000100,
582 0xa20c, 0xffffffff, 0x00000100,
583 0x971c, 0xffffffff, 0x00000100,
584 0x977c, 0xffffffff, 0x00000100,
585 0x3f80, 0xffffffff, 0x00000100,
586 0xa210, 0xffffffff, 0x00000100,
587 0xa214, 0xffffffff, 0x00000100,
588 0x4d8, 0xffffffff, 0x00000100,
589 0x9784, 0xffffffff, 0x00000100,
590 0x9698, 0xffffffff, 0x00000100,
591 0x4d4, 0xffffffff, 0x00000200,
592 0x30cc, 0xffffffff, 0x00000100,
593 0xd0c0, 0xffffffff, 0xff000100,
594 0x802c, 0xffffffff, 0x40000000,
595 0x915c, 0xffffffff, 0x00010000,
596 0x9178, 0xffffffff, 0x00050000,
597 0x917c, 0xffffffff, 0x00030002,
598 0x918c, 0xffffffff, 0x00010004,
599 0x9190, 0xffffffff, 0x00070006,
600 0x9194, 0xffffffff, 0x00050000,
601 0x9198, 0xffffffff, 0x00030002,
602 0x91a8, 0xffffffff, 0x00010004,
603 0x91ac, 0xffffffff, 0x00070006,
604 0x91e8, 0xffffffff, 0x00000000,
605 0x9294, 0xffffffff, 0x00000000,
606 0x929c, 0xffffffff, 0x00000001,
607 0x802c, 0xffffffff, 0xc0000000
608};
609
610static const u32 juniper_mgcg_init[] =
611{
612 0x802c, 0xffffffff, 0xc0000000,
613 0x5448, 0xffffffff, 0x00000100,
614 0x55e4, 0xffffffff, 0x00000100,
615 0x160c, 0xffffffff, 0x00000100,
616 0x5644, 0xffffffff, 0x00000100,
617 0xc164, 0xffffffff, 0x00000100,
618 0x8a18, 0xffffffff, 0x00000100,
619 0x897c, 0xffffffff, 0x06000100,
620 0x8b28, 0xffffffff, 0x00000100,
621 0x9144, 0xffffffff, 0x00000100,
622 0x9a60, 0xffffffff, 0x00000100,
623 0x9868, 0xffffffff, 0x00000100,
624 0x8d58, 0xffffffff, 0x00000100,
625 0x9510, 0xffffffff, 0x00000100,
626 0x949c, 0xffffffff, 0x00000100,
627 0x9654, 0xffffffff, 0x00000100,
628 0x9030, 0xffffffff, 0x00000100,
629 0x9034, 0xffffffff, 0x00000100,
630 0x9038, 0xffffffff, 0x00000100,
631 0x903c, 0xffffffff, 0x00000100,
632 0x9040, 0xffffffff, 0x00000100,
633 0xa200, 0xffffffff, 0x00000100,
634 0xa204, 0xffffffff, 0x00000100,
635 0xa208, 0xffffffff, 0x00000100,
636 0xa20c, 0xffffffff, 0x00000100,
637 0x971c, 0xffffffff, 0x00000100,
638 0xd0c0, 0xffffffff, 0xff000100,
639 0x802c, 0xffffffff, 0x40000000,
640 0x915c, 0xffffffff, 0x00010000,
641 0x9160, 0xffffffff, 0x00030002,
642 0x9178, 0xffffffff, 0x00070000,
643 0x917c, 0xffffffff, 0x00030002,
644 0x9180, 0xffffffff, 0x00050004,
645 0x918c, 0xffffffff, 0x00010006,
646 0x9190, 0xffffffff, 0x00090008,
647 0x9194, 0xffffffff, 0x00070000,
648 0x9198, 0xffffffff, 0x00030002,
649 0x919c, 0xffffffff, 0x00050004,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91ac, 0xffffffff, 0x00090008,
652 0x91b0, 0xffffffff, 0x00070000,
653 0x91b4, 0xffffffff, 0x00030002,
654 0x91b8, 0xffffffff, 0x00050004,
655 0x91c4, 0xffffffff, 0x00010006,
656 0x91c8, 0xffffffff, 0x00090008,
657 0x91cc, 0xffffffff, 0x00070000,
658 0x91d0, 0xffffffff, 0x00030002,
659 0x91d4, 0xffffffff, 0x00050004,
660 0x91e0, 0xffffffff, 0x00010006,
661 0x91e4, 0xffffffff, 0x00090008,
662 0x91e8, 0xffffffff, 0x00000000,
663 0x91ec, 0xffffffff, 0x00070000,
664 0x91f0, 0xffffffff, 0x00030002,
665 0x91f4, 0xffffffff, 0x00050004,
666 0x9200, 0xffffffff, 0x00010006,
667 0x9204, 0xffffffff, 0x00090008,
668 0x9208, 0xffffffff, 0x00070000,
669 0x920c, 0xffffffff, 0x00030002,
670 0x9210, 0xffffffff, 0x00050004,
671 0x921c, 0xffffffff, 0x00010006,
672 0x9220, 0xffffffff, 0x00090008,
673 0x9224, 0xffffffff, 0x00070000,
674 0x9228, 0xffffffff, 0x00030002,
675 0x922c, 0xffffffff, 0x00050004,
676 0x9238, 0xffffffff, 0x00010006,
677 0x923c, 0xffffffff, 0x00090008,
678 0x9240, 0xffffffff, 0x00070000,
679 0x9244, 0xffffffff, 0x00030002,
680 0x9248, 0xffffffff, 0x00050004,
681 0x9254, 0xffffffff, 0x00010006,
682 0x9258, 0xffffffff, 0x00090008,
683 0x925c, 0xffffffff, 0x00070000,
684 0x9260, 0xffffffff, 0x00030002,
685 0x9264, 0xffffffff, 0x00050004,
686 0x9270, 0xffffffff, 0x00010006,
687 0x9274, 0xffffffff, 0x00090008,
688 0x9278, 0xffffffff, 0x00070000,
689 0x927c, 0xffffffff, 0x00030002,
690 0x9280, 0xffffffff, 0x00050004,
691 0x928c, 0xffffffff, 0x00010006,
692 0x9290, 0xffffffff, 0x00090008,
693 0x9294, 0xffffffff, 0x00000000,
694 0x929c, 0xffffffff, 0x00000001,
695 0x802c, 0xffffffff, 0xc0000000,
696 0x977c, 0xffffffff, 0x00000100,
697 0x3f80, 0xffffffff, 0x00000100,
698 0xa210, 0xffffffff, 0x00000100,
699 0xa214, 0xffffffff, 0x00000100,
700 0x4d8, 0xffffffff, 0x00000100,
701 0x9784, 0xffffffff, 0x00000100,
702 0x9698, 0xffffffff, 0x00000100,
703 0x4d4, 0xffffffff, 0x00000200,
704 0x30cc, 0xffffffff, 0x00000100,
705 0x802c, 0xffffffff, 0xc0000000
706};
707
708static const u32 supersumo_golden_registers[] =
709{
710 0x5eb4, 0xffffffff, 0x00000002,
711 0x5c4, 0xffffffff, 0x00000001,
712 0x7030, 0xffffffff, 0x00000011,
713 0x7c30, 0xffffffff, 0x00000011,
714 0x6104, 0x01000300, 0x00000000,
715 0x5bc0, 0x00300000, 0x00000000,
716 0x8c04, 0xffffffff, 0x40600060,
717 0x8c08, 0xffffffff, 0x001c001c,
718 0x8c20, 0xffffffff, 0x00800080,
719 0x8c24, 0xffffffff, 0x00800080,
720 0x8c18, 0xffffffff, 0x20202078,
721 0x8c1c, 0xffffffff, 0x00001010,
722 0x918c, 0xffffffff, 0x00010006,
723 0x91a8, 0xffffffff, 0x00010006,
724 0x91c4, 0xffffffff, 0x00010006,
725 0x91e0, 0xffffffff, 0x00010006,
726 0x9200, 0xffffffff, 0x00010006,
727 0x9150, 0xffffffff, 0x6e944040,
728 0x917c, 0xffffffff, 0x00030002,
729 0x9180, 0xffffffff, 0x00050004,
730 0x9198, 0xffffffff, 0x00030002,
731 0x919c, 0xffffffff, 0x00050004,
732 0x91b4, 0xffffffff, 0x00030002,
733 0x91b8, 0xffffffff, 0x00050004,
734 0x91d0, 0xffffffff, 0x00030002,
735 0x91d4, 0xffffffff, 0x00050004,
736 0x91f0, 0xffffffff, 0x00030002,
737 0x91f4, 0xffffffff, 0x00050004,
738 0x915c, 0xffffffff, 0x00010000,
739 0x9160, 0xffffffff, 0x00030002,
740 0x3f90, 0xffff0000, 0xff000000,
741 0x9178, 0xffffffff, 0x00070000,
742 0x9194, 0xffffffff, 0x00070000,
743 0x91b0, 0xffffffff, 0x00070000,
744 0x91cc, 0xffffffff, 0x00070000,
745 0x91ec, 0xffffffff, 0x00070000,
746 0x9148, 0xffff0000, 0xff000000,
747 0x9190, 0xffffffff, 0x00090008,
748 0x91ac, 0xffffffff, 0x00090008,
749 0x91c8, 0xffffffff, 0x00090008,
750 0x91e4, 0xffffffff, 0x00090008,
751 0x9204, 0xffffffff, 0x00090008,
752 0x3f94, 0xffff0000, 0xff000000,
753 0x914c, 0xffff0000, 0xff000000,
754 0x929c, 0xffffffff, 0x00000001,
755 0x8a18, 0xffffffff, 0x00000100,
756 0x8b28, 0xffffffff, 0x00000100,
757 0x9144, 0xffffffff, 0x00000100,
758 0x5644, 0xffffffff, 0x00000100,
759 0x9b7c, 0xffffffff, 0x00000000,
760 0x8030, 0xffffffff, 0x0000100a,
761 0x8a14, 0xffffffff, 0x00000007,
762 0x8b24, 0xffffffff, 0x00ff0fff,
763 0x8b10, 0xffffffff, 0x00000000,
764 0x28a4c, 0x06000000, 0x06000000,
765 0x4d8, 0xffffffff, 0x00000100,
766 0x913c, 0xffff000f, 0x0100000a,
767 0x960c, 0xffffffff, 0x54763210,
768 0x88c4, 0xffffffff, 0x000000c2,
769 0x88d4, 0xffffffff, 0x00000010,
770 0x8974, 0xffffffff, 0x00000000,
771 0xc78, 0x00000080, 0x00000080,
772 0x5e78, 0xffffffff, 0x001000f0,
773 0xd02c, 0xffffffff, 0x08421000,
774 0xa008, 0xffffffff, 0x00010000,
775 0x8d00, 0xffffffff, 0x100e4848,
776 0x8d04, 0xffffffff, 0x00164745,
777 0x8c00, 0xffffffff, 0xe4000003,
778 0x8cf0, 0x1fffffff, 0x08e00620,
779 0x28350, 0xffffffff, 0x00000000,
780 0x9508, 0xffffffff, 0x00000002
781};
782
783static const u32 sumo_golden_registers[] =
784{
785 0x900c, 0x00ffffff, 0x0017071f,
786 0x8c18, 0xffffffff, 0x10101060,
787 0x8c1c, 0xffffffff, 0x00001010,
788 0x8c30, 0x0000000f, 0x00000005,
789 0x9688, 0x0000000f, 0x00000007
790};
791
792static const u32 wrestler_golden_registers[] =
793{
794 0x5eb4, 0xffffffff, 0x00000002,
795 0x5c4, 0xffffffff, 0x00000001,
796 0x7030, 0xffffffff, 0x00000011,
797 0x7c30, 0xffffffff, 0x00000011,
798 0x6104, 0x01000300, 0x00000000,
799 0x5bc0, 0x00300000, 0x00000000,
800 0x918c, 0xffffffff, 0x00010006,
801 0x91a8, 0xffffffff, 0x00010006,
802 0x9150, 0xffffffff, 0x6e944040,
803 0x917c, 0xffffffff, 0x00030002,
804 0x9198, 0xffffffff, 0x00030002,
805 0x915c, 0xffffffff, 0x00010000,
806 0x3f90, 0xffff0000, 0xff000000,
807 0x9178, 0xffffffff, 0x00070000,
808 0x9194, 0xffffffff, 0x00070000,
809 0x9148, 0xffff0000, 0xff000000,
810 0x9190, 0xffffffff, 0x00090008,
811 0x91ac, 0xffffffff, 0x00090008,
812 0x3f94, 0xffff0000, 0xff000000,
813 0x914c, 0xffff0000, 0xff000000,
814 0x929c, 0xffffffff, 0x00000001,
815 0x8a18, 0xffffffff, 0x00000100,
816 0x8b28, 0xffffffff, 0x00000100,
817 0x9144, 0xffffffff, 0x00000100,
818 0x9b7c, 0xffffffff, 0x00000000,
819 0x8030, 0xffffffff, 0x0000100a,
820 0x8a14, 0xffffffff, 0x00000001,
821 0x8b24, 0xffffffff, 0x00ff0fff,
822 0x8b10, 0xffffffff, 0x00000000,
823 0x28a4c, 0x06000000, 0x06000000,
824 0x4d8, 0xffffffff, 0x00000100,
825 0x913c, 0xffff000f, 0x0100000a,
826 0x960c, 0xffffffff, 0x54763210,
827 0x88c4, 0xffffffff, 0x000000c2,
828 0x88d4, 0xffffffff, 0x00000010,
829 0x8974, 0xffffffff, 0x00000000,
830 0xc78, 0x00000080, 0x00000080,
831 0x5e78, 0xffffffff, 0x001000f0,
832 0xd02c, 0xffffffff, 0x08421000,
833 0xa008, 0xffffffff, 0x00010000,
834 0x8d00, 0xffffffff, 0x100e4848,
835 0x8d04, 0xffffffff, 0x00164745,
836 0x8c00, 0xffffffff, 0xe4000003,
837 0x8cf0, 0x1fffffff, 0x08e00410,
838 0x28350, 0xffffffff, 0x00000000,
839 0x9508, 0xffffffff, 0x00000002,
840 0x900c, 0xffffffff, 0x0017071f,
841 0x8c18, 0xffffffff, 0x10101060,
842 0x8c1c, 0xffffffff, 0x00001010
843};
844
845static const u32 barts_golden_registers[] =
846{
847 0x5eb4, 0xffffffff, 0x00000002,
848 0x5e78, 0x8f311ff1, 0x001000f0,
849 0x3f90, 0xffff0000, 0xff000000,
850 0x9148, 0xffff0000, 0xff000000,
851 0x3f94, 0xffff0000, 0xff000000,
852 0x914c, 0xffff0000, 0xff000000,
853 0xc78, 0x00000080, 0x00000080,
854 0xbd4, 0x70073777, 0x00010001,
855 0xd02c, 0xbfffff1f, 0x08421000,
856 0xd0b8, 0x03773777, 0x02011003,
857 0x5bc0, 0x00200000, 0x50100000,
858 0x98f8, 0x33773777, 0x02011003,
859 0x98fc, 0xffffffff, 0x76543210,
860 0x7030, 0x31000311, 0x00000011,
861 0x2f48, 0x00000007, 0x02011003,
862 0x6b28, 0x00000010, 0x00000012,
863 0x7728, 0x00000010, 0x00000012,
864 0x10328, 0x00000010, 0x00000012,
865 0x10f28, 0x00000010, 0x00000012,
866 0x11b28, 0x00000010, 0x00000012,
867 0x12728, 0x00000010, 0x00000012,
868 0x240c, 0x000007ff, 0x00000380,
869 0x8a14, 0xf000001f, 0x00000007,
870 0x8b24, 0x3fff3fff, 0x00ff0fff,
871 0x8b10, 0x0000ff0f, 0x00000000,
872 0x28a4c, 0x07ffffff, 0x06000000,
873 0x10c, 0x00000001, 0x00010003,
874 0xa02c, 0xffffffff, 0x0000009b,
875 0x913c, 0x0000000f, 0x0100000a,
876 0x8d00, 0xffff7f7f, 0x100e4848,
877 0x8d04, 0x00ffffff, 0x00164745,
878 0x8c00, 0xfffc0003, 0xe4000003,
879 0x8c04, 0xf8ff00ff, 0x40600060,
880 0x8c08, 0x00ff00ff, 0x001c001c,
881 0x8cf0, 0x1fff1fff, 0x08e00620,
882 0x8c20, 0x0fff0fff, 0x00800080,
883 0x8c24, 0x0fff0fff, 0x00800080,
884 0x8c18, 0xffffffff, 0x20202078,
885 0x8c1c, 0x0000ffff, 0x00001010,
886 0x28350, 0x00000f01, 0x00000000,
887 0x9508, 0x3700001f, 0x00000002,
888 0x960c, 0xffffffff, 0x54763210,
889 0x88c4, 0x001f3ae3, 0x000000c2,
890 0x88d4, 0x0000001f, 0x00000010,
891 0x8974, 0xffffffff, 0x00000000
892};
893
894static const u32 turks_golden_registers[] =
895{
896 0x5eb4, 0xffffffff, 0x00000002,
897 0x5e78, 0x8f311ff1, 0x001000f0,
898 0x8c8, 0x00003000, 0x00001070,
899 0x8cc, 0x000fffff, 0x00040035,
900 0x3f90, 0xffff0000, 0xfff00000,
901 0x9148, 0xffff0000, 0xfff00000,
902 0x3f94, 0xffff0000, 0xfff00000,
903 0x914c, 0xffff0000, 0xfff00000,
904 0xc78, 0x00000080, 0x00000080,
905 0xbd4, 0x00073007, 0x00010002,
906 0xd02c, 0xbfffff1f, 0x08421000,
907 0xd0b8, 0x03773777, 0x02010002,
908 0x5bc0, 0x00200000, 0x50100000,
909 0x98f8, 0x33773777, 0x00010002,
910 0x98fc, 0xffffffff, 0x33221100,
911 0x7030, 0x31000311, 0x00000011,
912 0x2f48, 0x33773777, 0x00010002,
913 0x6b28, 0x00000010, 0x00000012,
914 0x7728, 0x00000010, 0x00000012,
915 0x10328, 0x00000010, 0x00000012,
916 0x10f28, 0x00000010, 0x00000012,
917 0x11b28, 0x00000010, 0x00000012,
918 0x12728, 0x00000010, 0x00000012,
919 0x240c, 0x000007ff, 0x00000380,
920 0x8a14, 0xf000001f, 0x00000007,
921 0x8b24, 0x3fff3fff, 0x00ff0fff,
922 0x8b10, 0x0000ff0f, 0x00000000,
923 0x28a4c, 0x07ffffff, 0x06000000,
924 0x10c, 0x00000001, 0x00010003,
925 0xa02c, 0xffffffff, 0x0000009b,
926 0x913c, 0x0000000f, 0x0100000a,
927 0x8d00, 0xffff7f7f, 0x100e4848,
928 0x8d04, 0x00ffffff, 0x00164745,
929 0x8c00, 0xfffc0003, 0xe4000003,
930 0x8c04, 0xf8ff00ff, 0x40600060,
931 0x8c08, 0x00ff00ff, 0x001c001c,
932 0x8cf0, 0x1fff1fff, 0x08e00410,
933 0x8c20, 0x0fff0fff, 0x00800080,
934 0x8c24, 0x0fff0fff, 0x00800080,
935 0x8c18, 0xffffffff, 0x20202078,
936 0x8c1c, 0x0000ffff, 0x00001010,
937 0x28350, 0x00000f01, 0x00000000,
938 0x9508, 0x3700001f, 0x00000002,
939 0x960c, 0xffffffff, 0x54763210,
940 0x88c4, 0x001f3ae3, 0x000000c2,
941 0x88d4, 0x0000001f, 0x00000010,
942 0x8974, 0xffffffff, 0x00000000
943};
944
945static const u32 caicos_golden_registers[] =
946{
947 0x5eb4, 0xffffffff, 0x00000002,
948 0x5e78, 0x8f311ff1, 0x001000f0,
949 0x8c8, 0x00003420, 0x00001450,
950 0x8cc, 0x000fffff, 0x00040035,
951 0x3f90, 0xffff0000, 0xfffc0000,
952 0x9148, 0xffff0000, 0xfffc0000,
953 0x3f94, 0xffff0000, 0xfffc0000,
954 0x914c, 0xffff0000, 0xfffc0000,
955 0xc78, 0x00000080, 0x00000080,
956 0xbd4, 0x00073007, 0x00010001,
957 0xd02c, 0xbfffff1f, 0x08421000,
958 0xd0b8, 0x03773777, 0x02010001,
959 0x5bc0, 0x00200000, 0x50100000,
960 0x98f8, 0x33773777, 0x02010001,
961 0x98fc, 0xffffffff, 0x33221100,
962 0x7030, 0x31000311, 0x00000011,
963 0x2f48, 0x33773777, 0x02010001,
964 0x6b28, 0x00000010, 0x00000012,
965 0x7728, 0x00000010, 0x00000012,
966 0x10328, 0x00000010, 0x00000012,
967 0x10f28, 0x00000010, 0x00000012,
968 0x11b28, 0x00000010, 0x00000012,
969 0x12728, 0x00000010, 0x00000012,
970 0x240c, 0x000007ff, 0x00000380,
971 0x8a14, 0xf000001f, 0x00000001,
972 0x8b24, 0x3fff3fff, 0x00ff0fff,
973 0x8b10, 0x0000ff0f, 0x00000000,
974 0x28a4c, 0x07ffffff, 0x06000000,
975 0x10c, 0x00000001, 0x00010003,
976 0xa02c, 0xffffffff, 0x0000009b,
977 0x913c, 0x0000000f, 0x0100000a,
978 0x8d00, 0xffff7f7f, 0x100e4848,
979 0x8d04, 0x00ffffff, 0x00164745,
980 0x8c00, 0xfffc0003, 0xe4000003,
981 0x8c04, 0xf8ff00ff, 0x40600060,
982 0x8c08, 0x00ff00ff, 0x001c001c,
983 0x8cf0, 0x1fff1fff, 0x08e00410,
984 0x8c20, 0x0fff0fff, 0x00800080,
985 0x8c24, 0x0fff0fff, 0x00800080,
986 0x8c18, 0xffffffff, 0x20202078,
987 0x8c1c, 0x0000ffff, 0x00001010,
988 0x28350, 0x00000f01, 0x00000000,
989 0x9508, 0x3700001f, 0x00000002,
990 0x960c, 0xffffffff, 0x54763210,
991 0x88c4, 0x001f3ae3, 0x000000c2,
992 0x88d4, 0x0000001f, 0x00000010,
993 0x8974, 0xffffffff, 0x00000000
994};
995
996static void evergreen_init_golden_registers(struct radeon_device *rdev)
997{
998 switch (rdev->family) {
999 case CHIP_CYPRESS:
1000 case CHIP_HEMLOCK:
1001 radeon_program_register_sequence(rdev,
1002 evergreen_golden_registers,
1003 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1004 radeon_program_register_sequence(rdev,
1005 evergreen_golden_registers2,
1006 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1007 radeon_program_register_sequence(rdev,
1008 cypress_mgcg_init,
1009 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1010 break;
1011 case CHIP_JUNIPER:
1012 radeon_program_register_sequence(rdev,
1013 evergreen_golden_registers,
1014 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1015 radeon_program_register_sequence(rdev,
1016 evergreen_golden_registers2,
1017 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1018 radeon_program_register_sequence(rdev,
1019 juniper_mgcg_init,
1020 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1021 break;
1022 case CHIP_REDWOOD:
1023 radeon_program_register_sequence(rdev,
1024 evergreen_golden_registers,
1025 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1026 radeon_program_register_sequence(rdev,
1027 evergreen_golden_registers2,
1028 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1029 radeon_program_register_sequence(rdev,
1030 redwood_mgcg_init,
1031 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1032 break;
1033 case CHIP_CEDAR:
1034 radeon_program_register_sequence(rdev,
1035 cedar_golden_registers,
1036 (const u32)ARRAY_SIZE(cedar_golden_registers));
1037 radeon_program_register_sequence(rdev,
1038 evergreen_golden_registers2,
1039 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1040 radeon_program_register_sequence(rdev,
1041 cedar_mgcg_init,
1042 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1043 break;
1044 case CHIP_PALM:
1045 radeon_program_register_sequence(rdev,
1046 wrestler_golden_registers,
1047 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1048 break;
1049 case CHIP_SUMO:
1050 radeon_program_register_sequence(rdev,
1051 supersumo_golden_registers,
1052 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1053 break;
1054 case CHIP_SUMO2:
1055 radeon_program_register_sequence(rdev,
1056 supersumo_golden_registers,
1057 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1058 radeon_program_register_sequence(rdev,
1059 sumo_golden_registers,
1060 (const u32)ARRAY_SIZE(sumo_golden_registers));
1061 break;
1062 case CHIP_BARTS:
1063 radeon_program_register_sequence(rdev,
1064 barts_golden_registers,
1065 (const u32)ARRAY_SIZE(barts_golden_registers));
1066 break;
1067 case CHIP_TURKS:
1068 radeon_program_register_sequence(rdev,
1069 turks_golden_registers,
1070 (const u32)ARRAY_SIZE(turks_golden_registers));
1071 break;
1072 case CHIP_CAICOS:
1073 radeon_program_register_sequence(rdev,
1074 caicos_golden_registers,
1075 (const u32)ARRAY_SIZE(caicos_golden_registers));
1076 break;
1077 default:
1078 break;
1079 }
1080}
1081
1082/**
1083 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1084 *
1085 * @rdev: radeon_device pointer
1086 * @reg: register offset in bytes
1087 * @val: register value
1088 *
1089 * Returns 0 for success or -EINVAL for an invalid register
1090 *
1091 */
1092int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1093 u32 reg, u32 *val)
1094{
1095 switch (reg) {
1096 case GRBM_STATUS:
1097 case GRBM_STATUS_SE0:
1098 case GRBM_STATUS_SE1:
1099 case SRBM_STATUS:
1100 case SRBM_STATUS2:
1101 case DMA_STATUS_REG:
1102 case UVD_STATUS:
1103 *val = RREG32(reg);
1104 return 0;
1105 default:
1106 return -EINVAL;
1107 }
1108}
1109
1110void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1111 unsigned *bankh, unsigned *mtaspect,
1112 unsigned *tile_split)
1113{
1114 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1115 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1116 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1117 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1118 switch (*bankw) {
1119 default:
1120 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1121 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1122 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1123 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1124 }
1125 switch (*bankh) {
1126 default:
1127 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1128 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1129 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1130 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1131 }
1132 switch (*mtaspect) {
1133 default:
1134 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1135 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1136 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1137 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1138 }
1139}
1140
1141static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1142 u32 cntl_reg, u32 status_reg)
1143{
1144 int r, i;
1145 struct atom_clock_dividers dividers;
1146
1147 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1148 clock, false, ÷rs);
1149 if (r)
1150 return r;
1151
1152 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1153
1154 for (i = 0; i < 100; i++) {
1155 if (RREG32(status_reg) & DCLK_STATUS)
1156 break;
1157 mdelay(10);
1158 }
1159 if (i == 100)
1160 return -ETIMEDOUT;
1161
1162 return 0;
1163}
1164
1165int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1166{
1167 int r = 0;
1168 u32 cg_scratch = RREG32(CG_SCRATCH1);
1169
1170 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1171 if (r)
1172 goto done;
1173 cg_scratch &= 0xffff0000;
1174 cg_scratch |= vclk / 100; /* Mhz */
1175
1176 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1177 if (r)
1178 goto done;
1179 cg_scratch &= 0x0000ffff;
1180 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1181
1182done:
1183 WREG32(CG_SCRATCH1, cg_scratch);
1184
1185 return r;
1186}
1187
1188int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1189{
1190 /* start off with something large */
1191 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1192 int r;
1193
1194 /* bypass vclk and dclk with bclk */
1195 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1196 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1197 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1198
1199 /* put PLL in bypass mode */
1200 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1201
1202 if (!vclk || !dclk) {
1203 /* keep the Bypass mode, put PLL to sleep */
1204 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1205 return 0;
1206 }
1207
1208 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1209 16384, 0x03FFFFFF, 0, 128, 5,
1210 &fb_div, &vclk_div, &dclk_div);
1211 if (r)
1212 return r;
1213
1214 /* set VCO_MODE to 1 */
1215 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1216
1217 /* toggle UPLL_SLEEP to 1 then back to 0 */
1218 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1219 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1220
1221 /* deassert UPLL_RESET */
1222 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1223
1224 mdelay(1);
1225
1226 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1227 if (r)
1228 return r;
1229
1230 /* assert UPLL_RESET again */
1231 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1232
1233 /* disable spread spectrum. */
1234 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1235
1236 /* set feedback divider */
1237 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1238
1239 /* set ref divider to 0 */
1240 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1241
1242 if (fb_div < 307200)
1243 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1244 else
1245 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1246
1247 /* set PDIV_A and PDIV_B */
1248 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1249 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1250 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1251
1252 /* give the PLL some time to settle */
1253 mdelay(15);
1254
1255 /* deassert PLL_RESET */
1256 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1257
1258 mdelay(15);
1259
1260 /* switch from bypass mode to normal mode */
1261 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1262
1263 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1264 if (r)
1265 return r;
1266
1267 /* switch VCLK and DCLK selection */
1268 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1269 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1270 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1271
1272 mdelay(100);
1273
1274 return 0;
1275}
1276
1277void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1278{
1279 int readrq;
1280 u16 v;
1281
1282 readrq = pcie_get_readrq(rdev->pdev);
1283 v = ffs(readrq) - 8;
1284 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1285 * to avoid hangs or perfomance issues
1286 */
1287 if ((v == 0) || (v == 6) || (v == 7))
1288 pcie_set_readrq(rdev->pdev, 512);
1289}
1290
1291void dce4_program_fmt(struct drm_encoder *encoder)
1292{
1293 struct drm_device *dev = encoder->dev;
1294 struct radeon_device *rdev = dev->dev_private;
1295 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1296 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1297 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1298 int bpc = 0;
1299 u32 tmp = 0;
1300 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1301
1302 if (connector) {
1303 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1304 bpc = radeon_get_monitor_bpc(connector);
1305 dither = radeon_connector->dither;
1306 }
1307
1308 /* LVDS/eDP FMT is set up by atom */
1309 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1310 return;
1311
1312 /* not needed for analog */
1313 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1314 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1315 return;
1316
1317 if (bpc == 0)
1318 return;
1319
1320 switch (bpc) {
1321 case 6:
1322 if (dither == RADEON_FMT_DITHER_ENABLE)
1323 /* XXX sort out optimal dither settings */
1324 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1325 FMT_SPATIAL_DITHER_EN);
1326 else
1327 tmp |= FMT_TRUNCATE_EN;
1328 break;
1329 case 8:
1330 if (dither == RADEON_FMT_DITHER_ENABLE)
1331 /* XXX sort out optimal dither settings */
1332 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1333 FMT_RGB_RANDOM_ENABLE |
1334 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1335 else
1336 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1337 break;
1338 case 10:
1339 default:
1340 /* not needed */
1341 break;
1342 }
1343
1344 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1345}
1346
1347static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1348{
1349 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1350 return true;
1351 else
1352 return false;
1353}
1354
1355static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1356{
1357 u32 pos1, pos2;
1358
1359 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1360 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361
1362 if (pos1 != pos2)
1363 return true;
1364 else
1365 return false;
1366}
1367
1368/**
1369 * dce4_wait_for_vblank - vblank wait asic callback.
1370 *
1371 * @rdev: radeon_device pointer
1372 * @crtc: crtc to wait for vblank on
1373 *
1374 * Wait for vblank on the requested crtc (evergreen+).
1375 */
1376void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1377{
1378 unsigned i = 0;
1379
1380 if (crtc >= rdev->num_crtc)
1381 return;
1382
1383 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1384 return;
1385
1386 /* depending on when we hit vblank, we may be close to active; if so,
1387 * wait for another frame.
1388 */
1389 while (dce4_is_in_vblank(rdev, crtc)) {
1390 if (i++ % 100 == 0) {
1391 if (!dce4_is_counter_moving(rdev, crtc))
1392 break;
1393 }
1394 }
1395
1396 while (!dce4_is_in_vblank(rdev, crtc)) {
1397 if (i++ % 100 == 0) {
1398 if (!dce4_is_counter_moving(rdev, crtc))
1399 break;
1400 }
1401 }
1402}
1403
1404/**
1405 * evergreen_page_flip - pageflip callback.
1406 *
1407 * @rdev: radeon_device pointer
1408 * @crtc_id: crtc to cleanup pageflip on
1409 * @crtc_base: new address of the crtc (GPU MC address)
1410 *
1411 * Triggers the actual pageflip by updating the primary
1412 * surface base address (evergreen+).
1413 */
1414void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1415 bool async)
1416{
1417 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1418
1419 /* update the scanout addresses */
1420 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1421 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1422 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1423 upper_32_bits(crtc_base));
1424 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1425 (u32)crtc_base);
1426 /* post the write */
1427 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1428}
1429
1430/**
1431 * evergreen_page_flip_pending - check if page flip is still pending
1432 *
1433 * @rdev: radeon_device pointer
1434 * @crtc_id: crtc to check
1435 *
1436 * Returns the current update pending status.
1437 */
1438bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1439{
1440 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1441
1442 /* Return current update_pending status: */
1443 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1444 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1445}
1446
1447/* get temperature in millidegrees */
1448int evergreen_get_temp(struct radeon_device *rdev)
1449{
1450 u32 temp, toffset;
1451 int actual_temp = 0;
1452
1453 if (rdev->family == CHIP_JUNIPER) {
1454 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1455 TOFFSET_SHIFT;
1456 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1457 TS0_ADC_DOUT_SHIFT;
1458
1459 if (toffset & 0x100)
1460 actual_temp = temp / 2 - (0x200 - toffset);
1461 else
1462 actual_temp = temp / 2 + toffset;
1463
1464 actual_temp = actual_temp * 1000;
1465
1466 } else {
1467 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1468 ASIC_T_SHIFT;
1469
1470 if (temp & 0x400)
1471 actual_temp = -256;
1472 else if (temp & 0x200)
1473 actual_temp = 255;
1474 else if (temp & 0x100) {
1475 actual_temp = temp & 0x1ff;
1476 actual_temp |= ~0x1ff;
1477 } else
1478 actual_temp = temp & 0xff;
1479
1480 actual_temp = (actual_temp * 1000) / 2;
1481 }
1482
1483 return actual_temp;
1484}
1485
1486int sumo_get_temp(struct radeon_device *rdev)
1487{
1488 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1489 int actual_temp = temp - 49;
1490
1491 return actual_temp * 1000;
1492}
1493
1494/**
1495 * sumo_pm_init_profile - Initialize power profiles callback.
1496 *
1497 * @rdev: radeon_device pointer
1498 *
1499 * Initialize the power states used in profile mode
1500 * (sumo, trinity, SI).
1501 * Used for profile mode only.
1502 */
1503void sumo_pm_init_profile(struct radeon_device *rdev)
1504{
1505 int idx;
1506
1507 /* default */
1508 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1509 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1510 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1511 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1512
1513 /* low,mid sh/mh */
1514 if (rdev->flags & RADEON_IS_MOBILITY)
1515 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1516 else
1517 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1518
1519 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1520 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1521 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1522 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1523
1524 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1525 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1526 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1527 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1528
1529 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1532 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1533
1534 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1538
1539 /* high sh/mh */
1540 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1541 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1544 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1545 rdev->pm.power_state[idx].num_clock_modes - 1;
1546
1547 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1548 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1550 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1551 rdev->pm.power_state[idx].num_clock_modes - 1;
1552}
1553
1554/**
1555 * btc_pm_init_profile - Initialize power profiles callback.
1556 *
1557 * @rdev: radeon_device pointer
1558 *
1559 * Initialize the power states used in profile mode
1560 * (BTC, cayman).
1561 * Used for profile mode only.
1562 */
1563void btc_pm_init_profile(struct radeon_device *rdev)
1564{
1565 int idx;
1566
1567 /* default */
1568 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1569 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1570 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1571 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1572 /* starting with BTC, there is one state that is used for both
1573 * MH and SH. Difference is that we always use the high clock index for
1574 * mclk.
1575 */
1576 if (rdev->flags & RADEON_IS_MOBILITY)
1577 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1578 else
1579 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1580 /* low sh */
1581 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1582 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1583 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1584 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1585 /* mid sh */
1586 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1587 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1588 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1589 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1590 /* high sh */
1591 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1592 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1593 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1594 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1595 /* low mh */
1596 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1597 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1598 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1599 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1600 /* mid mh */
1601 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1602 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1603 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1604 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1605 /* high mh */
1606 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1607 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1608 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1609 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1610}
1611
1612/**
1613 * evergreen_pm_misc - set additional pm hw parameters callback.
1614 *
1615 * @rdev: radeon_device pointer
1616 *
1617 * Set non-clock parameters associated with a power state
1618 * (voltage, etc.) (evergreen+).
1619 */
1620void evergreen_pm_misc(struct radeon_device *rdev)
1621{
1622 int req_ps_idx = rdev->pm.requested_power_state_index;
1623 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1624 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1625 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1626
1627 if (voltage->type == VOLTAGE_SW) {
1628 /* 0xff0x are flags rather then an actual voltage */
1629 if ((voltage->voltage & 0xff00) == 0xff00)
1630 return;
1631 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1632 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1633 rdev->pm.current_vddc = voltage->voltage;
1634 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1635 }
1636
1637 /* starting with BTC, there is one state that is used for both
1638 * MH and SH. Difference is that we always use the high clock index for
1639 * mclk and vddci.
1640 */
1641 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1642 (rdev->family >= CHIP_BARTS) &&
1643 rdev->pm.active_crtc_count &&
1644 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1645 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1646 voltage = &rdev->pm.power_state[req_ps_idx].
1647 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1648
1649 /* 0xff0x are flags rather then an actual voltage */
1650 if ((voltage->vddci & 0xff00) == 0xff00)
1651 return;
1652 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1653 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1654 rdev->pm.current_vddci = voltage->vddci;
1655 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1656 }
1657 }
1658}
1659
1660/**
1661 * evergreen_pm_prepare - pre-power state change callback.
1662 *
1663 * @rdev: radeon_device pointer
1664 *
1665 * Prepare for a power state change (evergreen+).
1666 */
1667void evergreen_pm_prepare(struct radeon_device *rdev)
1668{
1669 struct drm_device *ddev = rdev->ddev;
1670 struct drm_crtc *crtc;
1671 struct radeon_crtc *radeon_crtc;
1672 u32 tmp;
1673
1674 /* disable any active CRTCs */
1675 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1676 radeon_crtc = to_radeon_crtc(crtc);
1677 if (radeon_crtc->enabled) {
1678 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1679 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1680 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1681 }
1682 }
1683}
1684
1685/**
1686 * evergreen_pm_finish - post-power state change callback.
1687 *
1688 * @rdev: radeon_device pointer
1689 *
1690 * Clean up after a power state change (evergreen+).
1691 */
1692void evergreen_pm_finish(struct radeon_device *rdev)
1693{
1694 struct drm_device *ddev = rdev->ddev;
1695 struct drm_crtc *crtc;
1696 struct radeon_crtc *radeon_crtc;
1697 u32 tmp;
1698
1699 /* enable any active CRTCs */
1700 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1701 radeon_crtc = to_radeon_crtc(crtc);
1702 if (radeon_crtc->enabled) {
1703 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1704 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1705 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1706 }
1707 }
1708}
1709
1710/**
1711 * evergreen_hpd_sense - hpd sense callback.
1712 *
1713 * @rdev: radeon_device pointer
1714 * @hpd: hpd (hotplug detect) pin
1715 *
1716 * Checks if a digital monitor is connected (evergreen+).
1717 * Returns true if connected, false if not connected.
1718 */
1719bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1720{
1721 if (hpd == RADEON_HPD_NONE)
1722 return false;
1723
1724 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1725}
1726
1727/**
1728 * evergreen_hpd_set_polarity - hpd set polarity callback.
1729 *
1730 * @rdev: radeon_device pointer
1731 * @hpd: hpd (hotplug detect) pin
1732 *
1733 * Set the polarity of the hpd pin (evergreen+).
1734 */
1735void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1736 enum radeon_hpd_id hpd)
1737{
1738 bool connected = evergreen_hpd_sense(rdev, hpd);
1739
1740 if (hpd == RADEON_HPD_NONE)
1741 return;
1742
1743 if (connected)
1744 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1745 else
1746 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1747}
1748
1749/**
1750 * evergreen_hpd_init - hpd setup callback.
1751 *
1752 * @rdev: radeon_device pointer
1753 *
1754 * Setup the hpd pins used by the card (evergreen+).
1755 * Enable the pin, set the polarity, and enable the hpd interrupts.
1756 */
1757void evergreen_hpd_init(struct radeon_device *rdev)
1758{
1759 struct drm_device *dev = rdev->ddev;
1760 struct drm_connector *connector;
1761 unsigned enabled = 0;
1762 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1763 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1764
1765 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1766 enum radeon_hpd_id hpd =
1767 to_radeon_connector(connector)->hpd.hpd;
1768
1769 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1770 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1771 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1772 * aux dp channel on imac and help (but not completely fix)
1773 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1774 * also avoid interrupt storms during dpms.
1775 */
1776 continue;
1777 }
1778
1779 if (hpd == RADEON_HPD_NONE)
1780 continue;
1781
1782 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1783 enabled |= 1 << hpd;
1784
1785 radeon_hpd_set_polarity(rdev, hpd);
1786 }
1787 radeon_irq_kms_enable_hpd(rdev, enabled);
1788}
1789
1790/**
1791 * evergreen_hpd_fini - hpd tear down callback.
1792 *
1793 * @rdev: radeon_device pointer
1794 *
1795 * Tear down the hpd pins used by the card (evergreen+).
1796 * Disable the hpd interrupts.
1797 */
1798void evergreen_hpd_fini(struct radeon_device *rdev)
1799{
1800 struct drm_device *dev = rdev->ddev;
1801 struct drm_connector *connector;
1802 unsigned disabled = 0;
1803
1804 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1805 enum radeon_hpd_id hpd =
1806 to_radeon_connector(connector)->hpd.hpd;
1807
1808 if (hpd == RADEON_HPD_NONE)
1809 continue;
1810
1811 WREG32(DC_HPDx_CONTROL(hpd), 0);
1812 disabled |= 1 << hpd;
1813 }
1814 radeon_irq_kms_disable_hpd(rdev, disabled);
1815}
1816
1817/* watermark setup */
1818
1819static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1820 struct radeon_crtc *radeon_crtc,
1821 struct drm_display_mode *mode,
1822 struct drm_display_mode *other_mode)
1823{
1824 u32 tmp, buffer_alloc, i;
1825 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1826 /*
1827 * Line Buffer Setup
1828 * There are 3 line buffers, each one shared by 2 display controllers.
1829 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1830 * the display controllers. The paritioning is done via one of four
1831 * preset allocations specified in bits 2:0:
1832 * first display controller
1833 * 0 - first half of lb (3840 * 2)
1834 * 1 - first 3/4 of lb (5760 * 2)
1835 * 2 - whole lb (7680 * 2), other crtc must be disabled
1836 * 3 - first 1/4 of lb (1920 * 2)
1837 * second display controller
1838 * 4 - second half of lb (3840 * 2)
1839 * 5 - second 3/4 of lb (5760 * 2)
1840 * 6 - whole lb (7680 * 2), other crtc must be disabled
1841 * 7 - last 1/4 of lb (1920 * 2)
1842 */
1843 /* this can get tricky if we have two large displays on a paired group
1844 * of crtcs. Ideally for multiple large displays we'd assign them to
1845 * non-linked crtcs for maximum line buffer allocation.
1846 */
1847 if (radeon_crtc->base.enabled && mode) {
1848 if (other_mode) {
1849 tmp = 0; /* 1/2 */
1850 buffer_alloc = 1;
1851 } else {
1852 tmp = 2; /* whole */
1853 buffer_alloc = 2;
1854 }
1855 } else {
1856 tmp = 0;
1857 buffer_alloc = 0;
1858 }
1859
1860 /* second controller of the pair uses second half of the lb */
1861 if (radeon_crtc->crtc_id % 2)
1862 tmp += 4;
1863 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1864
1865 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1866 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1867 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1868 for (i = 0; i < rdev->usec_timeout; i++) {
1869 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1870 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1871 break;
1872 udelay(1);
1873 }
1874 }
1875
1876 if (radeon_crtc->base.enabled && mode) {
1877 switch (tmp) {
1878 case 0:
1879 case 4:
1880 default:
1881 if (ASIC_IS_DCE5(rdev))
1882 return 4096 * 2;
1883 else
1884 return 3840 * 2;
1885 case 1:
1886 case 5:
1887 if (ASIC_IS_DCE5(rdev))
1888 return 6144 * 2;
1889 else
1890 return 5760 * 2;
1891 case 2:
1892 case 6:
1893 if (ASIC_IS_DCE5(rdev))
1894 return 8192 * 2;
1895 else
1896 return 7680 * 2;
1897 case 3:
1898 case 7:
1899 if (ASIC_IS_DCE5(rdev))
1900 return 2048 * 2;
1901 else
1902 return 1920 * 2;
1903 }
1904 }
1905
1906 /* controller not enabled, so no lb used */
1907 return 0;
1908}
1909
1910u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1911{
1912 u32 tmp = RREG32(MC_SHARED_CHMAP);
1913
1914 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1915 case 0:
1916 default:
1917 return 1;
1918 case 1:
1919 return 2;
1920 case 2:
1921 return 4;
1922 case 3:
1923 return 8;
1924 }
1925}
1926
1927struct evergreen_wm_params {
1928 u32 dram_channels; /* number of dram channels */
1929 u32 yclk; /* bandwidth per dram data pin in kHz */
1930 u32 sclk; /* engine clock in kHz */
1931 u32 disp_clk; /* display clock in kHz */
1932 u32 src_width; /* viewport width */
1933 u32 active_time; /* active display time in ns */
1934 u32 blank_time; /* blank time in ns */
1935 bool interlaced; /* mode is interlaced */
1936 fixed20_12 vsc; /* vertical scale ratio */
1937 u32 num_heads; /* number of active crtcs */
1938 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1939 u32 lb_size; /* line buffer allocated to pipe */
1940 u32 vtaps; /* vertical scaler taps */
1941};
1942
1943static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1944{
1945 /* Calculate DRAM Bandwidth and the part allocated to display. */
1946 fixed20_12 dram_efficiency; /* 0.7 */
1947 fixed20_12 yclk, dram_channels, bandwidth;
1948 fixed20_12 a;
1949
1950 a.full = dfixed_const(1000);
1951 yclk.full = dfixed_const(wm->yclk);
1952 yclk.full = dfixed_div(yclk, a);
1953 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1954 a.full = dfixed_const(10);
1955 dram_efficiency.full = dfixed_const(7);
1956 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1957 bandwidth.full = dfixed_mul(dram_channels, yclk);
1958 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1959
1960 return dfixed_trunc(bandwidth);
1961}
1962
1963static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1964{
1965 /* Calculate DRAM Bandwidth and the part allocated to display. */
1966 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1967 fixed20_12 yclk, dram_channels, bandwidth;
1968 fixed20_12 a;
1969
1970 a.full = dfixed_const(1000);
1971 yclk.full = dfixed_const(wm->yclk);
1972 yclk.full = dfixed_div(yclk, a);
1973 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1974 a.full = dfixed_const(10);
1975 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1976 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1977 bandwidth.full = dfixed_mul(dram_channels, yclk);
1978 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1979
1980 return dfixed_trunc(bandwidth);
1981}
1982
1983static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1984{
1985 /* Calculate the display Data return Bandwidth */
1986 fixed20_12 return_efficiency; /* 0.8 */
1987 fixed20_12 sclk, bandwidth;
1988 fixed20_12 a;
1989
1990 a.full = dfixed_const(1000);
1991 sclk.full = dfixed_const(wm->sclk);
1992 sclk.full = dfixed_div(sclk, a);
1993 a.full = dfixed_const(10);
1994 return_efficiency.full = dfixed_const(8);
1995 return_efficiency.full = dfixed_div(return_efficiency, a);
1996 a.full = dfixed_const(32);
1997 bandwidth.full = dfixed_mul(a, sclk);
1998 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1999
2000 return dfixed_trunc(bandwidth);
2001}
2002
2003static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2004{
2005 /* Calculate the DMIF Request Bandwidth */
2006 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2007 fixed20_12 disp_clk, bandwidth;
2008 fixed20_12 a;
2009
2010 a.full = dfixed_const(1000);
2011 disp_clk.full = dfixed_const(wm->disp_clk);
2012 disp_clk.full = dfixed_div(disp_clk, a);
2013 a.full = dfixed_const(10);
2014 disp_clk_request_efficiency.full = dfixed_const(8);
2015 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2016 a.full = dfixed_const(32);
2017 bandwidth.full = dfixed_mul(a, disp_clk);
2018 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2019
2020 return dfixed_trunc(bandwidth);
2021}
2022
2023static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2024{
2025 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2026 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2027 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2028 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2029
2030 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2031}
2032
2033static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2034{
2035 /* Calculate the display mode Average Bandwidth
2036 * DisplayMode should contain the source and destination dimensions,
2037 * timing, etc.
2038 */
2039 fixed20_12 bpp;
2040 fixed20_12 line_time;
2041 fixed20_12 src_width;
2042 fixed20_12 bandwidth;
2043 fixed20_12 a;
2044
2045 a.full = dfixed_const(1000);
2046 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2047 line_time.full = dfixed_div(line_time, a);
2048 bpp.full = dfixed_const(wm->bytes_per_pixel);
2049 src_width.full = dfixed_const(wm->src_width);
2050 bandwidth.full = dfixed_mul(src_width, bpp);
2051 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2052 bandwidth.full = dfixed_div(bandwidth, line_time);
2053
2054 return dfixed_trunc(bandwidth);
2055}
2056
2057static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2058{
2059 /* First calcualte the latency in ns */
2060 u32 mc_latency = 2000; /* 2000 ns. */
2061 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2062 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2063 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2064 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2065 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2066 (wm->num_heads * cursor_line_pair_return_time);
2067 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2068 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2069 fixed20_12 a, b, c;
2070
2071 if (wm->num_heads == 0)
2072 return 0;
2073
2074 a.full = dfixed_const(2);
2075 b.full = dfixed_const(1);
2076 if ((wm->vsc.full > a.full) ||
2077 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2078 (wm->vtaps >= 5) ||
2079 ((wm->vsc.full >= a.full) && wm->interlaced))
2080 max_src_lines_per_dst_line = 4;
2081 else
2082 max_src_lines_per_dst_line = 2;
2083
2084 a.full = dfixed_const(available_bandwidth);
2085 b.full = dfixed_const(wm->num_heads);
2086 a.full = dfixed_div(a, b);
2087
2088 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2089
2090 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2091 b.full = dfixed_const(1000);
2092 c.full = dfixed_const(lb_fill_bw);
2093 b.full = dfixed_div(c, b);
2094 a.full = dfixed_div(a, b);
2095 line_fill_time = dfixed_trunc(a);
2096
2097 if (line_fill_time < wm->active_time)
2098 return latency;
2099 else
2100 return latency + (line_fill_time - wm->active_time);
2101
2102}
2103
2104static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2105{
2106 if (evergreen_average_bandwidth(wm) <=
2107 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2108 return true;
2109 else
2110 return false;
2111};
2112
2113static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2114{
2115 if (evergreen_average_bandwidth(wm) <=
2116 (evergreen_available_bandwidth(wm) / wm->num_heads))
2117 return true;
2118 else
2119 return false;
2120};
2121
2122static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2123{
2124 u32 lb_partitions = wm->lb_size / wm->src_width;
2125 u32 line_time = wm->active_time + wm->blank_time;
2126 u32 latency_tolerant_lines;
2127 u32 latency_hiding;
2128 fixed20_12 a;
2129
2130 a.full = dfixed_const(1);
2131 if (wm->vsc.full > a.full)
2132 latency_tolerant_lines = 1;
2133 else {
2134 if (lb_partitions <= (wm->vtaps + 1))
2135 latency_tolerant_lines = 1;
2136 else
2137 latency_tolerant_lines = 2;
2138 }
2139
2140 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2141
2142 if (evergreen_latency_watermark(wm) <= latency_hiding)
2143 return true;
2144 else
2145 return false;
2146}
2147
2148static void evergreen_program_watermarks(struct radeon_device *rdev,
2149 struct radeon_crtc *radeon_crtc,
2150 u32 lb_size, u32 num_heads)
2151{
2152 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2153 struct evergreen_wm_params wm_low, wm_high;
2154 u32 dram_channels;
2155 u32 active_time;
2156 u32 line_time = 0;
2157 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2158 u32 priority_a_mark = 0, priority_b_mark = 0;
2159 u32 priority_a_cnt = PRIORITY_OFF;
2160 u32 priority_b_cnt = PRIORITY_OFF;
2161 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2162 u32 tmp, arb_control3;
2163 fixed20_12 a, b, c;
2164
2165 if (radeon_crtc->base.enabled && num_heads && mode) {
2166 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2167 (u32)mode->clock);
2168 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2169 (u32)mode->clock);
2170 line_time = min(line_time, (u32)65535);
2171 priority_a_cnt = 0;
2172 priority_b_cnt = 0;
2173 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2174
2175 /* watermark for high clocks */
2176 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2177 wm_high.yclk =
2178 radeon_dpm_get_mclk(rdev, false) * 10;
2179 wm_high.sclk =
2180 radeon_dpm_get_sclk(rdev, false) * 10;
2181 } else {
2182 wm_high.yclk = rdev->pm.current_mclk * 10;
2183 wm_high.sclk = rdev->pm.current_sclk * 10;
2184 }
2185
2186 wm_high.disp_clk = mode->clock;
2187 wm_high.src_width = mode->crtc_hdisplay;
2188 wm_high.active_time = active_time;
2189 wm_high.blank_time = line_time - wm_high.active_time;
2190 wm_high.interlaced = false;
2191 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2192 wm_high.interlaced = true;
2193 wm_high.vsc = radeon_crtc->vsc;
2194 wm_high.vtaps = 1;
2195 if (radeon_crtc->rmx_type != RMX_OFF)
2196 wm_high.vtaps = 2;
2197 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2198 wm_high.lb_size = lb_size;
2199 wm_high.dram_channels = dram_channels;
2200 wm_high.num_heads = num_heads;
2201
2202 /* watermark for low clocks */
2203 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2204 wm_low.yclk =
2205 radeon_dpm_get_mclk(rdev, true) * 10;
2206 wm_low.sclk =
2207 radeon_dpm_get_sclk(rdev, true) * 10;
2208 } else {
2209 wm_low.yclk = rdev->pm.current_mclk * 10;
2210 wm_low.sclk = rdev->pm.current_sclk * 10;
2211 }
2212
2213 wm_low.disp_clk = mode->clock;
2214 wm_low.src_width = mode->crtc_hdisplay;
2215 wm_low.active_time = active_time;
2216 wm_low.blank_time = line_time - wm_low.active_time;
2217 wm_low.interlaced = false;
2218 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2219 wm_low.interlaced = true;
2220 wm_low.vsc = radeon_crtc->vsc;
2221 wm_low.vtaps = 1;
2222 if (radeon_crtc->rmx_type != RMX_OFF)
2223 wm_low.vtaps = 2;
2224 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2225 wm_low.lb_size = lb_size;
2226 wm_low.dram_channels = dram_channels;
2227 wm_low.num_heads = num_heads;
2228
2229 /* set for high clocks */
2230 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2231 /* set for low clocks */
2232 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2233
2234 /* possibly force display priority to high */
2235 /* should really do this at mode validation time... */
2236 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2237 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2238 !evergreen_check_latency_hiding(&wm_high) ||
2239 (rdev->disp_priority == 2)) {
2240 DRM_DEBUG_KMS("force priority a to high\n");
2241 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2242 }
2243 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2244 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2245 !evergreen_check_latency_hiding(&wm_low) ||
2246 (rdev->disp_priority == 2)) {
2247 DRM_DEBUG_KMS("force priority b to high\n");
2248 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2249 }
2250
2251 a.full = dfixed_const(1000);
2252 b.full = dfixed_const(mode->clock);
2253 b.full = dfixed_div(b, a);
2254 c.full = dfixed_const(latency_watermark_a);
2255 c.full = dfixed_mul(c, b);
2256 c.full = dfixed_mul(c, radeon_crtc->hsc);
2257 c.full = dfixed_div(c, a);
2258 a.full = dfixed_const(16);
2259 c.full = dfixed_div(c, a);
2260 priority_a_mark = dfixed_trunc(c);
2261 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2262
2263 a.full = dfixed_const(1000);
2264 b.full = dfixed_const(mode->clock);
2265 b.full = dfixed_div(b, a);
2266 c.full = dfixed_const(latency_watermark_b);
2267 c.full = dfixed_mul(c, b);
2268 c.full = dfixed_mul(c, radeon_crtc->hsc);
2269 c.full = dfixed_div(c, a);
2270 a.full = dfixed_const(16);
2271 c.full = dfixed_div(c, a);
2272 priority_b_mark = dfixed_trunc(c);
2273 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2274
2275 /* Save number of lines the linebuffer leads before the scanout */
2276 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2277 }
2278
2279 /* select wm A */
2280 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2281 tmp = arb_control3;
2282 tmp &= ~LATENCY_WATERMARK_MASK(3);
2283 tmp |= LATENCY_WATERMARK_MASK(1);
2284 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2285 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2286 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2287 LATENCY_HIGH_WATERMARK(line_time)));
2288 /* select wm B */
2289 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2290 tmp &= ~LATENCY_WATERMARK_MASK(3);
2291 tmp |= LATENCY_WATERMARK_MASK(2);
2292 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2293 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2294 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2295 LATENCY_HIGH_WATERMARK(line_time)));
2296 /* restore original selection */
2297 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2298
2299 /* write the priority marks */
2300 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2301 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2302
2303 /* save values for DPM */
2304 radeon_crtc->line_time = line_time;
2305 radeon_crtc->wm_high = latency_watermark_a;
2306 radeon_crtc->wm_low = latency_watermark_b;
2307}
2308
2309/**
2310 * evergreen_bandwidth_update - update display watermarks callback.
2311 *
2312 * @rdev: radeon_device pointer
2313 *
2314 * Update the display watermarks based on the requested mode(s)
2315 * (evergreen+).
2316 */
2317void evergreen_bandwidth_update(struct radeon_device *rdev)
2318{
2319 struct drm_display_mode *mode0 = NULL;
2320 struct drm_display_mode *mode1 = NULL;
2321 u32 num_heads = 0, lb_size;
2322 int i;
2323
2324 if (!rdev->mode_info.mode_config_initialized)
2325 return;
2326
2327 radeon_update_display_priority(rdev);
2328
2329 for (i = 0; i < rdev->num_crtc; i++) {
2330 if (rdev->mode_info.crtcs[i]->base.enabled)
2331 num_heads++;
2332 }
2333 for (i = 0; i < rdev->num_crtc; i += 2) {
2334 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2335 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2336 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2337 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2338 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2339 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2340 }
2341}
2342
2343/**
2344 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2345 *
2346 * @rdev: radeon_device pointer
2347 *
2348 * Wait for the MC (memory controller) to be idle.
2349 * (evergreen+).
2350 * Returns 0 if the MC is idle, -1 if not.
2351 */
2352int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2353{
2354 unsigned i;
2355 u32 tmp;
2356
2357 for (i = 0; i < rdev->usec_timeout; i++) {
2358 /* read MC_STATUS */
2359 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2360 if (!tmp)
2361 return 0;
2362 udelay(1);
2363 }
2364 return -1;
2365}
2366
2367/*
2368 * GART
2369 */
2370void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2371{
2372 unsigned i;
2373 u32 tmp;
2374
2375 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2376
2377 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2378 for (i = 0; i < rdev->usec_timeout; i++) {
2379 /* read MC_STATUS */
2380 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2381 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2382 if (tmp == 2) {
2383 pr_warn("[drm] r600 flush TLB failed\n");
2384 return;
2385 }
2386 if (tmp) {
2387 return;
2388 }
2389 udelay(1);
2390 }
2391}
2392
2393static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2394{
2395 u32 tmp;
2396 int r;
2397
2398 if (rdev->gart.robj == NULL) {
2399 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2400 return -EINVAL;
2401 }
2402 r = radeon_gart_table_vram_pin(rdev);
2403 if (r)
2404 return r;
2405 /* Setup L2 cache */
2406 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2407 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2408 EFFECTIVE_L2_QUEUE_SIZE(7));
2409 WREG32(VM_L2_CNTL2, 0);
2410 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2411 /* Setup TLB control */
2412 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2413 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2414 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2415 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2416 if (rdev->flags & RADEON_IS_IGP) {
2417 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2418 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2419 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2420 } else {
2421 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2422 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2423 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2424 if ((rdev->family == CHIP_JUNIPER) ||
2425 (rdev->family == CHIP_CYPRESS) ||
2426 (rdev->family == CHIP_HEMLOCK) ||
2427 (rdev->family == CHIP_BARTS))
2428 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2429 }
2430 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2431 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2432 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2433 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2434 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2435 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2436 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2437 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2438 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2439 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2440 (u32)(rdev->dummy_page.addr >> 12));
2441 WREG32(VM_CONTEXT1_CNTL, 0);
2442
2443 evergreen_pcie_gart_tlb_flush(rdev);
2444 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2445 (unsigned)(rdev->mc.gtt_size >> 20),
2446 (unsigned long long)rdev->gart.table_addr);
2447 rdev->gart.ready = true;
2448 return 0;
2449}
2450
2451static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2452{
2453 u32 tmp;
2454
2455 /* Disable all tables */
2456 WREG32(VM_CONTEXT0_CNTL, 0);
2457 WREG32(VM_CONTEXT1_CNTL, 0);
2458
2459 /* Setup L2 cache */
2460 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2461 EFFECTIVE_L2_QUEUE_SIZE(7));
2462 WREG32(VM_L2_CNTL2, 0);
2463 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2464 /* Setup TLB control */
2465 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2466 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2467 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2468 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2469 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2470 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2471 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2472 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2473 radeon_gart_table_vram_unpin(rdev);
2474}
2475
2476static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2477{
2478 evergreen_pcie_gart_disable(rdev);
2479 radeon_gart_table_vram_free(rdev);
2480 radeon_gart_fini(rdev);
2481}
2482
2483
2484static void evergreen_agp_enable(struct radeon_device *rdev)
2485{
2486 u32 tmp;
2487
2488 /* Setup L2 cache */
2489 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2490 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2491 EFFECTIVE_L2_QUEUE_SIZE(7));
2492 WREG32(VM_L2_CNTL2, 0);
2493 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2494 /* Setup TLB control */
2495 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2496 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2497 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2498 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2499 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2500 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2501 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2502 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2503 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2504 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2505 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2506 WREG32(VM_CONTEXT0_CNTL, 0);
2507 WREG32(VM_CONTEXT1_CNTL, 0);
2508}
2509
2510static const unsigned ni_dig_offsets[] =
2511{
2512 NI_DIG0_REGISTER_OFFSET,
2513 NI_DIG1_REGISTER_OFFSET,
2514 NI_DIG2_REGISTER_OFFSET,
2515 NI_DIG3_REGISTER_OFFSET,
2516 NI_DIG4_REGISTER_OFFSET,
2517 NI_DIG5_REGISTER_OFFSET
2518};
2519
2520static const unsigned ni_tx_offsets[] =
2521{
2522 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2523 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2524 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2525 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2526 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2527 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2528};
2529
2530static const unsigned evergreen_dp_offsets[] =
2531{
2532 EVERGREEN_DP0_REGISTER_OFFSET,
2533 EVERGREEN_DP1_REGISTER_OFFSET,
2534 EVERGREEN_DP2_REGISTER_OFFSET,
2535 EVERGREEN_DP3_REGISTER_OFFSET,
2536 EVERGREEN_DP4_REGISTER_OFFSET,
2537 EVERGREEN_DP5_REGISTER_OFFSET
2538};
2539
2540static const unsigned evergreen_disp_int_status[] =
2541{
2542 DISP_INTERRUPT_STATUS,
2543 DISP_INTERRUPT_STATUS_CONTINUE,
2544 DISP_INTERRUPT_STATUS_CONTINUE2,
2545 DISP_INTERRUPT_STATUS_CONTINUE3,
2546 DISP_INTERRUPT_STATUS_CONTINUE4,
2547 DISP_INTERRUPT_STATUS_CONTINUE5
2548};
2549
2550/*
2551 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2552 * We go from crtc to connector and it is not relible since it
2553 * should be an opposite direction .If crtc is enable then
2554 * find the dig_fe which selects this crtc and insure that it enable.
2555 * if such dig_fe is found then find dig_be which selects found dig_be and
2556 * insure that it enable and in DP_SST mode.
2557 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2558 * from dp symbols clocks .
2559 */
2560static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2561 unsigned crtc_id, unsigned *ret_dig_fe)
2562{
2563 unsigned i;
2564 unsigned dig_fe;
2565 unsigned dig_be;
2566 unsigned dig_en_be;
2567 unsigned uniphy_pll;
2568 unsigned digs_fe_selected;
2569 unsigned dig_be_mode;
2570 unsigned dig_fe_mask;
2571 bool is_enabled = false;
2572 bool found_crtc = false;
2573
2574 /* loop through all running dig_fe to find selected crtc */
2575 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2576 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2577 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2578 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2579 /* found running pipe */
2580 found_crtc = true;
2581 dig_fe_mask = 1 << i;
2582 dig_fe = i;
2583 break;
2584 }
2585 }
2586
2587 if (found_crtc) {
2588 /* loop through all running dig_be to find selected dig_fe */
2589 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2590 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2591 /* if dig_fe_selected by dig_be? */
2592 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2593 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2594 if (dig_fe_mask & digs_fe_selected &&
2595 /* if dig_be in sst mode? */
2596 dig_be_mode == NI_DIG_BE_DPSST) {
2597 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2598 ni_dig_offsets[i]);
2599 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2600 ni_tx_offsets[i]);
2601 /* dig_be enable and tx is running */
2602 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2603 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2604 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2605 is_enabled = true;
2606 *ret_dig_fe = dig_fe;
2607 break;
2608 }
2609 }
2610 }
2611 }
2612
2613 return is_enabled;
2614}
2615
2616/*
2617 * Blank dig when in dp sst mode
2618 * Dig ignores crtc timing
2619 */
2620static void evergreen_blank_dp_output(struct radeon_device *rdev,
2621 unsigned dig_fe)
2622{
2623 unsigned stream_ctrl;
2624 unsigned fifo_ctrl;
2625 unsigned counter = 0;
2626
2627 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2628 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2629 return;
2630 }
2631
2632 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2633 evergreen_dp_offsets[dig_fe]);
2634 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2635 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2636 return;
2637 }
2638
2639 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2640 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2641 evergreen_dp_offsets[dig_fe], stream_ctrl);
2642
2643 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2644 evergreen_dp_offsets[dig_fe]);
2645 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2646 msleep(1);
2647 counter++;
2648 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2649 evergreen_dp_offsets[dig_fe]);
2650 }
2651 if (counter >= 32 )
2652 DRM_ERROR("counter exceeds %d\n", counter);
2653
2654 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2655 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2656 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2657
2658}
2659
2660void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2661{
2662 u32 crtc_enabled, tmp, frame_count, blackout;
2663 int i, j;
2664 unsigned dig_fe;
2665
2666 if (!ASIC_IS_NODCE(rdev)) {
2667 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2668 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2669
2670 /* disable VGA render */
2671 WREG32(VGA_RENDER_CONTROL, 0);
2672 }
2673 /* blank the display controllers */
2674 for (i = 0; i < rdev->num_crtc; i++) {
2675 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2676 if (crtc_enabled) {
2677 save->crtc_enabled[i] = true;
2678 if (ASIC_IS_DCE6(rdev)) {
2679 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2680 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2681 radeon_wait_for_vblank(rdev, i);
2682 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2683 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2684 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2685 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2686 }
2687 } else {
2688 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2689 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2690 radeon_wait_for_vblank(rdev, i);
2691 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2692 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2693 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2694 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2695 }
2696 }
2697 /* wait for the next frame */
2698 frame_count = radeon_get_vblank_counter(rdev, i);
2699 for (j = 0; j < rdev->usec_timeout; j++) {
2700 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2701 break;
2702 udelay(1);
2703 }
2704 /*we should disable dig if it drives dp sst*/
2705 /*but we are in radeon_device_init and the topology is unknown*/
2706 /*and it is available after radeon_modeset_init*/
2707 /*the following method radeon_atom_encoder_dpms_dig*/
2708 /*does the job if we initialize it properly*/
2709 /*for now we do it this manually*/
2710 /**/
2711 if (ASIC_IS_DCE5(rdev) &&
2712 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2713 evergreen_blank_dp_output(rdev, dig_fe);
2714 /*we could remove 6 lines below*/
2715 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2716 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2717 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2718 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2719 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2720 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2721 save->crtc_enabled[i] = false;
2722 /* ***** */
2723 } else {
2724 save->crtc_enabled[i] = false;
2725 }
2726 }
2727
2728 radeon_mc_wait_for_idle(rdev);
2729
2730 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2731 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2732 /* Block CPU access */
2733 WREG32(BIF_FB_EN, 0);
2734 /* blackout the MC */
2735 blackout &= ~BLACKOUT_MODE_MASK;
2736 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2737 }
2738 /* wait for the MC to settle */
2739 udelay(100);
2740
2741 /* lock double buffered regs */
2742 for (i = 0; i < rdev->num_crtc; i++) {
2743 if (save->crtc_enabled[i]) {
2744 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2745 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2746 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2747 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2748 }
2749 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2750 if (!(tmp & 1)) {
2751 tmp |= 1;
2752 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2753 }
2754 }
2755 }
2756}
2757
2758void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2759{
2760 u32 tmp, frame_count;
2761 int i, j;
2762
2763 /* update crtc base addresses */
2764 for (i = 0; i < rdev->num_crtc; i++) {
2765 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2766 upper_32_bits(rdev->mc.vram_start));
2767 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2768 upper_32_bits(rdev->mc.vram_start));
2769 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2770 (u32)rdev->mc.vram_start);
2771 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2772 (u32)rdev->mc.vram_start);
2773 }
2774
2775 if (!ASIC_IS_NODCE(rdev)) {
2776 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2777 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2778 }
2779
2780 /* unlock regs and wait for update */
2781 for (i = 0; i < rdev->num_crtc; i++) {
2782 if (save->crtc_enabled[i]) {
2783 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2784 if ((tmp & 0x7) != 0) {
2785 tmp &= ~0x7;
2786 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2787 }
2788 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2789 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2790 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2791 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2792 }
2793 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2794 if (tmp & 1) {
2795 tmp &= ~1;
2796 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2797 }
2798 for (j = 0; j < rdev->usec_timeout; j++) {
2799 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2800 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2801 break;
2802 udelay(1);
2803 }
2804 }
2805 }
2806
2807 /* unblackout the MC */
2808 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2809 tmp &= ~BLACKOUT_MODE_MASK;
2810 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2811 /* allow CPU access */
2812 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2813
2814 for (i = 0; i < rdev->num_crtc; i++) {
2815 if (save->crtc_enabled[i]) {
2816 if (ASIC_IS_DCE6(rdev)) {
2817 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2818 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2819 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2820 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2821 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2822 } else {
2823 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2824 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2825 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2826 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2827 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2828 }
2829 /* wait for the next frame */
2830 frame_count = radeon_get_vblank_counter(rdev, i);
2831 for (j = 0; j < rdev->usec_timeout; j++) {
2832 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2833 break;
2834 udelay(1);
2835 }
2836 }
2837 }
2838 if (!ASIC_IS_NODCE(rdev)) {
2839 /* Unlock vga access */
2840 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2841 mdelay(1);
2842 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2843 }
2844}
2845
2846void evergreen_mc_program(struct radeon_device *rdev)
2847{
2848 struct evergreen_mc_save save;
2849 u32 tmp;
2850 int i, j;
2851
2852 /* Initialize HDP */
2853 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2854 WREG32((0x2c14 + j), 0x00000000);
2855 WREG32((0x2c18 + j), 0x00000000);
2856 WREG32((0x2c1c + j), 0x00000000);
2857 WREG32((0x2c20 + j), 0x00000000);
2858 WREG32((0x2c24 + j), 0x00000000);
2859 }
2860 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2861
2862 evergreen_mc_stop(rdev, &save);
2863 if (evergreen_mc_wait_for_idle(rdev)) {
2864 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2865 }
2866 /* Lockout access through VGA aperture*/
2867 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2868 /* Update configuration */
2869 if (rdev->flags & RADEON_IS_AGP) {
2870 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2871 /* VRAM before AGP */
2872 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2873 rdev->mc.vram_start >> 12);
2874 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2875 rdev->mc.gtt_end >> 12);
2876 } else {
2877 /* VRAM after AGP */
2878 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2879 rdev->mc.gtt_start >> 12);
2880 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2881 rdev->mc.vram_end >> 12);
2882 }
2883 } else {
2884 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2885 rdev->mc.vram_start >> 12);
2886 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2887 rdev->mc.vram_end >> 12);
2888 }
2889 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2890 /* llano/ontario only */
2891 if ((rdev->family == CHIP_PALM) ||
2892 (rdev->family == CHIP_SUMO) ||
2893 (rdev->family == CHIP_SUMO2)) {
2894 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2895 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2896 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2897 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2898 }
2899 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2900 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2901 WREG32(MC_VM_FB_LOCATION, tmp);
2902 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2903 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2904 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2905 if (rdev->flags & RADEON_IS_AGP) {
2906 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2907 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2908 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2909 } else {
2910 WREG32(MC_VM_AGP_BASE, 0);
2911 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2912 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2913 }
2914 if (evergreen_mc_wait_for_idle(rdev)) {
2915 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2916 }
2917 evergreen_mc_resume(rdev, &save);
2918 /* we need to own VRAM, so turn off the VGA renderer here
2919 * to stop it overwriting our objects */
2920 rv515_vga_render_disable(rdev);
2921}
2922
2923/*
2924 * CP.
2925 */
2926void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2927{
2928 struct radeon_ring *ring = &rdev->ring[ib->ring];
2929 u32 next_rptr;
2930
2931 /* set to DX10/11 mode */
2932 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2933 radeon_ring_write(ring, 1);
2934
2935 if (ring->rptr_save_reg) {
2936 next_rptr = ring->wptr + 3 + 4;
2937 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2938 radeon_ring_write(ring, ((ring->rptr_save_reg -
2939 PACKET3_SET_CONFIG_REG_START) >> 2));
2940 radeon_ring_write(ring, next_rptr);
2941 } else if (rdev->wb.enabled) {
2942 next_rptr = ring->wptr + 5 + 4;
2943 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2944 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2945 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2946 radeon_ring_write(ring, next_rptr);
2947 radeon_ring_write(ring, 0);
2948 }
2949
2950 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2951 radeon_ring_write(ring,
2952#ifdef __BIG_ENDIAN
2953 (2 << 0) |
2954#endif
2955 (ib->gpu_addr & 0xFFFFFFFC));
2956 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2957 radeon_ring_write(ring, ib->length_dw);
2958}
2959
2960
2961static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2962{
2963 const __be32 *fw_data;
2964 int i;
2965
2966 if (!rdev->me_fw || !rdev->pfp_fw)
2967 return -EINVAL;
2968
2969 r700_cp_stop(rdev);
2970 WREG32(CP_RB_CNTL,
2971#ifdef __BIG_ENDIAN
2972 BUF_SWAP_32BIT |
2973#endif
2974 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2975
2976 fw_data = (const __be32 *)rdev->pfp_fw->data;
2977 WREG32(CP_PFP_UCODE_ADDR, 0);
2978 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2979 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2980 WREG32(CP_PFP_UCODE_ADDR, 0);
2981
2982 fw_data = (const __be32 *)rdev->me_fw->data;
2983 WREG32(CP_ME_RAM_WADDR, 0);
2984 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2985 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2986
2987 WREG32(CP_PFP_UCODE_ADDR, 0);
2988 WREG32(CP_ME_RAM_WADDR, 0);
2989 WREG32(CP_ME_RAM_RADDR, 0);
2990 return 0;
2991}
2992
2993static int evergreen_cp_start(struct radeon_device *rdev)
2994{
2995 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2996 int r, i;
2997 uint32_t cp_me;
2998
2999 r = radeon_ring_lock(rdev, ring, 7);
3000 if (r) {
3001 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3002 return r;
3003 }
3004 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3005 radeon_ring_write(ring, 0x1);
3006 radeon_ring_write(ring, 0x0);
3007 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3008 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3009 radeon_ring_write(ring, 0);
3010 radeon_ring_write(ring, 0);
3011 radeon_ring_unlock_commit(rdev, ring, false);
3012
3013 cp_me = 0xff;
3014 WREG32(CP_ME_CNTL, cp_me);
3015
3016 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3017 if (r) {
3018 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3019 return r;
3020 }
3021
3022 /* setup clear context state */
3023 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3024 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3025
3026 for (i = 0; i < evergreen_default_size; i++)
3027 radeon_ring_write(ring, evergreen_default_state[i]);
3028
3029 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3030 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3031
3032 /* set clear context state */
3033 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3034 radeon_ring_write(ring, 0);
3035
3036 /* SQ_VTX_BASE_VTX_LOC */
3037 radeon_ring_write(ring, 0xc0026f00);
3038 radeon_ring_write(ring, 0x00000000);
3039 radeon_ring_write(ring, 0x00000000);
3040 radeon_ring_write(ring, 0x00000000);
3041
3042 /* Clear consts */
3043 radeon_ring_write(ring, 0xc0036f00);
3044 radeon_ring_write(ring, 0x00000bc4);
3045 radeon_ring_write(ring, 0xffffffff);
3046 radeon_ring_write(ring, 0xffffffff);
3047 radeon_ring_write(ring, 0xffffffff);
3048
3049 radeon_ring_write(ring, 0xc0026900);
3050 radeon_ring_write(ring, 0x00000316);
3051 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3052 radeon_ring_write(ring, 0x00000010); /* */
3053
3054 radeon_ring_unlock_commit(rdev, ring, false);
3055
3056 return 0;
3057}
3058
3059static int evergreen_cp_resume(struct radeon_device *rdev)
3060{
3061 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3062 u32 tmp;
3063 u32 rb_bufsz;
3064 int r;
3065
3066 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3067 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3068 SOFT_RESET_PA |
3069 SOFT_RESET_SH |
3070 SOFT_RESET_VGT |
3071 SOFT_RESET_SPI |
3072 SOFT_RESET_SX));
3073 RREG32(GRBM_SOFT_RESET);
3074 mdelay(15);
3075 WREG32(GRBM_SOFT_RESET, 0);
3076 RREG32(GRBM_SOFT_RESET);
3077
3078 /* Set ring buffer size */
3079 rb_bufsz = order_base_2(ring->ring_size / 8);
3080 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3081#ifdef __BIG_ENDIAN
3082 tmp |= BUF_SWAP_32BIT;
3083#endif
3084 WREG32(CP_RB_CNTL, tmp);
3085 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3086 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3087
3088 /* Set the write pointer delay */
3089 WREG32(CP_RB_WPTR_DELAY, 0);
3090
3091 /* Initialize the ring buffer's read and write pointers */
3092 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3093 WREG32(CP_RB_RPTR_WR, 0);
3094 ring->wptr = 0;
3095 WREG32(CP_RB_WPTR, ring->wptr);
3096
3097 /* set the wb address whether it's enabled or not */
3098 WREG32(CP_RB_RPTR_ADDR,
3099 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3100 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3101 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3102
3103 if (rdev->wb.enabled)
3104 WREG32(SCRATCH_UMSK, 0xff);
3105 else {
3106 tmp |= RB_NO_UPDATE;
3107 WREG32(SCRATCH_UMSK, 0);
3108 }
3109
3110 mdelay(1);
3111 WREG32(CP_RB_CNTL, tmp);
3112
3113 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3114 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3115
3116 evergreen_cp_start(rdev);
3117 ring->ready = true;
3118 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3119 if (r) {
3120 ring->ready = false;
3121 return r;
3122 }
3123 return 0;
3124}
3125
3126/*
3127 * Core functions
3128 */
3129static void evergreen_gpu_init(struct radeon_device *rdev)
3130{
3131 u32 gb_addr_config;
3132 u32 mc_shared_chmap, mc_arb_ramcfg;
3133 u32 sx_debug_1;
3134 u32 smx_dc_ctl0;
3135 u32 sq_config;
3136 u32 sq_lds_resource_mgmt;
3137 u32 sq_gpr_resource_mgmt_1;
3138 u32 sq_gpr_resource_mgmt_2;
3139 u32 sq_gpr_resource_mgmt_3;
3140 u32 sq_thread_resource_mgmt;
3141 u32 sq_thread_resource_mgmt_2;
3142 u32 sq_stack_resource_mgmt_1;
3143 u32 sq_stack_resource_mgmt_2;
3144 u32 sq_stack_resource_mgmt_3;
3145 u32 vgt_cache_invalidation;
3146 u32 hdp_host_path_cntl, tmp;
3147 u32 disabled_rb_mask;
3148 int i, j, ps_thread_count;
3149
3150 switch (rdev->family) {
3151 case CHIP_CYPRESS:
3152 case CHIP_HEMLOCK:
3153 rdev->config.evergreen.num_ses = 2;
3154 rdev->config.evergreen.max_pipes = 4;
3155 rdev->config.evergreen.max_tile_pipes = 8;
3156 rdev->config.evergreen.max_simds = 10;
3157 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3158 rdev->config.evergreen.max_gprs = 256;
3159 rdev->config.evergreen.max_threads = 248;
3160 rdev->config.evergreen.max_gs_threads = 32;
3161 rdev->config.evergreen.max_stack_entries = 512;
3162 rdev->config.evergreen.sx_num_of_sets = 4;
3163 rdev->config.evergreen.sx_max_export_size = 256;
3164 rdev->config.evergreen.sx_max_export_pos_size = 64;
3165 rdev->config.evergreen.sx_max_export_smx_size = 192;
3166 rdev->config.evergreen.max_hw_contexts = 8;
3167 rdev->config.evergreen.sq_num_cf_insts = 2;
3168
3169 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3170 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3171 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3172 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3173 break;
3174 case CHIP_JUNIPER:
3175 rdev->config.evergreen.num_ses = 1;
3176 rdev->config.evergreen.max_pipes = 4;
3177 rdev->config.evergreen.max_tile_pipes = 4;
3178 rdev->config.evergreen.max_simds = 10;
3179 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3180 rdev->config.evergreen.max_gprs = 256;
3181 rdev->config.evergreen.max_threads = 248;
3182 rdev->config.evergreen.max_gs_threads = 32;
3183 rdev->config.evergreen.max_stack_entries = 512;
3184 rdev->config.evergreen.sx_num_of_sets = 4;
3185 rdev->config.evergreen.sx_max_export_size = 256;
3186 rdev->config.evergreen.sx_max_export_pos_size = 64;
3187 rdev->config.evergreen.sx_max_export_smx_size = 192;
3188 rdev->config.evergreen.max_hw_contexts = 8;
3189 rdev->config.evergreen.sq_num_cf_insts = 2;
3190
3191 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3192 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3193 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3194 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3195 break;
3196 case CHIP_REDWOOD:
3197 rdev->config.evergreen.num_ses = 1;
3198 rdev->config.evergreen.max_pipes = 4;
3199 rdev->config.evergreen.max_tile_pipes = 4;
3200 rdev->config.evergreen.max_simds = 5;
3201 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3202 rdev->config.evergreen.max_gprs = 256;
3203 rdev->config.evergreen.max_threads = 248;
3204 rdev->config.evergreen.max_gs_threads = 32;
3205 rdev->config.evergreen.max_stack_entries = 256;
3206 rdev->config.evergreen.sx_num_of_sets = 4;
3207 rdev->config.evergreen.sx_max_export_size = 256;
3208 rdev->config.evergreen.sx_max_export_pos_size = 64;
3209 rdev->config.evergreen.sx_max_export_smx_size = 192;
3210 rdev->config.evergreen.max_hw_contexts = 8;
3211 rdev->config.evergreen.sq_num_cf_insts = 2;
3212
3213 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3214 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3215 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3216 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3217 break;
3218 case CHIP_CEDAR:
3219 default:
3220 rdev->config.evergreen.num_ses = 1;
3221 rdev->config.evergreen.max_pipes = 2;
3222 rdev->config.evergreen.max_tile_pipes = 2;
3223 rdev->config.evergreen.max_simds = 2;
3224 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3225 rdev->config.evergreen.max_gprs = 256;
3226 rdev->config.evergreen.max_threads = 192;
3227 rdev->config.evergreen.max_gs_threads = 16;
3228 rdev->config.evergreen.max_stack_entries = 256;
3229 rdev->config.evergreen.sx_num_of_sets = 4;
3230 rdev->config.evergreen.sx_max_export_size = 128;
3231 rdev->config.evergreen.sx_max_export_pos_size = 32;
3232 rdev->config.evergreen.sx_max_export_smx_size = 96;
3233 rdev->config.evergreen.max_hw_contexts = 4;
3234 rdev->config.evergreen.sq_num_cf_insts = 1;
3235
3236 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3237 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3238 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3239 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3240 break;
3241 case CHIP_PALM:
3242 rdev->config.evergreen.num_ses = 1;
3243 rdev->config.evergreen.max_pipes = 2;
3244 rdev->config.evergreen.max_tile_pipes = 2;
3245 rdev->config.evergreen.max_simds = 2;
3246 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3247 rdev->config.evergreen.max_gprs = 256;
3248 rdev->config.evergreen.max_threads = 192;
3249 rdev->config.evergreen.max_gs_threads = 16;
3250 rdev->config.evergreen.max_stack_entries = 256;
3251 rdev->config.evergreen.sx_num_of_sets = 4;
3252 rdev->config.evergreen.sx_max_export_size = 128;
3253 rdev->config.evergreen.sx_max_export_pos_size = 32;
3254 rdev->config.evergreen.sx_max_export_smx_size = 96;
3255 rdev->config.evergreen.max_hw_contexts = 4;
3256 rdev->config.evergreen.sq_num_cf_insts = 1;
3257
3258 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3259 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3260 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3261 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3262 break;
3263 case CHIP_SUMO:
3264 rdev->config.evergreen.num_ses = 1;
3265 rdev->config.evergreen.max_pipes = 4;
3266 rdev->config.evergreen.max_tile_pipes = 4;
3267 if (rdev->pdev->device == 0x9648)
3268 rdev->config.evergreen.max_simds = 3;
3269 else if ((rdev->pdev->device == 0x9647) ||
3270 (rdev->pdev->device == 0x964a))
3271 rdev->config.evergreen.max_simds = 4;
3272 else
3273 rdev->config.evergreen.max_simds = 5;
3274 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3275 rdev->config.evergreen.max_gprs = 256;
3276 rdev->config.evergreen.max_threads = 248;
3277 rdev->config.evergreen.max_gs_threads = 32;
3278 rdev->config.evergreen.max_stack_entries = 256;
3279 rdev->config.evergreen.sx_num_of_sets = 4;
3280 rdev->config.evergreen.sx_max_export_size = 256;
3281 rdev->config.evergreen.sx_max_export_pos_size = 64;
3282 rdev->config.evergreen.sx_max_export_smx_size = 192;
3283 rdev->config.evergreen.max_hw_contexts = 8;
3284 rdev->config.evergreen.sq_num_cf_insts = 2;
3285
3286 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3287 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3288 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3289 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3290 break;
3291 case CHIP_SUMO2:
3292 rdev->config.evergreen.num_ses = 1;
3293 rdev->config.evergreen.max_pipes = 4;
3294 rdev->config.evergreen.max_tile_pipes = 4;
3295 rdev->config.evergreen.max_simds = 2;
3296 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3297 rdev->config.evergreen.max_gprs = 256;
3298 rdev->config.evergreen.max_threads = 248;
3299 rdev->config.evergreen.max_gs_threads = 32;
3300 rdev->config.evergreen.max_stack_entries = 512;
3301 rdev->config.evergreen.sx_num_of_sets = 4;
3302 rdev->config.evergreen.sx_max_export_size = 256;
3303 rdev->config.evergreen.sx_max_export_pos_size = 64;
3304 rdev->config.evergreen.sx_max_export_smx_size = 192;
3305 rdev->config.evergreen.max_hw_contexts = 4;
3306 rdev->config.evergreen.sq_num_cf_insts = 2;
3307
3308 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3309 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3310 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3311 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3312 break;
3313 case CHIP_BARTS:
3314 rdev->config.evergreen.num_ses = 2;
3315 rdev->config.evergreen.max_pipes = 4;
3316 rdev->config.evergreen.max_tile_pipes = 8;
3317 rdev->config.evergreen.max_simds = 7;
3318 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3319 rdev->config.evergreen.max_gprs = 256;
3320 rdev->config.evergreen.max_threads = 248;
3321 rdev->config.evergreen.max_gs_threads = 32;
3322 rdev->config.evergreen.max_stack_entries = 512;
3323 rdev->config.evergreen.sx_num_of_sets = 4;
3324 rdev->config.evergreen.sx_max_export_size = 256;
3325 rdev->config.evergreen.sx_max_export_pos_size = 64;
3326 rdev->config.evergreen.sx_max_export_smx_size = 192;
3327 rdev->config.evergreen.max_hw_contexts = 8;
3328 rdev->config.evergreen.sq_num_cf_insts = 2;
3329
3330 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3331 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3332 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3333 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3334 break;
3335 case CHIP_TURKS:
3336 rdev->config.evergreen.num_ses = 1;
3337 rdev->config.evergreen.max_pipes = 4;
3338 rdev->config.evergreen.max_tile_pipes = 4;
3339 rdev->config.evergreen.max_simds = 6;
3340 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3341 rdev->config.evergreen.max_gprs = 256;
3342 rdev->config.evergreen.max_threads = 248;
3343 rdev->config.evergreen.max_gs_threads = 32;
3344 rdev->config.evergreen.max_stack_entries = 256;
3345 rdev->config.evergreen.sx_num_of_sets = 4;
3346 rdev->config.evergreen.sx_max_export_size = 256;
3347 rdev->config.evergreen.sx_max_export_pos_size = 64;
3348 rdev->config.evergreen.sx_max_export_smx_size = 192;
3349 rdev->config.evergreen.max_hw_contexts = 8;
3350 rdev->config.evergreen.sq_num_cf_insts = 2;
3351
3352 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3353 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3354 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3355 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3356 break;
3357 case CHIP_CAICOS:
3358 rdev->config.evergreen.num_ses = 1;
3359 rdev->config.evergreen.max_pipes = 2;
3360 rdev->config.evergreen.max_tile_pipes = 2;
3361 rdev->config.evergreen.max_simds = 2;
3362 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3363 rdev->config.evergreen.max_gprs = 256;
3364 rdev->config.evergreen.max_threads = 192;
3365 rdev->config.evergreen.max_gs_threads = 16;
3366 rdev->config.evergreen.max_stack_entries = 256;
3367 rdev->config.evergreen.sx_num_of_sets = 4;
3368 rdev->config.evergreen.sx_max_export_size = 128;
3369 rdev->config.evergreen.sx_max_export_pos_size = 32;
3370 rdev->config.evergreen.sx_max_export_smx_size = 96;
3371 rdev->config.evergreen.max_hw_contexts = 4;
3372 rdev->config.evergreen.sq_num_cf_insts = 1;
3373
3374 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3375 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3376 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3377 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3378 break;
3379 }
3380
3381 /* Initialize HDP */
3382 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3383 WREG32((0x2c14 + j), 0x00000000);
3384 WREG32((0x2c18 + j), 0x00000000);
3385 WREG32((0x2c1c + j), 0x00000000);
3386 WREG32((0x2c20 + j), 0x00000000);
3387 WREG32((0x2c24 + j), 0x00000000);
3388 }
3389
3390 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3391 WREG32(SRBM_INT_CNTL, 0x1);
3392 WREG32(SRBM_INT_ACK, 0x1);
3393
3394 evergreen_fix_pci_max_read_req_size(rdev);
3395
3396 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3397 if ((rdev->family == CHIP_PALM) ||
3398 (rdev->family == CHIP_SUMO) ||
3399 (rdev->family == CHIP_SUMO2))
3400 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3401 else
3402 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3403
3404 /* setup tiling info dword. gb_addr_config is not adequate since it does
3405 * not have bank info, so create a custom tiling dword.
3406 * bits 3:0 num_pipes
3407 * bits 7:4 num_banks
3408 * bits 11:8 group_size
3409 * bits 15:12 row_size
3410 */
3411 rdev->config.evergreen.tile_config = 0;
3412 switch (rdev->config.evergreen.max_tile_pipes) {
3413 case 1:
3414 default:
3415 rdev->config.evergreen.tile_config |= (0 << 0);
3416 break;
3417 case 2:
3418 rdev->config.evergreen.tile_config |= (1 << 0);
3419 break;
3420 case 4:
3421 rdev->config.evergreen.tile_config |= (2 << 0);
3422 break;
3423 case 8:
3424 rdev->config.evergreen.tile_config |= (3 << 0);
3425 break;
3426 }
3427 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3428 if (rdev->flags & RADEON_IS_IGP)
3429 rdev->config.evergreen.tile_config |= 1 << 4;
3430 else {
3431 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3432 case 0: /* four banks */
3433 rdev->config.evergreen.tile_config |= 0 << 4;
3434 break;
3435 case 1: /* eight banks */
3436 rdev->config.evergreen.tile_config |= 1 << 4;
3437 break;
3438 case 2: /* sixteen banks */
3439 default:
3440 rdev->config.evergreen.tile_config |= 2 << 4;
3441 break;
3442 }
3443 }
3444 rdev->config.evergreen.tile_config |= 0 << 8;
3445 rdev->config.evergreen.tile_config |=
3446 ((gb_addr_config & 0x30000000) >> 28) << 12;
3447
3448 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3449 u32 efuse_straps_4;
3450 u32 efuse_straps_3;
3451
3452 efuse_straps_4 = RREG32_RCU(0x204);
3453 efuse_straps_3 = RREG32_RCU(0x203);
3454 tmp = (((efuse_straps_4 & 0xf) << 4) |
3455 ((efuse_straps_3 & 0xf0000000) >> 28));
3456 } else {
3457 tmp = 0;
3458 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3459 u32 rb_disable_bitmap;
3460
3461 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3462 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3463 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3464 tmp <<= 4;
3465 tmp |= rb_disable_bitmap;
3466 }
3467 }
3468 /* enabled rb are just the one not disabled :) */
3469 disabled_rb_mask = tmp;
3470 tmp = 0;
3471 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3472 tmp |= (1 << i);
3473 /* if all the backends are disabled, fix it up here */
3474 if ((disabled_rb_mask & tmp) == tmp) {
3475 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3476 disabled_rb_mask &= ~(1 << i);
3477 }
3478
3479 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3480 u32 simd_disable_bitmap;
3481
3482 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3483 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3484 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3485 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3486 tmp <<= 16;
3487 tmp |= simd_disable_bitmap;
3488 }
3489 rdev->config.evergreen.active_simds = hweight32(~tmp);
3490
3491 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3492 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3493
3494 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3495 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3496 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3497 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3498 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3499 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3500 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3501
3502 if ((rdev->config.evergreen.max_backends == 1) &&
3503 (rdev->flags & RADEON_IS_IGP)) {
3504 if ((disabled_rb_mask & 3) == 1) {
3505 /* RB0 disabled, RB1 enabled */
3506 tmp = 0x11111111;
3507 } else {
3508 /* RB1 disabled, RB0 enabled */
3509 tmp = 0x00000000;
3510 }
3511 } else {
3512 tmp = gb_addr_config & NUM_PIPES_MASK;
3513 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3514 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3515 }
3516 rdev->config.evergreen.backend_map = tmp;
3517 WREG32(GB_BACKEND_MAP, tmp);
3518
3519 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3520 WREG32(CGTS_TCC_DISABLE, 0);
3521 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3522 WREG32(CGTS_USER_TCC_DISABLE, 0);
3523
3524 /* set HW defaults for 3D engine */
3525 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3526 ROQ_IB2_START(0x2b)));
3527
3528 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3529
3530 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3531 SYNC_GRADIENT |
3532 SYNC_WALKER |
3533 SYNC_ALIGNER));
3534
3535 sx_debug_1 = RREG32(SX_DEBUG_1);
3536 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3537 WREG32(SX_DEBUG_1, sx_debug_1);
3538
3539
3540 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3541 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3542 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3543 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3544
3545 if (rdev->family <= CHIP_SUMO2)
3546 WREG32(SMX_SAR_CTL0, 0x00010000);
3547
3548 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3549 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3550 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3551
3552 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3553 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3554 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3555
3556 WREG32(VGT_NUM_INSTANCES, 1);
3557 WREG32(SPI_CONFIG_CNTL, 0);
3558 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3559 WREG32(CP_PERFMON_CNTL, 0);
3560
3561 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3562 FETCH_FIFO_HIWATER(0x4) |
3563 DONE_FIFO_HIWATER(0xe0) |
3564 ALU_UPDATE_FIFO_HIWATER(0x8)));
3565
3566 sq_config = RREG32(SQ_CONFIG);
3567 sq_config &= ~(PS_PRIO(3) |
3568 VS_PRIO(3) |
3569 GS_PRIO(3) |
3570 ES_PRIO(3));
3571 sq_config |= (VC_ENABLE |
3572 EXPORT_SRC_C |
3573 PS_PRIO(0) |
3574 VS_PRIO(1) |
3575 GS_PRIO(2) |
3576 ES_PRIO(3));
3577
3578 switch (rdev->family) {
3579 case CHIP_CEDAR:
3580 case CHIP_PALM:
3581 case CHIP_SUMO:
3582 case CHIP_SUMO2:
3583 case CHIP_CAICOS:
3584 /* no vertex cache */
3585 sq_config &= ~VC_ENABLE;
3586 break;
3587 default:
3588 break;
3589 }
3590
3591 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3592
3593 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3594 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3595 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3596 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3597 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3598 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3599 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3600
3601 switch (rdev->family) {
3602 case CHIP_CEDAR:
3603 case CHIP_PALM:
3604 case CHIP_SUMO:
3605 case CHIP_SUMO2:
3606 ps_thread_count = 96;
3607 break;
3608 default:
3609 ps_thread_count = 128;
3610 break;
3611 }
3612
3613 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3614 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3615 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3616 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3617 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619
3620 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3621 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3622 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3623 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626
3627 WREG32(SQ_CONFIG, sq_config);
3628 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3629 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3630 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3631 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3632 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3633 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3634 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3635 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3636 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3637 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3638
3639 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3640 FORCE_EOV_MAX_REZ_CNT(255)));
3641
3642 switch (rdev->family) {
3643 case CHIP_CEDAR:
3644 case CHIP_PALM:
3645 case CHIP_SUMO:
3646 case CHIP_SUMO2:
3647 case CHIP_CAICOS:
3648 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3649 break;
3650 default:
3651 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3652 break;
3653 }
3654 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3655 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3656
3657 WREG32(VGT_GS_VERTEX_REUSE, 16);
3658 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3659 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3660
3661 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3662 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3663
3664 WREG32(CB_PERF_CTR0_SEL_0, 0);
3665 WREG32(CB_PERF_CTR0_SEL_1, 0);
3666 WREG32(CB_PERF_CTR1_SEL_0, 0);
3667 WREG32(CB_PERF_CTR1_SEL_1, 0);
3668 WREG32(CB_PERF_CTR2_SEL_0, 0);
3669 WREG32(CB_PERF_CTR2_SEL_1, 0);
3670 WREG32(CB_PERF_CTR3_SEL_0, 0);
3671 WREG32(CB_PERF_CTR3_SEL_1, 0);
3672
3673 /* clear render buffer base addresses */
3674 WREG32(CB_COLOR0_BASE, 0);
3675 WREG32(CB_COLOR1_BASE, 0);
3676 WREG32(CB_COLOR2_BASE, 0);
3677 WREG32(CB_COLOR3_BASE, 0);
3678 WREG32(CB_COLOR4_BASE, 0);
3679 WREG32(CB_COLOR5_BASE, 0);
3680 WREG32(CB_COLOR6_BASE, 0);
3681 WREG32(CB_COLOR7_BASE, 0);
3682 WREG32(CB_COLOR8_BASE, 0);
3683 WREG32(CB_COLOR9_BASE, 0);
3684 WREG32(CB_COLOR10_BASE, 0);
3685 WREG32(CB_COLOR11_BASE, 0);
3686
3687 /* set the shader const cache sizes to 0 */
3688 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3689 WREG32(i, 0);
3690 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3691 WREG32(i, 0);
3692
3693 tmp = RREG32(HDP_MISC_CNTL);
3694 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3695 WREG32(HDP_MISC_CNTL, tmp);
3696
3697 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3698 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3699
3700 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3701
3702 udelay(50);
3703
3704}
3705
3706int evergreen_mc_init(struct radeon_device *rdev)
3707{
3708 u32 tmp;
3709 int chansize, numchan;
3710
3711 /* Get VRAM informations */
3712 rdev->mc.vram_is_ddr = true;
3713 if ((rdev->family == CHIP_PALM) ||
3714 (rdev->family == CHIP_SUMO) ||
3715 (rdev->family == CHIP_SUMO2))
3716 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3717 else
3718 tmp = RREG32(MC_ARB_RAMCFG);
3719 if (tmp & CHANSIZE_OVERRIDE) {
3720 chansize = 16;
3721 } else if (tmp & CHANSIZE_MASK) {
3722 chansize = 64;
3723 } else {
3724 chansize = 32;
3725 }
3726 tmp = RREG32(MC_SHARED_CHMAP);
3727 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3728 case 0:
3729 default:
3730 numchan = 1;
3731 break;
3732 case 1:
3733 numchan = 2;
3734 break;
3735 case 2:
3736 numchan = 4;
3737 break;
3738 case 3:
3739 numchan = 8;
3740 break;
3741 }
3742 rdev->mc.vram_width = numchan * chansize;
3743 /* Could aper size report 0 ? */
3744 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3745 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3746 /* Setup GPU memory space */
3747 if ((rdev->family == CHIP_PALM) ||
3748 (rdev->family == CHIP_SUMO) ||
3749 (rdev->family == CHIP_SUMO2)) {
3750 /* size in bytes on fusion */
3751 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3752 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3753 } else {
3754 /* size in MB on evergreen/cayman/tn */
3755 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3756 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3757 }
3758 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3759 r700_vram_gtt_location(rdev, &rdev->mc);
3760 radeon_update_bandwidth_info(rdev);
3761
3762 return 0;
3763}
3764
3765void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3766{
3767 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3768 RREG32(GRBM_STATUS));
3769 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3770 RREG32(GRBM_STATUS_SE0));
3771 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3772 RREG32(GRBM_STATUS_SE1));
3773 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3774 RREG32(SRBM_STATUS));
3775 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3776 RREG32(SRBM_STATUS2));
3777 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3778 RREG32(CP_STALLED_STAT1));
3779 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3780 RREG32(CP_STALLED_STAT2));
3781 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3782 RREG32(CP_BUSY_STAT));
3783 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3784 RREG32(CP_STAT));
3785 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3786 RREG32(DMA_STATUS_REG));
3787 if (rdev->family >= CHIP_CAYMAN) {
3788 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3789 RREG32(DMA_STATUS_REG + 0x800));
3790 }
3791}
3792
3793bool evergreen_is_display_hung(struct radeon_device *rdev)
3794{
3795 u32 crtc_hung = 0;
3796 u32 crtc_status[6];
3797 u32 i, j, tmp;
3798
3799 for (i = 0; i < rdev->num_crtc; i++) {
3800 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3801 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3802 crtc_hung |= (1 << i);
3803 }
3804 }
3805
3806 for (j = 0; j < 10; j++) {
3807 for (i = 0; i < rdev->num_crtc; i++) {
3808 if (crtc_hung & (1 << i)) {
3809 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3810 if (tmp != crtc_status[i])
3811 crtc_hung &= ~(1 << i);
3812 }
3813 }
3814 if (crtc_hung == 0)
3815 return false;
3816 udelay(100);
3817 }
3818
3819 return true;
3820}
3821
3822u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3823{
3824 u32 reset_mask = 0;
3825 u32 tmp;
3826
3827 /* GRBM_STATUS */
3828 tmp = RREG32(GRBM_STATUS);
3829 if (tmp & (PA_BUSY | SC_BUSY |
3830 SH_BUSY | SX_BUSY |
3831 TA_BUSY | VGT_BUSY |
3832 DB_BUSY | CB_BUSY |
3833 SPI_BUSY | VGT_BUSY_NO_DMA))
3834 reset_mask |= RADEON_RESET_GFX;
3835
3836 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3837 CP_BUSY | CP_COHERENCY_BUSY))
3838 reset_mask |= RADEON_RESET_CP;
3839
3840 if (tmp & GRBM_EE_BUSY)
3841 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3842
3843 /* DMA_STATUS_REG */
3844 tmp = RREG32(DMA_STATUS_REG);
3845 if (!(tmp & DMA_IDLE))
3846 reset_mask |= RADEON_RESET_DMA;
3847
3848 /* SRBM_STATUS2 */
3849 tmp = RREG32(SRBM_STATUS2);
3850 if (tmp & DMA_BUSY)
3851 reset_mask |= RADEON_RESET_DMA;
3852
3853 /* SRBM_STATUS */
3854 tmp = RREG32(SRBM_STATUS);
3855 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3856 reset_mask |= RADEON_RESET_RLC;
3857
3858 if (tmp & IH_BUSY)
3859 reset_mask |= RADEON_RESET_IH;
3860
3861 if (tmp & SEM_BUSY)
3862 reset_mask |= RADEON_RESET_SEM;
3863
3864 if (tmp & GRBM_RQ_PENDING)
3865 reset_mask |= RADEON_RESET_GRBM;
3866
3867 if (tmp & VMC_BUSY)
3868 reset_mask |= RADEON_RESET_VMC;
3869
3870 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3871 MCC_BUSY | MCD_BUSY))
3872 reset_mask |= RADEON_RESET_MC;
3873
3874 if (evergreen_is_display_hung(rdev))
3875 reset_mask |= RADEON_RESET_DISPLAY;
3876
3877 /* VM_L2_STATUS */
3878 tmp = RREG32(VM_L2_STATUS);
3879 if (tmp & L2_BUSY)
3880 reset_mask |= RADEON_RESET_VMC;
3881
3882 /* Skip MC reset as it's mostly likely not hung, just busy */
3883 if (reset_mask & RADEON_RESET_MC) {
3884 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3885 reset_mask &= ~RADEON_RESET_MC;
3886 }
3887
3888 return reset_mask;
3889}
3890
3891static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3892{
3893 struct evergreen_mc_save save;
3894 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3895 u32 tmp;
3896
3897 if (reset_mask == 0)
3898 return;
3899
3900 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3901
3902 evergreen_print_gpu_status_regs(rdev);
3903
3904 /* Disable CP parsing/prefetching */
3905 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3906
3907 if (reset_mask & RADEON_RESET_DMA) {
3908 /* Disable DMA */
3909 tmp = RREG32(DMA_RB_CNTL);
3910 tmp &= ~DMA_RB_ENABLE;
3911 WREG32(DMA_RB_CNTL, tmp);
3912 }
3913
3914 udelay(50);
3915
3916 evergreen_mc_stop(rdev, &save);
3917 if (evergreen_mc_wait_for_idle(rdev)) {
3918 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3919 }
3920
3921 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3922 grbm_soft_reset |= SOFT_RESET_DB |
3923 SOFT_RESET_CB |
3924 SOFT_RESET_PA |
3925 SOFT_RESET_SC |
3926 SOFT_RESET_SPI |
3927 SOFT_RESET_SX |
3928 SOFT_RESET_SH |
3929 SOFT_RESET_TC |
3930 SOFT_RESET_TA |
3931 SOFT_RESET_VC |
3932 SOFT_RESET_VGT;
3933 }
3934
3935 if (reset_mask & RADEON_RESET_CP) {
3936 grbm_soft_reset |= SOFT_RESET_CP |
3937 SOFT_RESET_VGT;
3938
3939 srbm_soft_reset |= SOFT_RESET_GRBM;
3940 }
3941
3942 if (reset_mask & RADEON_RESET_DMA)
3943 srbm_soft_reset |= SOFT_RESET_DMA;
3944
3945 if (reset_mask & RADEON_RESET_DISPLAY)
3946 srbm_soft_reset |= SOFT_RESET_DC;
3947
3948 if (reset_mask & RADEON_RESET_RLC)
3949 srbm_soft_reset |= SOFT_RESET_RLC;
3950
3951 if (reset_mask & RADEON_RESET_SEM)
3952 srbm_soft_reset |= SOFT_RESET_SEM;
3953
3954 if (reset_mask & RADEON_RESET_IH)
3955 srbm_soft_reset |= SOFT_RESET_IH;
3956
3957 if (reset_mask & RADEON_RESET_GRBM)
3958 srbm_soft_reset |= SOFT_RESET_GRBM;
3959
3960 if (reset_mask & RADEON_RESET_VMC)
3961 srbm_soft_reset |= SOFT_RESET_VMC;
3962
3963 if (!(rdev->flags & RADEON_IS_IGP)) {
3964 if (reset_mask & RADEON_RESET_MC)
3965 srbm_soft_reset |= SOFT_RESET_MC;
3966 }
3967
3968 if (grbm_soft_reset) {
3969 tmp = RREG32(GRBM_SOFT_RESET);
3970 tmp |= grbm_soft_reset;
3971 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3972 WREG32(GRBM_SOFT_RESET, tmp);
3973 tmp = RREG32(GRBM_SOFT_RESET);
3974
3975 udelay(50);
3976
3977 tmp &= ~grbm_soft_reset;
3978 WREG32(GRBM_SOFT_RESET, tmp);
3979 tmp = RREG32(GRBM_SOFT_RESET);
3980 }
3981
3982 if (srbm_soft_reset) {
3983 tmp = RREG32(SRBM_SOFT_RESET);
3984 tmp |= srbm_soft_reset;
3985 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3986 WREG32(SRBM_SOFT_RESET, tmp);
3987 tmp = RREG32(SRBM_SOFT_RESET);
3988
3989 udelay(50);
3990
3991 tmp &= ~srbm_soft_reset;
3992 WREG32(SRBM_SOFT_RESET, tmp);
3993 tmp = RREG32(SRBM_SOFT_RESET);
3994 }
3995
3996 /* Wait a little for things to settle down */
3997 udelay(50);
3998
3999 evergreen_mc_resume(rdev, &save);
4000 udelay(50);
4001
4002 evergreen_print_gpu_status_regs(rdev);
4003}
4004
4005void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4006{
4007 struct evergreen_mc_save save;
4008 u32 tmp, i;
4009
4010 dev_info(rdev->dev, "GPU pci config reset\n");
4011
4012 /* disable dpm? */
4013
4014 /* Disable CP parsing/prefetching */
4015 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4016 udelay(50);
4017 /* Disable DMA */
4018 tmp = RREG32(DMA_RB_CNTL);
4019 tmp &= ~DMA_RB_ENABLE;
4020 WREG32(DMA_RB_CNTL, tmp);
4021 /* XXX other engines? */
4022
4023 /* halt the rlc */
4024 r600_rlc_stop(rdev);
4025
4026 udelay(50);
4027
4028 /* set mclk/sclk to bypass */
4029 rv770_set_clk_bypass_mode(rdev);
4030 /* disable BM */
4031 pci_clear_master(rdev->pdev);
4032 /* disable mem access */
4033 evergreen_mc_stop(rdev, &save);
4034 if (evergreen_mc_wait_for_idle(rdev)) {
4035 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4036 }
4037 /* reset */
4038 radeon_pci_config_reset(rdev);
4039 /* wait for asic to come out of reset */
4040 for (i = 0; i < rdev->usec_timeout; i++) {
4041 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4042 break;
4043 udelay(1);
4044 }
4045}
4046
4047int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4048{
4049 u32 reset_mask;
4050
4051 if (hard) {
4052 evergreen_gpu_pci_config_reset(rdev);
4053 return 0;
4054 }
4055
4056 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4057
4058 if (reset_mask)
4059 r600_set_bios_scratch_engine_hung(rdev, true);
4060
4061 /* try soft reset */
4062 evergreen_gpu_soft_reset(rdev, reset_mask);
4063
4064 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4065
4066 /* try pci config reset */
4067 if (reset_mask && radeon_hard_reset)
4068 evergreen_gpu_pci_config_reset(rdev);
4069
4070 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4071
4072 if (!reset_mask)
4073 r600_set_bios_scratch_engine_hung(rdev, false);
4074
4075 return 0;
4076}
4077
4078/**
4079 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4080 *
4081 * @rdev: radeon_device pointer
4082 * @ring: radeon_ring structure holding ring information
4083 *
4084 * Check if the GFX engine is locked up.
4085 * Returns true if the engine appears to be locked up, false if not.
4086 */
4087bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4088{
4089 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4090
4091 if (!(reset_mask & (RADEON_RESET_GFX |
4092 RADEON_RESET_COMPUTE |
4093 RADEON_RESET_CP))) {
4094 radeon_ring_lockup_update(rdev, ring);
4095 return false;
4096 }
4097 return radeon_ring_test_lockup(rdev, ring);
4098}
4099
4100/*
4101 * RLC
4102 */
4103#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4104#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4105
4106void sumo_rlc_fini(struct radeon_device *rdev)
4107{
4108 int r;
4109
4110 /* save restore block */
4111 if (rdev->rlc.save_restore_obj) {
4112 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4113 if (unlikely(r != 0))
4114 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4115 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4116 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4117
4118 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4119 rdev->rlc.save_restore_obj = NULL;
4120 }
4121
4122 /* clear state block */
4123 if (rdev->rlc.clear_state_obj) {
4124 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4125 if (unlikely(r != 0))
4126 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4127 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4128 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4129
4130 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4131 rdev->rlc.clear_state_obj = NULL;
4132 }
4133
4134 /* clear state block */
4135 if (rdev->rlc.cp_table_obj) {
4136 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4137 if (unlikely(r != 0))
4138 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4139 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4140 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4141
4142 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4143 rdev->rlc.cp_table_obj = NULL;
4144 }
4145}
4146
4147#define CP_ME_TABLE_SIZE 96
4148
4149int sumo_rlc_init(struct radeon_device *rdev)
4150{
4151 const u32 *src_ptr;
4152 volatile u32 *dst_ptr;
4153 u32 dws, data, i, j, k, reg_num;
4154 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4155 u64 reg_list_mc_addr;
4156 const struct cs_section_def *cs_data;
4157 int r;
4158
4159 src_ptr = rdev->rlc.reg_list;
4160 dws = rdev->rlc.reg_list_size;
4161 if (rdev->family >= CHIP_BONAIRE) {
4162 dws += (5 * 16) + 48 + 48 + 64;
4163 }
4164 cs_data = rdev->rlc.cs_data;
4165
4166 if (src_ptr) {
4167 /* save restore block */
4168 if (rdev->rlc.save_restore_obj == NULL) {
4169 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4170 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4171 NULL, &rdev->rlc.save_restore_obj);
4172 if (r) {
4173 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4174 return r;
4175 }
4176 }
4177
4178 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4179 if (unlikely(r != 0)) {
4180 sumo_rlc_fini(rdev);
4181 return r;
4182 }
4183 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4184 &rdev->rlc.save_restore_gpu_addr);
4185 if (r) {
4186 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4187 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4188 sumo_rlc_fini(rdev);
4189 return r;
4190 }
4191
4192 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4193 if (r) {
4194 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4195 sumo_rlc_fini(rdev);
4196 return r;
4197 }
4198 /* write the sr buffer */
4199 dst_ptr = rdev->rlc.sr_ptr;
4200 if (rdev->family >= CHIP_TAHITI) {
4201 /* SI */
4202 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4203 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4204 } else {
4205 /* ON/LN/TN */
4206 /* format:
4207 * dw0: (reg2 << 16) | reg1
4208 * dw1: reg1 save space
4209 * dw2: reg2 save space
4210 */
4211 for (i = 0; i < dws; i++) {
4212 data = src_ptr[i] >> 2;
4213 i++;
4214 if (i < dws)
4215 data |= (src_ptr[i] >> 2) << 16;
4216 j = (((i - 1) * 3) / 2);
4217 dst_ptr[j] = cpu_to_le32(data);
4218 }
4219 j = ((i * 3) / 2);
4220 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4221 }
4222 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4223 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4224 }
4225
4226 if (cs_data) {
4227 /* clear state block */
4228 if (rdev->family >= CHIP_BONAIRE) {
4229 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4230 } else if (rdev->family >= CHIP_TAHITI) {
4231 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4232 dws = rdev->rlc.clear_state_size + (256 / 4);
4233 } else {
4234 reg_list_num = 0;
4235 dws = 0;
4236 for (i = 0; cs_data[i].section != NULL; i++) {
4237 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4238 reg_list_num++;
4239 dws += cs_data[i].section[j].reg_count;
4240 }
4241 }
4242 reg_list_blk_index = (3 * reg_list_num + 2);
4243 dws += reg_list_blk_index;
4244 rdev->rlc.clear_state_size = dws;
4245 }
4246
4247 if (rdev->rlc.clear_state_obj == NULL) {
4248 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4249 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4250 NULL, &rdev->rlc.clear_state_obj);
4251 if (r) {
4252 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4253 sumo_rlc_fini(rdev);
4254 return r;
4255 }
4256 }
4257 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4258 if (unlikely(r != 0)) {
4259 sumo_rlc_fini(rdev);
4260 return r;
4261 }
4262 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4263 &rdev->rlc.clear_state_gpu_addr);
4264 if (r) {
4265 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4266 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4267 sumo_rlc_fini(rdev);
4268 return r;
4269 }
4270
4271 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4272 if (r) {
4273 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4274 sumo_rlc_fini(rdev);
4275 return r;
4276 }
4277 /* set up the cs buffer */
4278 dst_ptr = rdev->rlc.cs_ptr;
4279 if (rdev->family >= CHIP_BONAIRE) {
4280 cik_get_csb_buffer(rdev, dst_ptr);
4281 } else if (rdev->family >= CHIP_TAHITI) {
4282 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4283 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4284 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4285 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4286 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4287 } else {
4288 reg_list_hdr_blk_index = 0;
4289 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4290 data = upper_32_bits(reg_list_mc_addr);
4291 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4292 reg_list_hdr_blk_index++;
4293 for (i = 0; cs_data[i].section != NULL; i++) {
4294 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4295 reg_num = cs_data[i].section[j].reg_count;
4296 data = reg_list_mc_addr & 0xffffffff;
4297 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4298 reg_list_hdr_blk_index++;
4299
4300 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4301 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4302 reg_list_hdr_blk_index++;
4303
4304 data = 0x08000000 | (reg_num * 4);
4305 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4306 reg_list_hdr_blk_index++;
4307
4308 for (k = 0; k < reg_num; k++) {
4309 data = cs_data[i].section[j].extent[k];
4310 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4311 }
4312 reg_list_mc_addr += reg_num * 4;
4313 reg_list_blk_index += reg_num;
4314 }
4315 }
4316 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4317 }
4318 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4319 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4320 }
4321
4322 if (rdev->rlc.cp_table_size) {
4323 if (rdev->rlc.cp_table_obj == NULL) {
4324 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4325 PAGE_SIZE, true,
4326 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4327 NULL, &rdev->rlc.cp_table_obj);
4328 if (r) {
4329 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4330 sumo_rlc_fini(rdev);
4331 return r;
4332 }
4333 }
4334
4335 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4336 if (unlikely(r != 0)) {
4337 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4338 sumo_rlc_fini(rdev);
4339 return r;
4340 }
4341 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4342 &rdev->rlc.cp_table_gpu_addr);
4343 if (r) {
4344 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4345 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4346 sumo_rlc_fini(rdev);
4347 return r;
4348 }
4349 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4350 if (r) {
4351 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4352 sumo_rlc_fini(rdev);
4353 return r;
4354 }
4355
4356 cik_init_cp_pg_table(rdev);
4357
4358 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4359 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4360
4361 }
4362
4363 return 0;
4364}
4365
4366static void evergreen_rlc_start(struct radeon_device *rdev)
4367{
4368 u32 mask = RLC_ENABLE;
4369
4370 if (rdev->flags & RADEON_IS_IGP) {
4371 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4372 }
4373
4374 WREG32(RLC_CNTL, mask);
4375}
4376
4377int evergreen_rlc_resume(struct radeon_device *rdev)
4378{
4379 u32 i;
4380 const __be32 *fw_data;
4381
4382 if (!rdev->rlc_fw)
4383 return -EINVAL;
4384
4385 r600_rlc_stop(rdev);
4386
4387 WREG32(RLC_HB_CNTL, 0);
4388
4389 if (rdev->flags & RADEON_IS_IGP) {
4390 if (rdev->family == CHIP_ARUBA) {
4391 u32 always_on_bitmap =
4392 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4393 /* find out the number of active simds */
4394 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4395 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4396 tmp = hweight32(~tmp);
4397 if (tmp == rdev->config.cayman.max_simds_per_se) {
4398 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4399 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4400 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4401 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4402 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4403 }
4404 } else {
4405 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4406 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4407 }
4408 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4409 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4410 } else {
4411 WREG32(RLC_HB_BASE, 0);
4412 WREG32(RLC_HB_RPTR, 0);
4413 WREG32(RLC_HB_WPTR, 0);
4414 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4415 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4416 }
4417 WREG32(RLC_MC_CNTL, 0);
4418 WREG32(RLC_UCODE_CNTL, 0);
4419
4420 fw_data = (const __be32 *)rdev->rlc_fw->data;
4421 if (rdev->family >= CHIP_ARUBA) {
4422 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4423 WREG32(RLC_UCODE_ADDR, i);
4424 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4425 }
4426 } else if (rdev->family >= CHIP_CAYMAN) {
4427 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4428 WREG32(RLC_UCODE_ADDR, i);
4429 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4430 }
4431 } else {
4432 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4433 WREG32(RLC_UCODE_ADDR, i);
4434 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4435 }
4436 }
4437 WREG32(RLC_UCODE_ADDR, 0);
4438
4439 evergreen_rlc_start(rdev);
4440
4441 return 0;
4442}
4443
4444/* Interrupts */
4445
4446u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4447{
4448 if (crtc >= rdev->num_crtc)
4449 return 0;
4450 else
4451 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4452}
4453
4454void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4455{
4456 int i;
4457 u32 tmp;
4458
4459 if (rdev->family >= CHIP_CAYMAN) {
4460 cayman_cp_int_cntl_setup(rdev, 0,
4461 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4462 cayman_cp_int_cntl_setup(rdev, 1, 0);
4463 cayman_cp_int_cntl_setup(rdev, 2, 0);
4464 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4465 WREG32(CAYMAN_DMA1_CNTL, tmp);
4466 } else
4467 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4468 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4469 WREG32(DMA_CNTL, tmp);
4470 WREG32(GRBM_INT_CNTL, 0);
4471 WREG32(SRBM_INT_CNTL, 0);
4472 for (i = 0; i < rdev->num_crtc; i++)
4473 WREG32(INT_MASK + crtc_offsets[i], 0);
4474 for (i = 0; i < rdev->num_crtc; i++)
4475 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4476
4477 /* only one DAC on DCE5 */
4478 if (!ASIC_IS_DCE5(rdev))
4479 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4480 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4481
4482 for (i = 0; i < 6; i++)
4483 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4484}
4485
4486/* Note that the order we write back regs here is important */
4487int evergreen_irq_set(struct radeon_device *rdev)
4488{
4489 int i;
4490 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4491 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4492 u32 grbm_int_cntl = 0;
4493 u32 dma_cntl, dma_cntl1 = 0;
4494 u32 thermal_int = 0;
4495
4496 if (!rdev->irq.installed) {
4497 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4498 return -EINVAL;
4499 }
4500 /* don't enable anything if the ih is disabled */
4501 if (!rdev->ih.enabled) {
4502 r600_disable_interrupts(rdev);
4503 /* force the active interrupt state to all disabled */
4504 evergreen_disable_interrupt_state(rdev);
4505 return 0;
4506 }
4507
4508 if (rdev->family == CHIP_ARUBA)
4509 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4510 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4511 else
4512 thermal_int = RREG32(CG_THERMAL_INT) &
4513 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4514
4515 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4516
4517 if (rdev->family >= CHIP_CAYMAN) {
4518 /* enable CP interrupts on all rings */
4519 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4520 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4521 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4522 }
4523 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4524 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4525 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4526 }
4527 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4528 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4529 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4530 }
4531 } else {
4532 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4533 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4534 cp_int_cntl |= RB_INT_ENABLE;
4535 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4536 }
4537 }
4538
4539 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4540 DRM_DEBUG("r600_irq_set: sw int dma\n");
4541 dma_cntl |= TRAP_ENABLE;
4542 }
4543
4544 if (rdev->family >= CHIP_CAYMAN) {
4545 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4546 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4547 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4548 dma_cntl1 |= TRAP_ENABLE;
4549 }
4550 }
4551
4552 if (rdev->irq.dpm_thermal) {
4553 DRM_DEBUG("dpm thermal\n");
4554 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4555 }
4556
4557 if (rdev->family >= CHIP_CAYMAN) {
4558 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4559 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4560 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4561 } else
4562 WREG32(CP_INT_CNTL, cp_int_cntl);
4563
4564 WREG32(DMA_CNTL, dma_cntl);
4565
4566 if (rdev->family >= CHIP_CAYMAN)
4567 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4568
4569 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4570
4571 for (i = 0; i < rdev->num_crtc; i++) {
4572 radeon_irq_kms_set_irq_n_enabled(
4573 rdev, INT_MASK + crtc_offsets[i],
4574 VBLANK_INT_MASK,
4575 rdev->irq.crtc_vblank_int[i] ||
4576 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4577 }
4578
4579 for (i = 0; i < rdev->num_crtc; i++)
4580 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4581
4582 for (i = 0; i < 6; i++) {
4583 radeon_irq_kms_set_irq_n_enabled(
4584 rdev, DC_HPDx_INT_CONTROL(i),
4585 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4586 rdev->irq.hpd[i], "HPD", i);
4587 }
4588
4589 if (rdev->family == CHIP_ARUBA)
4590 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4591 else
4592 WREG32(CG_THERMAL_INT, thermal_int);
4593
4594 for (i = 0; i < 6; i++) {
4595 radeon_irq_kms_set_irq_n_enabled(
4596 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4597 AFMT_AZ_FORMAT_WTRIG_MASK,
4598 rdev->irq.afmt[i], "HDMI", i);
4599 }
4600
4601 /* posting read */
4602 RREG32(SRBM_STATUS);
4603
4604 return 0;
4605}
4606
4607/* Note that the order we write back regs here is important */
4608static void evergreen_irq_ack(struct radeon_device *rdev)
4609{
4610 int i, j;
4611 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4612 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4613 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4614
4615 for (i = 0; i < 6; i++) {
4616 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4617 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4618 if (i < rdev->num_crtc)
4619 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4620 }
4621
4622 /* We write back each interrupt register in pairs of two */
4623 for (i = 0; i < rdev->num_crtc; i += 2) {
4624 for (j = i; j < (i + 2); j++) {
4625 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4626 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4627 GRPH_PFLIP_INT_CLEAR);
4628 }
4629
4630 for (j = i; j < (i + 2); j++) {
4631 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4632 WREG32(VBLANK_STATUS + crtc_offsets[j],
4633 VBLANK_ACK);
4634 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4635 WREG32(VLINE_STATUS + crtc_offsets[j],
4636 VLINE_ACK);
4637 }
4638 }
4639
4640 for (i = 0; i < 6; i++) {
4641 if (disp_int[i] & DC_HPD1_INTERRUPT)
4642 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4643 }
4644
4645 for (i = 0; i < 6; i++) {
4646 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4647 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4648 }
4649
4650 for (i = 0; i < 6; i++) {
4651 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4652 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4653 AFMT_AZ_FORMAT_WTRIG_ACK);
4654 }
4655}
4656
4657static void evergreen_irq_disable(struct radeon_device *rdev)
4658{
4659 r600_disable_interrupts(rdev);
4660 /* Wait and acknowledge irq */
4661 mdelay(1);
4662 evergreen_irq_ack(rdev);
4663 evergreen_disable_interrupt_state(rdev);
4664}
4665
4666void evergreen_irq_suspend(struct radeon_device *rdev)
4667{
4668 evergreen_irq_disable(rdev);
4669 r600_rlc_stop(rdev);
4670}
4671
4672static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4673{
4674 u32 wptr, tmp;
4675
4676 if (rdev->wb.enabled)
4677 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4678 else
4679 wptr = RREG32(IH_RB_WPTR);
4680
4681 if (wptr & RB_OVERFLOW) {
4682 wptr &= ~RB_OVERFLOW;
4683 /* When a ring buffer overflow happen start parsing interrupt
4684 * from the last not overwritten vector (wptr + 16). Hopefully
4685 * this should allow us to catchup.
4686 */
4687 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4688 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4689 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4690 tmp = RREG32(IH_RB_CNTL);
4691 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4692 WREG32(IH_RB_CNTL, tmp);
4693 }
4694 return (wptr & rdev->ih.ptr_mask);
4695}
4696
4697int evergreen_irq_process(struct radeon_device *rdev)
4698{
4699 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4700 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4701 u32 crtc_idx, hpd_idx, afmt_idx;
4702 u32 mask;
4703 u32 wptr;
4704 u32 rptr;
4705 u32 src_id, src_data;
4706 u32 ring_index;
4707 bool queue_hotplug = false;
4708 bool queue_hdmi = false;
4709 bool queue_dp = false;
4710 bool queue_thermal = false;
4711 u32 status, addr;
4712 const char *event_name;
4713
4714 if (!rdev->ih.enabled || rdev->shutdown)
4715 return IRQ_NONE;
4716
4717 wptr = evergreen_get_ih_wptr(rdev);
4718
4719restart_ih:
4720 /* is somebody else already processing irqs? */
4721 if (atomic_xchg(&rdev->ih.lock, 1))
4722 return IRQ_NONE;
4723
4724 rptr = rdev->ih.rptr;
4725 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4726
4727 /* Order reading of wptr vs. reading of IH ring data */
4728 rmb();
4729
4730 /* display interrupts */
4731 evergreen_irq_ack(rdev);
4732
4733 while (rptr != wptr) {
4734 /* wptr/rptr are in bytes! */
4735 ring_index = rptr / 4;
4736 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4737 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4738
4739 switch (src_id) {
4740 case 1: /* D1 vblank/vline */
4741 case 2: /* D2 vblank/vline */
4742 case 3: /* D3 vblank/vline */
4743 case 4: /* D4 vblank/vline */
4744 case 5: /* D5 vblank/vline */
4745 case 6: /* D6 vblank/vline */
4746 crtc_idx = src_id - 1;
4747
4748 if (src_data == 0) { /* vblank */
4749 mask = LB_D1_VBLANK_INTERRUPT;
4750 event_name = "vblank";
4751
4752 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4753 drm_handle_vblank(rdev->ddev, crtc_idx);
4754 rdev->pm.vblank_sync = true;
4755 wake_up(&rdev->irq.vblank_queue);
4756 }
4757 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4758 radeon_crtc_handle_vblank(rdev,
4759 crtc_idx);
4760 }
4761
4762 } else if (src_data == 1) { /* vline */
4763 mask = LB_D1_VLINE_INTERRUPT;
4764 event_name = "vline";
4765 } else {
4766 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4767 src_id, src_data);
4768 break;
4769 }
4770
4771 if (!(disp_int[crtc_idx] & mask)) {
4772 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4773 crtc_idx + 1, event_name);
4774 }
4775
4776 disp_int[crtc_idx] &= ~mask;
4777 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4778
4779 break;
4780 case 8: /* D1 page flip */
4781 case 10: /* D2 page flip */
4782 case 12: /* D3 page flip */
4783 case 14: /* D4 page flip */
4784 case 16: /* D5 page flip */
4785 case 18: /* D6 page flip */
4786 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4787 if (radeon_use_pflipirq > 0)
4788 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4789 break;
4790 case 42: /* HPD hotplug */
4791 if (src_data <= 5) {
4792 hpd_idx = src_data;
4793 mask = DC_HPD1_INTERRUPT;
4794 queue_hotplug = true;
4795 event_name = "HPD";
4796
4797 } else if (src_data <= 11) {
4798 hpd_idx = src_data - 6;
4799 mask = DC_HPD1_RX_INTERRUPT;
4800 queue_dp = true;
4801 event_name = "HPD_RX";
4802
4803 } else {
4804 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4805 src_id, src_data);
4806 break;
4807 }
4808
4809 if (!(disp_int[hpd_idx] & mask))
4810 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4811
4812 disp_int[hpd_idx] &= ~mask;
4813 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4814
4815 break;
4816 case 44: /* hdmi */
4817 afmt_idx = src_data;
4818 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4819 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4820
4821 if (afmt_idx > 5) {
4822 DRM_ERROR("Unhandled interrupt: %d %d\n",
4823 src_id, src_data);
4824 break;
4825 }
4826 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4827 queue_hdmi = true;
4828 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4829 break;
4830 case 96:
4831 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4832 WREG32(SRBM_INT_ACK, 0x1);
4833 break;
4834 case 124: /* UVD */
4835 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4836 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4837 break;
4838 case 146:
4839 case 147:
4840 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4841 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4842 /* reset addr and status */
4843 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4844 if (addr == 0x0 && status == 0x0)
4845 break;
4846 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4847 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4848 addr);
4849 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4850 status);
4851 cayman_vm_decode_fault(rdev, status, addr);
4852 break;
4853 case 176: /* CP_INT in ring buffer */
4854 case 177: /* CP_INT in IB1 */
4855 case 178: /* CP_INT in IB2 */
4856 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4857 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4858 break;
4859 case 181: /* CP EOP event */
4860 DRM_DEBUG("IH: CP EOP\n");
4861 if (rdev->family >= CHIP_CAYMAN) {
4862 switch (src_data) {
4863 case 0:
4864 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4865 break;
4866 case 1:
4867 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4868 break;
4869 case 2:
4870 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4871 break;
4872 }
4873 } else
4874 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4875 break;
4876 case 224: /* DMA trap event */
4877 DRM_DEBUG("IH: DMA trap\n");
4878 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4879 break;
4880 case 230: /* thermal low to high */
4881 DRM_DEBUG("IH: thermal low to high\n");
4882 rdev->pm.dpm.thermal.high_to_low = false;
4883 queue_thermal = true;
4884 break;
4885 case 231: /* thermal high to low */
4886 DRM_DEBUG("IH: thermal high to low\n");
4887 rdev->pm.dpm.thermal.high_to_low = true;
4888 queue_thermal = true;
4889 break;
4890 case 233: /* GUI IDLE */
4891 DRM_DEBUG("IH: GUI idle\n");
4892 break;
4893 case 244: /* DMA trap event */
4894 if (rdev->family >= CHIP_CAYMAN) {
4895 DRM_DEBUG("IH: DMA1 trap\n");
4896 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4897 }
4898 break;
4899 default:
4900 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4901 break;
4902 }
4903
4904 /* wptr/rptr are in bytes! */
4905 rptr += 16;
4906 rptr &= rdev->ih.ptr_mask;
4907 WREG32(IH_RB_RPTR, rptr);
4908 }
4909 if (queue_dp)
4910 schedule_work(&rdev->dp_work);
4911 if (queue_hotplug)
4912 schedule_delayed_work(&rdev->hotplug_work, 0);
4913 if (queue_hdmi)
4914 schedule_work(&rdev->audio_work);
4915 if (queue_thermal && rdev->pm.dpm_enabled)
4916 schedule_work(&rdev->pm.dpm.thermal.work);
4917 rdev->ih.rptr = rptr;
4918 atomic_set(&rdev->ih.lock, 0);
4919
4920 /* make sure wptr hasn't changed while processing */
4921 wptr = evergreen_get_ih_wptr(rdev);
4922 if (wptr != rptr)
4923 goto restart_ih;
4924
4925 return IRQ_HANDLED;
4926}
4927
4928static void evergreen_uvd_init(struct radeon_device *rdev)
4929{
4930 int r;
4931
4932 if (!rdev->has_uvd)
4933 return;
4934
4935 r = radeon_uvd_init(rdev);
4936 if (r) {
4937 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4938 /*
4939 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4940 * to early fails uvd_v2_2_resume() and thus nothing happens
4941 * there. So it is pointless to try to go through that code
4942 * hence why we disable uvd here.
4943 */
4944 rdev->has_uvd = 0;
4945 return;
4946 }
4947 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4948 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4949}
4950
4951static void evergreen_uvd_start(struct radeon_device *rdev)
4952{
4953 int r;
4954
4955 if (!rdev->has_uvd)
4956 return;
4957
4958 r = uvd_v2_2_resume(rdev);
4959 if (r) {
4960 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4961 goto error;
4962 }
4963 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4964 if (r) {
4965 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4966 goto error;
4967 }
4968 return;
4969
4970error:
4971 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4972}
4973
4974static void evergreen_uvd_resume(struct radeon_device *rdev)
4975{
4976 struct radeon_ring *ring;
4977 int r;
4978
4979 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4980 return;
4981
4982 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4983 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4984 if (r) {
4985 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4986 return;
4987 }
4988 r = uvd_v1_0_init(rdev);
4989 if (r) {
4990 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4991 return;
4992 }
4993}
4994
4995static int evergreen_startup(struct radeon_device *rdev)
4996{
4997 struct radeon_ring *ring;
4998 int r;
4999
5000 /* enable pcie gen2 link */
5001 evergreen_pcie_gen2_enable(rdev);
5002 /* enable aspm */
5003 evergreen_program_aspm(rdev);
5004
5005 /* scratch needs to be initialized before MC */
5006 r = r600_vram_scratch_init(rdev);
5007 if (r)
5008 return r;
5009
5010 evergreen_mc_program(rdev);
5011
5012 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5013 r = ni_mc_load_microcode(rdev);
5014 if (r) {
5015 DRM_ERROR("Failed to load MC firmware!\n");
5016 return r;
5017 }
5018 }
5019
5020 if (rdev->flags & RADEON_IS_AGP) {
5021 evergreen_agp_enable(rdev);
5022 } else {
5023 r = evergreen_pcie_gart_enable(rdev);
5024 if (r)
5025 return r;
5026 }
5027 evergreen_gpu_init(rdev);
5028
5029 /* allocate rlc buffers */
5030 if (rdev->flags & RADEON_IS_IGP) {
5031 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5032 rdev->rlc.reg_list_size =
5033 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5034 rdev->rlc.cs_data = evergreen_cs_data;
5035 r = sumo_rlc_init(rdev);
5036 if (r) {
5037 DRM_ERROR("Failed to init rlc BOs!\n");
5038 return r;
5039 }
5040 }
5041
5042 /* allocate wb buffer */
5043 r = radeon_wb_init(rdev);
5044 if (r)
5045 return r;
5046
5047 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5048 if (r) {
5049 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5050 return r;
5051 }
5052
5053 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5054 if (r) {
5055 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5056 return r;
5057 }
5058
5059 evergreen_uvd_start(rdev);
5060
5061 /* Enable IRQ */
5062 if (!rdev->irq.installed) {
5063 r = radeon_irq_kms_init(rdev);
5064 if (r)
5065 return r;
5066 }
5067
5068 r = r600_irq_init(rdev);
5069 if (r) {
5070 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5071 radeon_irq_kms_fini(rdev);
5072 return r;
5073 }
5074 evergreen_irq_set(rdev);
5075
5076 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5077 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5078 RADEON_CP_PACKET2);
5079 if (r)
5080 return r;
5081
5082 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5083 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5084 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5085 if (r)
5086 return r;
5087
5088 r = evergreen_cp_load_microcode(rdev);
5089 if (r)
5090 return r;
5091 r = evergreen_cp_resume(rdev);
5092 if (r)
5093 return r;
5094 r = r600_dma_resume(rdev);
5095 if (r)
5096 return r;
5097
5098 evergreen_uvd_resume(rdev);
5099
5100 r = radeon_ib_pool_init(rdev);
5101 if (r) {
5102 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5103 return r;
5104 }
5105
5106 r = radeon_audio_init(rdev);
5107 if (r) {
5108 DRM_ERROR("radeon: audio init failed\n");
5109 return r;
5110 }
5111
5112 return 0;
5113}
5114
5115int evergreen_resume(struct radeon_device *rdev)
5116{
5117 int r;
5118
5119 /* reset the asic, the gfx blocks are often in a bad state
5120 * after the driver is unloaded or after a resume
5121 */
5122 if (radeon_asic_reset(rdev))
5123 dev_warn(rdev->dev, "GPU reset failed !\n");
5124 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5125 * posting will perform necessary task to bring back GPU into good
5126 * shape.
5127 */
5128 /* post card */
5129 atom_asic_init(rdev->mode_info.atom_context);
5130
5131 /* init golden registers */
5132 evergreen_init_golden_registers(rdev);
5133
5134 if (rdev->pm.pm_method == PM_METHOD_DPM)
5135 radeon_pm_resume(rdev);
5136
5137 rdev->accel_working = true;
5138 r = evergreen_startup(rdev);
5139 if (r) {
5140 DRM_ERROR("evergreen startup failed on resume\n");
5141 rdev->accel_working = false;
5142 return r;
5143 }
5144
5145 return r;
5146
5147}
5148
5149int evergreen_suspend(struct radeon_device *rdev)
5150{
5151 radeon_pm_suspend(rdev);
5152 radeon_audio_fini(rdev);
5153 if (rdev->has_uvd) {
5154 uvd_v1_0_fini(rdev);
5155 radeon_uvd_suspend(rdev);
5156 }
5157 r700_cp_stop(rdev);
5158 r600_dma_stop(rdev);
5159 evergreen_irq_suspend(rdev);
5160 radeon_wb_disable(rdev);
5161 evergreen_pcie_gart_disable(rdev);
5162
5163 return 0;
5164}
5165
5166/* Plan is to move initialization in that function and use
5167 * helper function so that radeon_device_init pretty much
5168 * do nothing more than calling asic specific function. This
5169 * should also allow to remove a bunch of callback function
5170 * like vram_info.
5171 */
5172int evergreen_init(struct radeon_device *rdev)
5173{
5174 int r;
5175
5176 /* Read BIOS */
5177 if (!radeon_get_bios(rdev)) {
5178 if (ASIC_IS_AVIVO(rdev))
5179 return -EINVAL;
5180 }
5181 /* Must be an ATOMBIOS */
5182 if (!rdev->is_atom_bios) {
5183 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5184 return -EINVAL;
5185 }
5186 r = radeon_atombios_init(rdev);
5187 if (r)
5188 return r;
5189 /* reset the asic, the gfx blocks are often in a bad state
5190 * after the driver is unloaded or after a resume
5191 */
5192 if (radeon_asic_reset(rdev))
5193 dev_warn(rdev->dev, "GPU reset failed !\n");
5194 /* Post card if necessary */
5195 if (!radeon_card_posted(rdev)) {
5196 if (!rdev->bios) {
5197 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5198 return -EINVAL;
5199 }
5200 DRM_INFO("GPU not posted. posting now...\n");
5201 atom_asic_init(rdev->mode_info.atom_context);
5202 }
5203 /* init golden registers */
5204 evergreen_init_golden_registers(rdev);
5205 /* Initialize scratch registers */
5206 r600_scratch_init(rdev);
5207 /* Initialize surface registers */
5208 radeon_surface_init(rdev);
5209 /* Initialize clocks */
5210 radeon_get_clock_info(rdev->ddev);
5211 /* Fence driver */
5212 r = radeon_fence_driver_init(rdev);
5213 if (r)
5214 return r;
5215 /* initialize AGP */
5216 if (rdev->flags & RADEON_IS_AGP) {
5217 r = radeon_agp_init(rdev);
5218 if (r)
5219 radeon_agp_disable(rdev);
5220 }
5221 /* initialize memory controller */
5222 r = evergreen_mc_init(rdev);
5223 if (r)
5224 return r;
5225 /* Memory manager */
5226 r = radeon_bo_init(rdev);
5227 if (r)
5228 return r;
5229
5230 if (ASIC_IS_DCE5(rdev)) {
5231 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5232 r = ni_init_microcode(rdev);
5233 if (r) {
5234 DRM_ERROR("Failed to load firmware!\n");
5235 return r;
5236 }
5237 }
5238 } else {
5239 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5240 r = r600_init_microcode(rdev);
5241 if (r) {
5242 DRM_ERROR("Failed to load firmware!\n");
5243 return r;
5244 }
5245 }
5246 }
5247
5248 /* Initialize power management */
5249 radeon_pm_init(rdev);
5250
5251 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5252 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5253
5254 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5255 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5256
5257 evergreen_uvd_init(rdev);
5258
5259 rdev->ih.ring_obj = NULL;
5260 r600_ih_ring_init(rdev, 64 * 1024);
5261
5262 r = r600_pcie_gart_init(rdev);
5263 if (r)
5264 return r;
5265
5266 rdev->accel_working = true;
5267 r = evergreen_startup(rdev);
5268 if (r) {
5269 dev_err(rdev->dev, "disabling GPU acceleration\n");
5270 r700_cp_fini(rdev);
5271 r600_dma_fini(rdev);
5272 r600_irq_fini(rdev);
5273 if (rdev->flags & RADEON_IS_IGP)
5274 sumo_rlc_fini(rdev);
5275 radeon_wb_fini(rdev);
5276 radeon_ib_pool_fini(rdev);
5277 radeon_irq_kms_fini(rdev);
5278 evergreen_pcie_gart_fini(rdev);
5279 rdev->accel_working = false;
5280 }
5281
5282 /* Don't start up if the MC ucode is missing on BTC parts.
5283 * The default clocks and voltages before the MC ucode
5284 * is loaded are not suffient for advanced operations.
5285 */
5286 if (ASIC_IS_DCE5(rdev)) {
5287 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5288 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5289 return -EINVAL;
5290 }
5291 }
5292
5293 return 0;
5294}
5295
5296void evergreen_fini(struct radeon_device *rdev)
5297{
5298 radeon_pm_fini(rdev);
5299 radeon_audio_fini(rdev);
5300 r700_cp_fini(rdev);
5301 r600_dma_fini(rdev);
5302 r600_irq_fini(rdev);
5303 if (rdev->flags & RADEON_IS_IGP)
5304 sumo_rlc_fini(rdev);
5305 radeon_wb_fini(rdev);
5306 radeon_ib_pool_fini(rdev);
5307 radeon_irq_kms_fini(rdev);
5308 uvd_v1_0_fini(rdev);
5309 radeon_uvd_fini(rdev);
5310 evergreen_pcie_gart_fini(rdev);
5311 r600_vram_scratch_fini(rdev);
5312 radeon_gem_fini(rdev);
5313 radeon_fence_driver_fini(rdev);
5314 radeon_agp_fini(rdev);
5315 radeon_bo_fini(rdev);
5316 radeon_atombios_fini(rdev);
5317 kfree(rdev->bios);
5318 rdev->bios = NULL;
5319}
5320
5321void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5322{
5323 u32 link_width_cntl, speed_cntl;
5324
5325 if (radeon_pcie_gen2 == 0)
5326 return;
5327
5328 if (rdev->flags & RADEON_IS_IGP)
5329 return;
5330
5331 if (!(rdev->flags & RADEON_IS_PCIE))
5332 return;
5333
5334 /* x2 cards have a special sequence */
5335 if (ASIC_IS_X2(rdev))
5336 return;
5337
5338 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5339 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5340 return;
5341
5342 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5343 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5344 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5345 return;
5346 }
5347
5348 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5349
5350 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5351 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5352
5353 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5354 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5355 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5356
5357 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5358 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5359 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5360
5361 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5362 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5363 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5364
5365 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5366 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5367 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5368
5369 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5370 speed_cntl |= LC_GEN2_EN_STRAP;
5371 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5372
5373 } else {
5374 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5375 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5376 if (1)
5377 link_width_cntl |= LC_UPCONFIGURE_DIS;
5378 else
5379 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5380 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5381 }
5382}
5383
5384void evergreen_program_aspm(struct radeon_device *rdev)
5385{
5386 u32 data, orig;
5387 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5388 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5389 /* fusion_platform = true
5390 * if the system is a fusion system
5391 * (APU or DGPU in a fusion system).
5392 * todo: check if the system is a fusion platform.
5393 */
5394 bool fusion_platform = false;
5395
5396 if (radeon_aspm == 0)
5397 return;
5398
5399 if (!(rdev->flags & RADEON_IS_PCIE))
5400 return;
5401
5402 switch (rdev->family) {
5403 case CHIP_CYPRESS:
5404 case CHIP_HEMLOCK:
5405 case CHIP_JUNIPER:
5406 case CHIP_REDWOOD:
5407 case CHIP_CEDAR:
5408 case CHIP_SUMO:
5409 case CHIP_SUMO2:
5410 case CHIP_PALM:
5411 case CHIP_ARUBA:
5412 disable_l0s = true;
5413 break;
5414 default:
5415 disable_l0s = false;
5416 break;
5417 }
5418
5419 if (rdev->flags & RADEON_IS_IGP)
5420 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5421
5422 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5423 if (fusion_platform)
5424 data &= ~MULTI_PIF;
5425 else
5426 data |= MULTI_PIF;
5427 if (data != orig)
5428 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5429
5430 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5431 if (fusion_platform)
5432 data &= ~MULTI_PIF;
5433 else
5434 data |= MULTI_PIF;
5435 if (data != orig)
5436 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5437
5438 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5439 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5440 if (!disable_l0s) {
5441 if (rdev->family >= CHIP_BARTS)
5442 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5443 else
5444 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5445 }
5446
5447 if (!disable_l1) {
5448 if (rdev->family >= CHIP_BARTS)
5449 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5450 else
5451 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5452
5453 if (!disable_plloff_in_l1) {
5454 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5455 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5456 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5457 if (data != orig)
5458 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5459
5460 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5461 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5462 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5463 if (data != orig)
5464 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5465
5466 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5467 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5468 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5469 if (data != orig)
5470 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5471
5472 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5473 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5474 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5475 if (data != orig)
5476 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5477
5478 if (rdev->family >= CHIP_BARTS) {
5479 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5480 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5481 data |= PLL_RAMP_UP_TIME_0(4);
5482 if (data != orig)
5483 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5484
5485 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5486 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5487 data |= PLL_RAMP_UP_TIME_1(4);
5488 if (data != orig)
5489 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5490
5491 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5492 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5493 data |= PLL_RAMP_UP_TIME_0(4);
5494 if (data != orig)
5495 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5496
5497 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5498 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5499 data |= PLL_RAMP_UP_TIME_1(4);
5500 if (data != orig)
5501 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5502 }
5503
5504 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5505 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5506 data |= LC_DYN_LANES_PWR_STATE(3);
5507 if (data != orig)
5508 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5509
5510 if (rdev->family >= CHIP_BARTS) {
5511 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5512 data &= ~LS2_EXIT_TIME_MASK;
5513 data |= LS2_EXIT_TIME(1);
5514 if (data != orig)
5515 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5516
5517 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5518 data &= ~LS2_EXIT_TIME_MASK;
5519 data |= LS2_EXIT_TIME(1);
5520 if (data != orig)
5521 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5522 }
5523 }
5524 }
5525
5526 /* evergreen parts only */
5527 if (rdev->family < CHIP_BARTS)
5528 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5529
5530 if (pcie_lc_cntl != pcie_lc_cntl_old)
5531 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5532}