Loading...
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include "drmP.h"
28#include "radeon.h"
29#include "radeon_asic.h"
30#include "radeon_drm.h"
31#include "evergreend.h"
32#include "atom.h"
33#include "avivod.h"
34#include "evergreen_reg.h"
35#include "evergreen_blit_shaders.h"
36
37#define EVERGREEN_PFP_UCODE_SIZE 1120
38#define EVERGREEN_PM4_UCODE_SIZE 1376
39
40static void evergreen_gpu_init(struct radeon_device *rdev);
41void evergreen_fini(struct radeon_device *rdev);
42static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
43
44void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
45{
46 u16 ctl, v;
47 int cap, err;
48
49 cap = pci_pcie_cap(rdev->pdev);
50 if (!cap)
51 return;
52
53 err = pci_read_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, &ctl);
54 if (err)
55 return;
56
57 v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
58
59 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
60 * to avoid hangs or perfomance issues
61 */
62 if ((v == 0) || (v == 6) || (v == 7)) {
63 ctl &= ~PCI_EXP_DEVCTL_READRQ;
64 ctl |= (2 << 12);
65 pci_write_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, ctl);
66 }
67}
68
69void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
70{
71 /* enable the pflip int */
72 radeon_irq_kms_pflip_irq_get(rdev, crtc);
73}
74
75void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
76{
77 /* disable the pflip int */
78 radeon_irq_kms_pflip_irq_put(rdev, crtc);
79}
80
81u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
82{
83 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
84 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
85
86 /* Lock the graphics update lock */
87 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
88 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
89
90 /* update the scanout addresses */
91 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
92 upper_32_bits(crtc_base));
93 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
94 (u32)crtc_base);
95
96 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
97 upper_32_bits(crtc_base));
98 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
99 (u32)crtc_base);
100
101 /* Wait for update_pending to go high. */
102 while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
103 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
104
105 /* Unlock the lock, so double-buffering can take place inside vblank */
106 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
107 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
108
109 /* Return current update_pending status: */
110 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
111}
112
113/* get temperature in millidegrees */
114int evergreen_get_temp(struct radeon_device *rdev)
115{
116 u32 temp, toffset;
117 int actual_temp = 0;
118
119 if (rdev->family == CHIP_JUNIPER) {
120 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
121 TOFFSET_SHIFT;
122 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
123 TS0_ADC_DOUT_SHIFT;
124
125 if (toffset & 0x100)
126 actual_temp = temp / 2 - (0x200 - toffset);
127 else
128 actual_temp = temp / 2 + toffset;
129
130 actual_temp = actual_temp * 1000;
131
132 } else {
133 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
134 ASIC_T_SHIFT;
135
136 if (temp & 0x400)
137 actual_temp = -256;
138 else if (temp & 0x200)
139 actual_temp = 255;
140 else if (temp & 0x100) {
141 actual_temp = temp & 0x1ff;
142 actual_temp |= ~0x1ff;
143 } else
144 actual_temp = temp & 0xff;
145
146 actual_temp = (actual_temp * 1000) / 2;
147 }
148
149 return actual_temp;
150}
151
152int sumo_get_temp(struct radeon_device *rdev)
153{
154 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
155 int actual_temp = temp - 49;
156
157 return actual_temp * 1000;
158}
159
160void evergreen_pm_misc(struct radeon_device *rdev)
161{
162 int req_ps_idx = rdev->pm.requested_power_state_index;
163 int req_cm_idx = rdev->pm.requested_clock_mode_index;
164 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
165 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
166
167 if (voltage->type == VOLTAGE_SW) {
168 /* 0xff01 is a flag rather then an actual voltage */
169 if (voltage->voltage == 0xff01)
170 return;
171 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
172 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
173 rdev->pm.current_vddc = voltage->voltage;
174 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
175 }
176 /* 0xff01 is a flag rather then an actual voltage */
177 if (voltage->vddci == 0xff01)
178 return;
179 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
180 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
181 rdev->pm.current_vddci = voltage->vddci;
182 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
183 }
184 }
185}
186
187void evergreen_pm_prepare(struct radeon_device *rdev)
188{
189 struct drm_device *ddev = rdev->ddev;
190 struct drm_crtc *crtc;
191 struct radeon_crtc *radeon_crtc;
192 u32 tmp;
193
194 /* disable any active CRTCs */
195 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
196 radeon_crtc = to_radeon_crtc(crtc);
197 if (radeon_crtc->enabled) {
198 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
199 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
200 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
201 }
202 }
203}
204
205void evergreen_pm_finish(struct radeon_device *rdev)
206{
207 struct drm_device *ddev = rdev->ddev;
208 struct drm_crtc *crtc;
209 struct radeon_crtc *radeon_crtc;
210 u32 tmp;
211
212 /* enable any active CRTCs */
213 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
214 radeon_crtc = to_radeon_crtc(crtc);
215 if (radeon_crtc->enabled) {
216 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
217 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
218 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
219 }
220 }
221}
222
223bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
224{
225 bool connected = false;
226
227 switch (hpd) {
228 case RADEON_HPD_1:
229 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
230 connected = true;
231 break;
232 case RADEON_HPD_2:
233 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
234 connected = true;
235 break;
236 case RADEON_HPD_3:
237 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
238 connected = true;
239 break;
240 case RADEON_HPD_4:
241 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
242 connected = true;
243 break;
244 case RADEON_HPD_5:
245 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
246 connected = true;
247 break;
248 case RADEON_HPD_6:
249 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
250 connected = true;
251 break;
252 default:
253 break;
254 }
255
256 return connected;
257}
258
259void evergreen_hpd_set_polarity(struct radeon_device *rdev,
260 enum radeon_hpd_id hpd)
261{
262 u32 tmp;
263 bool connected = evergreen_hpd_sense(rdev, hpd);
264
265 switch (hpd) {
266 case RADEON_HPD_1:
267 tmp = RREG32(DC_HPD1_INT_CONTROL);
268 if (connected)
269 tmp &= ~DC_HPDx_INT_POLARITY;
270 else
271 tmp |= DC_HPDx_INT_POLARITY;
272 WREG32(DC_HPD1_INT_CONTROL, tmp);
273 break;
274 case RADEON_HPD_2:
275 tmp = RREG32(DC_HPD2_INT_CONTROL);
276 if (connected)
277 tmp &= ~DC_HPDx_INT_POLARITY;
278 else
279 tmp |= DC_HPDx_INT_POLARITY;
280 WREG32(DC_HPD2_INT_CONTROL, tmp);
281 break;
282 case RADEON_HPD_3:
283 tmp = RREG32(DC_HPD3_INT_CONTROL);
284 if (connected)
285 tmp &= ~DC_HPDx_INT_POLARITY;
286 else
287 tmp |= DC_HPDx_INT_POLARITY;
288 WREG32(DC_HPD3_INT_CONTROL, tmp);
289 break;
290 case RADEON_HPD_4:
291 tmp = RREG32(DC_HPD4_INT_CONTROL);
292 if (connected)
293 tmp &= ~DC_HPDx_INT_POLARITY;
294 else
295 tmp |= DC_HPDx_INT_POLARITY;
296 WREG32(DC_HPD4_INT_CONTROL, tmp);
297 break;
298 case RADEON_HPD_5:
299 tmp = RREG32(DC_HPD5_INT_CONTROL);
300 if (connected)
301 tmp &= ~DC_HPDx_INT_POLARITY;
302 else
303 tmp |= DC_HPDx_INT_POLARITY;
304 WREG32(DC_HPD5_INT_CONTROL, tmp);
305 break;
306 case RADEON_HPD_6:
307 tmp = RREG32(DC_HPD6_INT_CONTROL);
308 if (connected)
309 tmp &= ~DC_HPDx_INT_POLARITY;
310 else
311 tmp |= DC_HPDx_INT_POLARITY;
312 WREG32(DC_HPD6_INT_CONTROL, tmp);
313 break;
314 default:
315 break;
316 }
317}
318
319void evergreen_hpd_init(struct radeon_device *rdev)
320{
321 struct drm_device *dev = rdev->ddev;
322 struct drm_connector *connector;
323 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
324 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
325
326 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
327 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
328 switch (radeon_connector->hpd.hpd) {
329 case RADEON_HPD_1:
330 WREG32(DC_HPD1_CONTROL, tmp);
331 rdev->irq.hpd[0] = true;
332 break;
333 case RADEON_HPD_2:
334 WREG32(DC_HPD2_CONTROL, tmp);
335 rdev->irq.hpd[1] = true;
336 break;
337 case RADEON_HPD_3:
338 WREG32(DC_HPD3_CONTROL, tmp);
339 rdev->irq.hpd[2] = true;
340 break;
341 case RADEON_HPD_4:
342 WREG32(DC_HPD4_CONTROL, tmp);
343 rdev->irq.hpd[3] = true;
344 break;
345 case RADEON_HPD_5:
346 WREG32(DC_HPD5_CONTROL, tmp);
347 rdev->irq.hpd[4] = true;
348 break;
349 case RADEON_HPD_6:
350 WREG32(DC_HPD6_CONTROL, tmp);
351 rdev->irq.hpd[5] = true;
352 break;
353 default:
354 break;
355 }
356 }
357 if (rdev->irq.installed)
358 evergreen_irq_set(rdev);
359}
360
361void evergreen_hpd_fini(struct radeon_device *rdev)
362{
363 struct drm_device *dev = rdev->ddev;
364 struct drm_connector *connector;
365
366 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
367 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
368 switch (radeon_connector->hpd.hpd) {
369 case RADEON_HPD_1:
370 WREG32(DC_HPD1_CONTROL, 0);
371 rdev->irq.hpd[0] = false;
372 break;
373 case RADEON_HPD_2:
374 WREG32(DC_HPD2_CONTROL, 0);
375 rdev->irq.hpd[1] = false;
376 break;
377 case RADEON_HPD_3:
378 WREG32(DC_HPD3_CONTROL, 0);
379 rdev->irq.hpd[2] = false;
380 break;
381 case RADEON_HPD_4:
382 WREG32(DC_HPD4_CONTROL, 0);
383 rdev->irq.hpd[3] = false;
384 break;
385 case RADEON_HPD_5:
386 WREG32(DC_HPD5_CONTROL, 0);
387 rdev->irq.hpd[4] = false;
388 break;
389 case RADEON_HPD_6:
390 WREG32(DC_HPD6_CONTROL, 0);
391 rdev->irq.hpd[5] = false;
392 break;
393 default:
394 break;
395 }
396 }
397}
398
399/* watermark setup */
400
401static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
402 struct radeon_crtc *radeon_crtc,
403 struct drm_display_mode *mode,
404 struct drm_display_mode *other_mode)
405{
406 u32 tmp;
407 /*
408 * Line Buffer Setup
409 * There are 3 line buffers, each one shared by 2 display controllers.
410 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
411 * the display controllers. The paritioning is done via one of four
412 * preset allocations specified in bits 2:0:
413 * first display controller
414 * 0 - first half of lb (3840 * 2)
415 * 1 - first 3/4 of lb (5760 * 2)
416 * 2 - whole lb (7680 * 2), other crtc must be disabled
417 * 3 - first 1/4 of lb (1920 * 2)
418 * second display controller
419 * 4 - second half of lb (3840 * 2)
420 * 5 - second 3/4 of lb (5760 * 2)
421 * 6 - whole lb (7680 * 2), other crtc must be disabled
422 * 7 - last 1/4 of lb (1920 * 2)
423 */
424 /* this can get tricky if we have two large displays on a paired group
425 * of crtcs. Ideally for multiple large displays we'd assign them to
426 * non-linked crtcs for maximum line buffer allocation.
427 */
428 if (radeon_crtc->base.enabled && mode) {
429 if (other_mode)
430 tmp = 0; /* 1/2 */
431 else
432 tmp = 2; /* whole */
433 } else
434 tmp = 0;
435
436 /* second controller of the pair uses second half of the lb */
437 if (radeon_crtc->crtc_id % 2)
438 tmp += 4;
439 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
440
441 if (radeon_crtc->base.enabled && mode) {
442 switch (tmp) {
443 case 0:
444 case 4:
445 default:
446 if (ASIC_IS_DCE5(rdev))
447 return 4096 * 2;
448 else
449 return 3840 * 2;
450 case 1:
451 case 5:
452 if (ASIC_IS_DCE5(rdev))
453 return 6144 * 2;
454 else
455 return 5760 * 2;
456 case 2:
457 case 6:
458 if (ASIC_IS_DCE5(rdev))
459 return 8192 * 2;
460 else
461 return 7680 * 2;
462 case 3:
463 case 7:
464 if (ASIC_IS_DCE5(rdev))
465 return 2048 * 2;
466 else
467 return 1920 * 2;
468 }
469 }
470
471 /* controller not enabled, so no lb used */
472 return 0;
473}
474
475static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
476{
477 u32 tmp = RREG32(MC_SHARED_CHMAP);
478
479 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
480 case 0:
481 default:
482 return 1;
483 case 1:
484 return 2;
485 case 2:
486 return 4;
487 case 3:
488 return 8;
489 }
490}
491
492struct evergreen_wm_params {
493 u32 dram_channels; /* number of dram channels */
494 u32 yclk; /* bandwidth per dram data pin in kHz */
495 u32 sclk; /* engine clock in kHz */
496 u32 disp_clk; /* display clock in kHz */
497 u32 src_width; /* viewport width */
498 u32 active_time; /* active display time in ns */
499 u32 blank_time; /* blank time in ns */
500 bool interlaced; /* mode is interlaced */
501 fixed20_12 vsc; /* vertical scale ratio */
502 u32 num_heads; /* number of active crtcs */
503 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
504 u32 lb_size; /* line buffer allocated to pipe */
505 u32 vtaps; /* vertical scaler taps */
506};
507
508static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
509{
510 /* Calculate DRAM Bandwidth and the part allocated to display. */
511 fixed20_12 dram_efficiency; /* 0.7 */
512 fixed20_12 yclk, dram_channels, bandwidth;
513 fixed20_12 a;
514
515 a.full = dfixed_const(1000);
516 yclk.full = dfixed_const(wm->yclk);
517 yclk.full = dfixed_div(yclk, a);
518 dram_channels.full = dfixed_const(wm->dram_channels * 4);
519 a.full = dfixed_const(10);
520 dram_efficiency.full = dfixed_const(7);
521 dram_efficiency.full = dfixed_div(dram_efficiency, a);
522 bandwidth.full = dfixed_mul(dram_channels, yclk);
523 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
524
525 return dfixed_trunc(bandwidth);
526}
527
528static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
529{
530 /* Calculate DRAM Bandwidth and the part allocated to display. */
531 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
532 fixed20_12 yclk, dram_channels, bandwidth;
533 fixed20_12 a;
534
535 a.full = dfixed_const(1000);
536 yclk.full = dfixed_const(wm->yclk);
537 yclk.full = dfixed_div(yclk, a);
538 dram_channels.full = dfixed_const(wm->dram_channels * 4);
539 a.full = dfixed_const(10);
540 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
541 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
542 bandwidth.full = dfixed_mul(dram_channels, yclk);
543 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
544
545 return dfixed_trunc(bandwidth);
546}
547
548static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
549{
550 /* Calculate the display Data return Bandwidth */
551 fixed20_12 return_efficiency; /* 0.8 */
552 fixed20_12 sclk, bandwidth;
553 fixed20_12 a;
554
555 a.full = dfixed_const(1000);
556 sclk.full = dfixed_const(wm->sclk);
557 sclk.full = dfixed_div(sclk, a);
558 a.full = dfixed_const(10);
559 return_efficiency.full = dfixed_const(8);
560 return_efficiency.full = dfixed_div(return_efficiency, a);
561 a.full = dfixed_const(32);
562 bandwidth.full = dfixed_mul(a, sclk);
563 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
564
565 return dfixed_trunc(bandwidth);
566}
567
568static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
569{
570 /* Calculate the DMIF Request Bandwidth */
571 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
572 fixed20_12 disp_clk, bandwidth;
573 fixed20_12 a;
574
575 a.full = dfixed_const(1000);
576 disp_clk.full = dfixed_const(wm->disp_clk);
577 disp_clk.full = dfixed_div(disp_clk, a);
578 a.full = dfixed_const(10);
579 disp_clk_request_efficiency.full = dfixed_const(8);
580 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
581 a.full = dfixed_const(32);
582 bandwidth.full = dfixed_mul(a, disp_clk);
583 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
584
585 return dfixed_trunc(bandwidth);
586}
587
588static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
589{
590 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
591 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
592 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
593 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
594
595 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
596}
597
598static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
599{
600 /* Calculate the display mode Average Bandwidth
601 * DisplayMode should contain the source and destination dimensions,
602 * timing, etc.
603 */
604 fixed20_12 bpp;
605 fixed20_12 line_time;
606 fixed20_12 src_width;
607 fixed20_12 bandwidth;
608 fixed20_12 a;
609
610 a.full = dfixed_const(1000);
611 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
612 line_time.full = dfixed_div(line_time, a);
613 bpp.full = dfixed_const(wm->bytes_per_pixel);
614 src_width.full = dfixed_const(wm->src_width);
615 bandwidth.full = dfixed_mul(src_width, bpp);
616 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
617 bandwidth.full = dfixed_div(bandwidth, line_time);
618
619 return dfixed_trunc(bandwidth);
620}
621
622static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
623{
624 /* First calcualte the latency in ns */
625 u32 mc_latency = 2000; /* 2000 ns. */
626 u32 available_bandwidth = evergreen_available_bandwidth(wm);
627 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
628 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
629 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
630 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
631 (wm->num_heads * cursor_line_pair_return_time);
632 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
633 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
634 fixed20_12 a, b, c;
635
636 if (wm->num_heads == 0)
637 return 0;
638
639 a.full = dfixed_const(2);
640 b.full = dfixed_const(1);
641 if ((wm->vsc.full > a.full) ||
642 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
643 (wm->vtaps >= 5) ||
644 ((wm->vsc.full >= a.full) && wm->interlaced))
645 max_src_lines_per_dst_line = 4;
646 else
647 max_src_lines_per_dst_line = 2;
648
649 a.full = dfixed_const(available_bandwidth);
650 b.full = dfixed_const(wm->num_heads);
651 a.full = dfixed_div(a, b);
652
653 b.full = dfixed_const(1000);
654 c.full = dfixed_const(wm->disp_clk);
655 b.full = dfixed_div(c, b);
656 c.full = dfixed_const(wm->bytes_per_pixel);
657 b.full = dfixed_mul(b, c);
658
659 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
660
661 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
662 b.full = dfixed_const(1000);
663 c.full = dfixed_const(lb_fill_bw);
664 b.full = dfixed_div(c, b);
665 a.full = dfixed_div(a, b);
666 line_fill_time = dfixed_trunc(a);
667
668 if (line_fill_time < wm->active_time)
669 return latency;
670 else
671 return latency + (line_fill_time - wm->active_time);
672
673}
674
675static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
676{
677 if (evergreen_average_bandwidth(wm) <=
678 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
679 return true;
680 else
681 return false;
682};
683
684static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
685{
686 if (evergreen_average_bandwidth(wm) <=
687 (evergreen_available_bandwidth(wm) / wm->num_heads))
688 return true;
689 else
690 return false;
691};
692
693static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
694{
695 u32 lb_partitions = wm->lb_size / wm->src_width;
696 u32 line_time = wm->active_time + wm->blank_time;
697 u32 latency_tolerant_lines;
698 u32 latency_hiding;
699 fixed20_12 a;
700
701 a.full = dfixed_const(1);
702 if (wm->vsc.full > a.full)
703 latency_tolerant_lines = 1;
704 else {
705 if (lb_partitions <= (wm->vtaps + 1))
706 latency_tolerant_lines = 1;
707 else
708 latency_tolerant_lines = 2;
709 }
710
711 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
712
713 if (evergreen_latency_watermark(wm) <= latency_hiding)
714 return true;
715 else
716 return false;
717}
718
719static void evergreen_program_watermarks(struct radeon_device *rdev,
720 struct radeon_crtc *radeon_crtc,
721 u32 lb_size, u32 num_heads)
722{
723 struct drm_display_mode *mode = &radeon_crtc->base.mode;
724 struct evergreen_wm_params wm;
725 u32 pixel_period;
726 u32 line_time = 0;
727 u32 latency_watermark_a = 0, latency_watermark_b = 0;
728 u32 priority_a_mark = 0, priority_b_mark = 0;
729 u32 priority_a_cnt = PRIORITY_OFF;
730 u32 priority_b_cnt = PRIORITY_OFF;
731 u32 pipe_offset = radeon_crtc->crtc_id * 16;
732 u32 tmp, arb_control3;
733 fixed20_12 a, b, c;
734
735 if (radeon_crtc->base.enabled && num_heads && mode) {
736 pixel_period = 1000000 / (u32)mode->clock;
737 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
738 priority_a_cnt = 0;
739 priority_b_cnt = 0;
740
741 wm.yclk = rdev->pm.current_mclk * 10;
742 wm.sclk = rdev->pm.current_sclk * 10;
743 wm.disp_clk = mode->clock;
744 wm.src_width = mode->crtc_hdisplay;
745 wm.active_time = mode->crtc_hdisplay * pixel_period;
746 wm.blank_time = line_time - wm.active_time;
747 wm.interlaced = false;
748 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
749 wm.interlaced = true;
750 wm.vsc = radeon_crtc->vsc;
751 wm.vtaps = 1;
752 if (radeon_crtc->rmx_type != RMX_OFF)
753 wm.vtaps = 2;
754 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
755 wm.lb_size = lb_size;
756 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
757 wm.num_heads = num_heads;
758
759 /* set for high clocks */
760 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
761 /* set for low clocks */
762 /* wm.yclk = low clk; wm.sclk = low clk */
763 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
764
765 /* possibly force display priority to high */
766 /* should really do this at mode validation time... */
767 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
768 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
769 !evergreen_check_latency_hiding(&wm) ||
770 (rdev->disp_priority == 2)) {
771 DRM_DEBUG_KMS("force priority to high\n");
772 priority_a_cnt |= PRIORITY_ALWAYS_ON;
773 priority_b_cnt |= PRIORITY_ALWAYS_ON;
774 }
775
776 a.full = dfixed_const(1000);
777 b.full = dfixed_const(mode->clock);
778 b.full = dfixed_div(b, a);
779 c.full = dfixed_const(latency_watermark_a);
780 c.full = dfixed_mul(c, b);
781 c.full = dfixed_mul(c, radeon_crtc->hsc);
782 c.full = dfixed_div(c, a);
783 a.full = dfixed_const(16);
784 c.full = dfixed_div(c, a);
785 priority_a_mark = dfixed_trunc(c);
786 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
787
788 a.full = dfixed_const(1000);
789 b.full = dfixed_const(mode->clock);
790 b.full = dfixed_div(b, a);
791 c.full = dfixed_const(latency_watermark_b);
792 c.full = dfixed_mul(c, b);
793 c.full = dfixed_mul(c, radeon_crtc->hsc);
794 c.full = dfixed_div(c, a);
795 a.full = dfixed_const(16);
796 c.full = dfixed_div(c, a);
797 priority_b_mark = dfixed_trunc(c);
798 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
799 }
800
801 /* select wm A */
802 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
803 tmp = arb_control3;
804 tmp &= ~LATENCY_WATERMARK_MASK(3);
805 tmp |= LATENCY_WATERMARK_MASK(1);
806 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
807 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
808 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
809 LATENCY_HIGH_WATERMARK(line_time)));
810 /* select wm B */
811 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
812 tmp &= ~LATENCY_WATERMARK_MASK(3);
813 tmp |= LATENCY_WATERMARK_MASK(2);
814 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
815 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
816 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
817 LATENCY_HIGH_WATERMARK(line_time)));
818 /* restore original selection */
819 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
820
821 /* write the priority marks */
822 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
823 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
824
825}
826
827void evergreen_bandwidth_update(struct radeon_device *rdev)
828{
829 struct drm_display_mode *mode0 = NULL;
830 struct drm_display_mode *mode1 = NULL;
831 u32 num_heads = 0, lb_size;
832 int i;
833
834 radeon_update_display_priority(rdev);
835
836 for (i = 0; i < rdev->num_crtc; i++) {
837 if (rdev->mode_info.crtcs[i]->base.enabled)
838 num_heads++;
839 }
840 for (i = 0; i < rdev->num_crtc; i += 2) {
841 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
842 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
843 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
844 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
845 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
846 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
847 }
848}
849
850int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
851{
852 unsigned i;
853 u32 tmp;
854
855 for (i = 0; i < rdev->usec_timeout; i++) {
856 /* read MC_STATUS */
857 tmp = RREG32(SRBM_STATUS) & 0x1F00;
858 if (!tmp)
859 return 0;
860 udelay(1);
861 }
862 return -1;
863}
864
865/*
866 * GART
867 */
868void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
869{
870 unsigned i;
871 u32 tmp;
872
873 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
874
875 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
876 for (i = 0; i < rdev->usec_timeout; i++) {
877 /* read MC_STATUS */
878 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
879 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
880 if (tmp == 2) {
881 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
882 return;
883 }
884 if (tmp) {
885 return;
886 }
887 udelay(1);
888 }
889}
890
891int evergreen_pcie_gart_enable(struct radeon_device *rdev)
892{
893 u32 tmp;
894 int r;
895
896 if (rdev->gart.table.vram.robj == NULL) {
897 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
898 return -EINVAL;
899 }
900 r = radeon_gart_table_vram_pin(rdev);
901 if (r)
902 return r;
903 radeon_gart_restore(rdev);
904 /* Setup L2 cache */
905 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
906 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
907 EFFECTIVE_L2_QUEUE_SIZE(7));
908 WREG32(VM_L2_CNTL2, 0);
909 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
910 /* Setup TLB control */
911 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
912 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
913 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
914 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
915 if (rdev->flags & RADEON_IS_IGP) {
916 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
917 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
918 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
919 } else {
920 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
921 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
922 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
923 }
924 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
925 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
926 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
927 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
928 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
929 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
930 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
931 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
932 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
933 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
934 (u32)(rdev->dummy_page.addr >> 12));
935 WREG32(VM_CONTEXT1_CNTL, 0);
936
937 evergreen_pcie_gart_tlb_flush(rdev);
938 rdev->gart.ready = true;
939 return 0;
940}
941
942void evergreen_pcie_gart_disable(struct radeon_device *rdev)
943{
944 u32 tmp;
945 int r;
946
947 /* Disable all tables */
948 WREG32(VM_CONTEXT0_CNTL, 0);
949 WREG32(VM_CONTEXT1_CNTL, 0);
950
951 /* Setup L2 cache */
952 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
953 EFFECTIVE_L2_QUEUE_SIZE(7));
954 WREG32(VM_L2_CNTL2, 0);
955 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
956 /* Setup TLB control */
957 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
958 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
959 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
960 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
961 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
962 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
963 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
964 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
965 if (rdev->gart.table.vram.robj) {
966 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
967 if (likely(r == 0)) {
968 radeon_bo_kunmap(rdev->gart.table.vram.robj);
969 radeon_bo_unpin(rdev->gart.table.vram.robj);
970 radeon_bo_unreserve(rdev->gart.table.vram.robj);
971 }
972 }
973}
974
975void evergreen_pcie_gart_fini(struct radeon_device *rdev)
976{
977 evergreen_pcie_gart_disable(rdev);
978 radeon_gart_table_vram_free(rdev);
979 radeon_gart_fini(rdev);
980}
981
982
983void evergreen_agp_enable(struct radeon_device *rdev)
984{
985 u32 tmp;
986
987 /* Setup L2 cache */
988 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
989 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
990 EFFECTIVE_L2_QUEUE_SIZE(7));
991 WREG32(VM_L2_CNTL2, 0);
992 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
993 /* Setup TLB control */
994 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
995 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
996 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
997 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
998 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
999 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1000 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1001 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1002 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1003 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1004 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1005 WREG32(VM_CONTEXT0_CNTL, 0);
1006 WREG32(VM_CONTEXT1_CNTL, 0);
1007}
1008
1009void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1010{
1011 save->vga_control[0] = RREG32(D1VGA_CONTROL);
1012 save->vga_control[1] = RREG32(D2VGA_CONTROL);
1013 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
1014 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
1015 save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
1016 save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
1017 if (rdev->num_crtc >= 4) {
1018 save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
1019 save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
1020 save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
1021 save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
1022 }
1023 if (rdev->num_crtc >= 6) {
1024 save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
1025 save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
1026 save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
1027 save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1028 }
1029
1030 /* Stop all video */
1031 WREG32(VGA_RENDER_CONTROL, 0);
1032 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1033 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1034 if (rdev->num_crtc >= 4) {
1035 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1036 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1037 }
1038 if (rdev->num_crtc >= 6) {
1039 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1040 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1041 }
1042 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1043 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1044 if (rdev->num_crtc >= 4) {
1045 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1046 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1047 }
1048 if (rdev->num_crtc >= 6) {
1049 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1050 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1051 }
1052 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1053 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1054 if (rdev->num_crtc >= 4) {
1055 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1056 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1057 }
1058 if (rdev->num_crtc >= 6) {
1059 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1060 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1061 }
1062
1063 WREG32(D1VGA_CONTROL, 0);
1064 WREG32(D2VGA_CONTROL, 0);
1065 if (rdev->num_crtc >= 4) {
1066 WREG32(EVERGREEN_D3VGA_CONTROL, 0);
1067 WREG32(EVERGREEN_D4VGA_CONTROL, 0);
1068 }
1069 if (rdev->num_crtc >= 6) {
1070 WREG32(EVERGREEN_D5VGA_CONTROL, 0);
1071 WREG32(EVERGREEN_D6VGA_CONTROL, 0);
1072 }
1073}
1074
1075void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1076{
1077 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1078 upper_32_bits(rdev->mc.vram_start));
1079 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1080 upper_32_bits(rdev->mc.vram_start));
1081 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1082 (u32)rdev->mc.vram_start);
1083 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1084 (u32)rdev->mc.vram_start);
1085
1086 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1087 upper_32_bits(rdev->mc.vram_start));
1088 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1089 upper_32_bits(rdev->mc.vram_start));
1090 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1091 (u32)rdev->mc.vram_start);
1092 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1093 (u32)rdev->mc.vram_start);
1094
1095 if (rdev->num_crtc >= 4) {
1096 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1097 upper_32_bits(rdev->mc.vram_start));
1098 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1099 upper_32_bits(rdev->mc.vram_start));
1100 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1101 (u32)rdev->mc.vram_start);
1102 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1103 (u32)rdev->mc.vram_start);
1104
1105 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1106 upper_32_bits(rdev->mc.vram_start));
1107 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1108 upper_32_bits(rdev->mc.vram_start));
1109 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1110 (u32)rdev->mc.vram_start);
1111 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1112 (u32)rdev->mc.vram_start);
1113 }
1114 if (rdev->num_crtc >= 6) {
1115 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1116 upper_32_bits(rdev->mc.vram_start));
1117 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1118 upper_32_bits(rdev->mc.vram_start));
1119 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1120 (u32)rdev->mc.vram_start);
1121 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1122 (u32)rdev->mc.vram_start);
1123
1124 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1125 upper_32_bits(rdev->mc.vram_start));
1126 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1127 upper_32_bits(rdev->mc.vram_start));
1128 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1129 (u32)rdev->mc.vram_start);
1130 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1131 (u32)rdev->mc.vram_start);
1132 }
1133
1134 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1135 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1136 /* Unlock host access */
1137 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1138 mdelay(1);
1139 /* Restore video state */
1140 WREG32(D1VGA_CONTROL, save->vga_control[0]);
1141 WREG32(D2VGA_CONTROL, save->vga_control[1]);
1142 if (rdev->num_crtc >= 4) {
1143 WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
1144 WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
1145 }
1146 if (rdev->num_crtc >= 6) {
1147 WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
1148 WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
1149 }
1150 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1151 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1152 if (rdev->num_crtc >= 4) {
1153 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1154 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1155 }
1156 if (rdev->num_crtc >= 6) {
1157 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1158 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1159 }
1160 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
1161 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1162 if (rdev->num_crtc >= 4) {
1163 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
1164 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
1165 }
1166 if (rdev->num_crtc >= 6) {
1167 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
1168 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1169 }
1170 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1171 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1172 if (rdev->num_crtc >= 4) {
1173 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1174 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1175 }
1176 if (rdev->num_crtc >= 6) {
1177 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1178 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1179 }
1180 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1181}
1182
1183void evergreen_mc_program(struct radeon_device *rdev)
1184{
1185 struct evergreen_mc_save save;
1186 u32 tmp;
1187 int i, j;
1188
1189 /* Initialize HDP */
1190 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1191 WREG32((0x2c14 + j), 0x00000000);
1192 WREG32((0x2c18 + j), 0x00000000);
1193 WREG32((0x2c1c + j), 0x00000000);
1194 WREG32((0x2c20 + j), 0x00000000);
1195 WREG32((0x2c24 + j), 0x00000000);
1196 }
1197 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1198
1199 evergreen_mc_stop(rdev, &save);
1200 if (evergreen_mc_wait_for_idle(rdev)) {
1201 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1202 }
1203 /* Lockout access through VGA aperture*/
1204 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1205 /* Update configuration */
1206 if (rdev->flags & RADEON_IS_AGP) {
1207 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1208 /* VRAM before AGP */
1209 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1210 rdev->mc.vram_start >> 12);
1211 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1212 rdev->mc.gtt_end >> 12);
1213 } else {
1214 /* VRAM after AGP */
1215 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1216 rdev->mc.gtt_start >> 12);
1217 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1218 rdev->mc.vram_end >> 12);
1219 }
1220 } else {
1221 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1222 rdev->mc.vram_start >> 12);
1223 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1224 rdev->mc.vram_end >> 12);
1225 }
1226 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1227 if (rdev->flags & RADEON_IS_IGP) {
1228 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1229 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1230 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1231 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1232 }
1233 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1234 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1235 WREG32(MC_VM_FB_LOCATION, tmp);
1236 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1237 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1238 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1239 if (rdev->flags & RADEON_IS_AGP) {
1240 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1241 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1242 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1243 } else {
1244 WREG32(MC_VM_AGP_BASE, 0);
1245 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1246 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1247 }
1248 if (evergreen_mc_wait_for_idle(rdev)) {
1249 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1250 }
1251 evergreen_mc_resume(rdev, &save);
1252 /* we need to own VRAM, so turn off the VGA renderer here
1253 * to stop it overwriting our objects */
1254 rv515_vga_render_disable(rdev);
1255}
1256
1257/*
1258 * CP.
1259 */
1260void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1261{
1262 /* set to DX10/11 mode */
1263 radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
1264 radeon_ring_write(rdev, 1);
1265 /* FIXME: implement */
1266 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1267 radeon_ring_write(rdev,
1268#ifdef __BIG_ENDIAN
1269 (2 << 0) |
1270#endif
1271 (ib->gpu_addr & 0xFFFFFFFC));
1272 radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
1273 radeon_ring_write(rdev, ib->length_dw);
1274}
1275
1276
1277static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1278{
1279 const __be32 *fw_data;
1280 int i;
1281
1282 if (!rdev->me_fw || !rdev->pfp_fw)
1283 return -EINVAL;
1284
1285 r700_cp_stop(rdev);
1286 WREG32(CP_RB_CNTL,
1287#ifdef __BIG_ENDIAN
1288 BUF_SWAP_32BIT |
1289#endif
1290 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1291
1292 fw_data = (const __be32 *)rdev->pfp_fw->data;
1293 WREG32(CP_PFP_UCODE_ADDR, 0);
1294 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1295 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1296 WREG32(CP_PFP_UCODE_ADDR, 0);
1297
1298 fw_data = (const __be32 *)rdev->me_fw->data;
1299 WREG32(CP_ME_RAM_WADDR, 0);
1300 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1301 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1302
1303 WREG32(CP_PFP_UCODE_ADDR, 0);
1304 WREG32(CP_ME_RAM_WADDR, 0);
1305 WREG32(CP_ME_RAM_RADDR, 0);
1306 return 0;
1307}
1308
1309static int evergreen_cp_start(struct radeon_device *rdev)
1310{
1311 int r, i;
1312 uint32_t cp_me;
1313
1314 r = radeon_ring_lock(rdev, 7);
1315 if (r) {
1316 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1317 return r;
1318 }
1319 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1320 radeon_ring_write(rdev, 0x1);
1321 radeon_ring_write(rdev, 0x0);
1322 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1323 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1324 radeon_ring_write(rdev, 0);
1325 radeon_ring_write(rdev, 0);
1326 radeon_ring_unlock_commit(rdev);
1327
1328 cp_me = 0xff;
1329 WREG32(CP_ME_CNTL, cp_me);
1330
1331 r = radeon_ring_lock(rdev, evergreen_default_size + 19);
1332 if (r) {
1333 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1334 return r;
1335 }
1336
1337 /* setup clear context state */
1338 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1339 radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1340
1341 for (i = 0; i < evergreen_default_size; i++)
1342 radeon_ring_write(rdev, evergreen_default_state[i]);
1343
1344 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1345 radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
1346
1347 /* set clear context state */
1348 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
1349 radeon_ring_write(rdev, 0);
1350
1351 /* SQ_VTX_BASE_VTX_LOC */
1352 radeon_ring_write(rdev, 0xc0026f00);
1353 radeon_ring_write(rdev, 0x00000000);
1354 radeon_ring_write(rdev, 0x00000000);
1355 radeon_ring_write(rdev, 0x00000000);
1356
1357 /* Clear consts */
1358 radeon_ring_write(rdev, 0xc0036f00);
1359 radeon_ring_write(rdev, 0x00000bc4);
1360 radeon_ring_write(rdev, 0xffffffff);
1361 radeon_ring_write(rdev, 0xffffffff);
1362 radeon_ring_write(rdev, 0xffffffff);
1363
1364 radeon_ring_write(rdev, 0xc0026900);
1365 radeon_ring_write(rdev, 0x00000316);
1366 radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1367 radeon_ring_write(rdev, 0x00000010); /* */
1368
1369 radeon_ring_unlock_commit(rdev);
1370
1371 return 0;
1372}
1373
1374int evergreen_cp_resume(struct radeon_device *rdev)
1375{
1376 u32 tmp;
1377 u32 rb_bufsz;
1378 int r;
1379
1380 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1381 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1382 SOFT_RESET_PA |
1383 SOFT_RESET_SH |
1384 SOFT_RESET_VGT |
1385 SOFT_RESET_SPI |
1386 SOFT_RESET_SX));
1387 RREG32(GRBM_SOFT_RESET);
1388 mdelay(15);
1389 WREG32(GRBM_SOFT_RESET, 0);
1390 RREG32(GRBM_SOFT_RESET);
1391
1392 /* Set ring buffer size */
1393 rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1394 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1395#ifdef __BIG_ENDIAN
1396 tmp |= BUF_SWAP_32BIT;
1397#endif
1398 WREG32(CP_RB_CNTL, tmp);
1399 WREG32(CP_SEM_WAIT_TIMER, 0x4);
1400
1401 /* Set the write pointer delay */
1402 WREG32(CP_RB_WPTR_DELAY, 0);
1403
1404 /* Initialize the ring buffer's read and write pointers */
1405 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1406 WREG32(CP_RB_RPTR_WR, 0);
1407 rdev->cp.wptr = 0;
1408 WREG32(CP_RB_WPTR, rdev->cp.wptr);
1409
1410 /* set the wb address wether it's enabled or not */
1411 WREG32(CP_RB_RPTR_ADDR,
1412 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1413 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1414 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1415
1416 if (rdev->wb.enabled)
1417 WREG32(SCRATCH_UMSK, 0xff);
1418 else {
1419 tmp |= RB_NO_UPDATE;
1420 WREG32(SCRATCH_UMSK, 0);
1421 }
1422
1423 mdelay(1);
1424 WREG32(CP_RB_CNTL, tmp);
1425
1426 WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1427 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1428
1429 rdev->cp.rptr = RREG32(CP_RB_RPTR);
1430
1431 evergreen_cp_start(rdev);
1432 rdev->cp.ready = true;
1433 r = radeon_ring_test(rdev);
1434 if (r) {
1435 rdev->cp.ready = false;
1436 return r;
1437 }
1438 return 0;
1439}
1440
1441/*
1442 * Core functions
1443 */
1444static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1445 u32 num_tile_pipes,
1446 u32 num_backends,
1447 u32 backend_disable_mask)
1448{
1449 u32 backend_map = 0;
1450 u32 enabled_backends_mask = 0;
1451 u32 enabled_backends_count = 0;
1452 u32 cur_pipe;
1453 u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
1454 u32 cur_backend = 0;
1455 u32 i;
1456 bool force_no_swizzle;
1457
1458 if (num_tile_pipes > EVERGREEN_MAX_PIPES)
1459 num_tile_pipes = EVERGREEN_MAX_PIPES;
1460 if (num_tile_pipes < 1)
1461 num_tile_pipes = 1;
1462 if (num_backends > EVERGREEN_MAX_BACKENDS)
1463 num_backends = EVERGREEN_MAX_BACKENDS;
1464 if (num_backends < 1)
1465 num_backends = 1;
1466
1467 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1468 if (((backend_disable_mask >> i) & 1) == 0) {
1469 enabled_backends_mask |= (1 << i);
1470 ++enabled_backends_count;
1471 }
1472 if (enabled_backends_count == num_backends)
1473 break;
1474 }
1475
1476 if (enabled_backends_count == 0) {
1477 enabled_backends_mask = 1;
1478 enabled_backends_count = 1;
1479 }
1480
1481 if (enabled_backends_count != num_backends)
1482 num_backends = enabled_backends_count;
1483
1484 memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * EVERGREEN_MAX_PIPES);
1485 switch (rdev->family) {
1486 case CHIP_CEDAR:
1487 case CHIP_REDWOOD:
1488 case CHIP_PALM:
1489 case CHIP_SUMO:
1490 case CHIP_SUMO2:
1491 case CHIP_TURKS:
1492 case CHIP_CAICOS:
1493 force_no_swizzle = false;
1494 break;
1495 case CHIP_CYPRESS:
1496 case CHIP_HEMLOCK:
1497 case CHIP_JUNIPER:
1498 case CHIP_BARTS:
1499 default:
1500 force_no_swizzle = true;
1501 break;
1502 }
1503 if (force_no_swizzle) {
1504 bool last_backend_enabled = false;
1505
1506 force_no_swizzle = false;
1507 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1508 if (((enabled_backends_mask >> i) & 1) == 1) {
1509 if (last_backend_enabled)
1510 force_no_swizzle = true;
1511 last_backend_enabled = true;
1512 } else
1513 last_backend_enabled = false;
1514 }
1515 }
1516
1517 switch (num_tile_pipes) {
1518 case 1:
1519 case 3:
1520 case 5:
1521 case 7:
1522 DRM_ERROR("odd number of pipes!\n");
1523 break;
1524 case 2:
1525 swizzle_pipe[0] = 0;
1526 swizzle_pipe[1] = 1;
1527 break;
1528 case 4:
1529 if (force_no_swizzle) {
1530 swizzle_pipe[0] = 0;
1531 swizzle_pipe[1] = 1;
1532 swizzle_pipe[2] = 2;
1533 swizzle_pipe[3] = 3;
1534 } else {
1535 swizzle_pipe[0] = 0;
1536 swizzle_pipe[1] = 2;
1537 swizzle_pipe[2] = 1;
1538 swizzle_pipe[3] = 3;
1539 }
1540 break;
1541 case 6:
1542 if (force_no_swizzle) {
1543 swizzle_pipe[0] = 0;
1544 swizzle_pipe[1] = 1;
1545 swizzle_pipe[2] = 2;
1546 swizzle_pipe[3] = 3;
1547 swizzle_pipe[4] = 4;
1548 swizzle_pipe[5] = 5;
1549 } else {
1550 swizzle_pipe[0] = 0;
1551 swizzle_pipe[1] = 2;
1552 swizzle_pipe[2] = 4;
1553 swizzle_pipe[3] = 1;
1554 swizzle_pipe[4] = 3;
1555 swizzle_pipe[5] = 5;
1556 }
1557 break;
1558 case 8:
1559 if (force_no_swizzle) {
1560 swizzle_pipe[0] = 0;
1561 swizzle_pipe[1] = 1;
1562 swizzle_pipe[2] = 2;
1563 swizzle_pipe[3] = 3;
1564 swizzle_pipe[4] = 4;
1565 swizzle_pipe[5] = 5;
1566 swizzle_pipe[6] = 6;
1567 swizzle_pipe[7] = 7;
1568 } else {
1569 swizzle_pipe[0] = 0;
1570 swizzle_pipe[1] = 2;
1571 swizzle_pipe[2] = 4;
1572 swizzle_pipe[3] = 6;
1573 swizzle_pipe[4] = 1;
1574 swizzle_pipe[5] = 3;
1575 swizzle_pipe[6] = 5;
1576 swizzle_pipe[7] = 7;
1577 }
1578 break;
1579 }
1580
1581 for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1582 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1583 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1584
1585 backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
1586
1587 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1588 }
1589
1590 return backend_map;
1591}
1592
1593static void evergreen_gpu_init(struct radeon_device *rdev)
1594{
1595 u32 cc_rb_backend_disable = 0;
1596 u32 cc_gc_shader_pipe_config;
1597 u32 gb_addr_config = 0;
1598 u32 mc_shared_chmap, mc_arb_ramcfg;
1599 u32 gb_backend_map;
1600 u32 grbm_gfx_index;
1601 u32 sx_debug_1;
1602 u32 smx_dc_ctl0;
1603 u32 sq_config;
1604 u32 sq_lds_resource_mgmt;
1605 u32 sq_gpr_resource_mgmt_1;
1606 u32 sq_gpr_resource_mgmt_2;
1607 u32 sq_gpr_resource_mgmt_3;
1608 u32 sq_thread_resource_mgmt;
1609 u32 sq_thread_resource_mgmt_2;
1610 u32 sq_stack_resource_mgmt_1;
1611 u32 sq_stack_resource_mgmt_2;
1612 u32 sq_stack_resource_mgmt_3;
1613 u32 vgt_cache_invalidation;
1614 u32 hdp_host_path_cntl, tmp;
1615 int i, j, num_shader_engines, ps_thread_count;
1616
1617 switch (rdev->family) {
1618 case CHIP_CYPRESS:
1619 case CHIP_HEMLOCK:
1620 rdev->config.evergreen.num_ses = 2;
1621 rdev->config.evergreen.max_pipes = 4;
1622 rdev->config.evergreen.max_tile_pipes = 8;
1623 rdev->config.evergreen.max_simds = 10;
1624 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1625 rdev->config.evergreen.max_gprs = 256;
1626 rdev->config.evergreen.max_threads = 248;
1627 rdev->config.evergreen.max_gs_threads = 32;
1628 rdev->config.evergreen.max_stack_entries = 512;
1629 rdev->config.evergreen.sx_num_of_sets = 4;
1630 rdev->config.evergreen.sx_max_export_size = 256;
1631 rdev->config.evergreen.sx_max_export_pos_size = 64;
1632 rdev->config.evergreen.sx_max_export_smx_size = 192;
1633 rdev->config.evergreen.max_hw_contexts = 8;
1634 rdev->config.evergreen.sq_num_cf_insts = 2;
1635
1636 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1637 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1638 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1639 break;
1640 case CHIP_JUNIPER:
1641 rdev->config.evergreen.num_ses = 1;
1642 rdev->config.evergreen.max_pipes = 4;
1643 rdev->config.evergreen.max_tile_pipes = 4;
1644 rdev->config.evergreen.max_simds = 10;
1645 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1646 rdev->config.evergreen.max_gprs = 256;
1647 rdev->config.evergreen.max_threads = 248;
1648 rdev->config.evergreen.max_gs_threads = 32;
1649 rdev->config.evergreen.max_stack_entries = 512;
1650 rdev->config.evergreen.sx_num_of_sets = 4;
1651 rdev->config.evergreen.sx_max_export_size = 256;
1652 rdev->config.evergreen.sx_max_export_pos_size = 64;
1653 rdev->config.evergreen.sx_max_export_smx_size = 192;
1654 rdev->config.evergreen.max_hw_contexts = 8;
1655 rdev->config.evergreen.sq_num_cf_insts = 2;
1656
1657 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1658 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1659 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1660 break;
1661 case CHIP_REDWOOD:
1662 rdev->config.evergreen.num_ses = 1;
1663 rdev->config.evergreen.max_pipes = 4;
1664 rdev->config.evergreen.max_tile_pipes = 4;
1665 rdev->config.evergreen.max_simds = 5;
1666 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1667 rdev->config.evergreen.max_gprs = 256;
1668 rdev->config.evergreen.max_threads = 248;
1669 rdev->config.evergreen.max_gs_threads = 32;
1670 rdev->config.evergreen.max_stack_entries = 256;
1671 rdev->config.evergreen.sx_num_of_sets = 4;
1672 rdev->config.evergreen.sx_max_export_size = 256;
1673 rdev->config.evergreen.sx_max_export_pos_size = 64;
1674 rdev->config.evergreen.sx_max_export_smx_size = 192;
1675 rdev->config.evergreen.max_hw_contexts = 8;
1676 rdev->config.evergreen.sq_num_cf_insts = 2;
1677
1678 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1679 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1680 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1681 break;
1682 case CHIP_CEDAR:
1683 default:
1684 rdev->config.evergreen.num_ses = 1;
1685 rdev->config.evergreen.max_pipes = 2;
1686 rdev->config.evergreen.max_tile_pipes = 2;
1687 rdev->config.evergreen.max_simds = 2;
1688 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1689 rdev->config.evergreen.max_gprs = 256;
1690 rdev->config.evergreen.max_threads = 192;
1691 rdev->config.evergreen.max_gs_threads = 16;
1692 rdev->config.evergreen.max_stack_entries = 256;
1693 rdev->config.evergreen.sx_num_of_sets = 4;
1694 rdev->config.evergreen.sx_max_export_size = 128;
1695 rdev->config.evergreen.sx_max_export_pos_size = 32;
1696 rdev->config.evergreen.sx_max_export_smx_size = 96;
1697 rdev->config.evergreen.max_hw_contexts = 4;
1698 rdev->config.evergreen.sq_num_cf_insts = 1;
1699
1700 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1701 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1702 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1703 break;
1704 case CHIP_PALM:
1705 rdev->config.evergreen.num_ses = 1;
1706 rdev->config.evergreen.max_pipes = 2;
1707 rdev->config.evergreen.max_tile_pipes = 2;
1708 rdev->config.evergreen.max_simds = 2;
1709 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1710 rdev->config.evergreen.max_gprs = 256;
1711 rdev->config.evergreen.max_threads = 192;
1712 rdev->config.evergreen.max_gs_threads = 16;
1713 rdev->config.evergreen.max_stack_entries = 256;
1714 rdev->config.evergreen.sx_num_of_sets = 4;
1715 rdev->config.evergreen.sx_max_export_size = 128;
1716 rdev->config.evergreen.sx_max_export_pos_size = 32;
1717 rdev->config.evergreen.sx_max_export_smx_size = 96;
1718 rdev->config.evergreen.max_hw_contexts = 4;
1719 rdev->config.evergreen.sq_num_cf_insts = 1;
1720
1721 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1722 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1723 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1724 break;
1725 case CHIP_SUMO:
1726 rdev->config.evergreen.num_ses = 1;
1727 rdev->config.evergreen.max_pipes = 4;
1728 rdev->config.evergreen.max_tile_pipes = 2;
1729 if (rdev->pdev->device == 0x9648)
1730 rdev->config.evergreen.max_simds = 3;
1731 else if ((rdev->pdev->device == 0x9647) ||
1732 (rdev->pdev->device == 0x964a))
1733 rdev->config.evergreen.max_simds = 4;
1734 else
1735 rdev->config.evergreen.max_simds = 5;
1736 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1737 rdev->config.evergreen.max_gprs = 256;
1738 rdev->config.evergreen.max_threads = 248;
1739 rdev->config.evergreen.max_gs_threads = 32;
1740 rdev->config.evergreen.max_stack_entries = 256;
1741 rdev->config.evergreen.sx_num_of_sets = 4;
1742 rdev->config.evergreen.sx_max_export_size = 256;
1743 rdev->config.evergreen.sx_max_export_pos_size = 64;
1744 rdev->config.evergreen.sx_max_export_smx_size = 192;
1745 rdev->config.evergreen.max_hw_contexts = 8;
1746 rdev->config.evergreen.sq_num_cf_insts = 2;
1747
1748 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1749 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1750 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1751 break;
1752 case CHIP_SUMO2:
1753 rdev->config.evergreen.num_ses = 1;
1754 rdev->config.evergreen.max_pipes = 4;
1755 rdev->config.evergreen.max_tile_pipes = 4;
1756 rdev->config.evergreen.max_simds = 2;
1757 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1758 rdev->config.evergreen.max_gprs = 256;
1759 rdev->config.evergreen.max_threads = 248;
1760 rdev->config.evergreen.max_gs_threads = 32;
1761 rdev->config.evergreen.max_stack_entries = 512;
1762 rdev->config.evergreen.sx_num_of_sets = 4;
1763 rdev->config.evergreen.sx_max_export_size = 256;
1764 rdev->config.evergreen.sx_max_export_pos_size = 64;
1765 rdev->config.evergreen.sx_max_export_smx_size = 192;
1766 rdev->config.evergreen.max_hw_contexts = 8;
1767 rdev->config.evergreen.sq_num_cf_insts = 2;
1768
1769 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1770 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1771 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1772 break;
1773 case CHIP_BARTS:
1774 rdev->config.evergreen.num_ses = 2;
1775 rdev->config.evergreen.max_pipes = 4;
1776 rdev->config.evergreen.max_tile_pipes = 8;
1777 rdev->config.evergreen.max_simds = 7;
1778 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1779 rdev->config.evergreen.max_gprs = 256;
1780 rdev->config.evergreen.max_threads = 248;
1781 rdev->config.evergreen.max_gs_threads = 32;
1782 rdev->config.evergreen.max_stack_entries = 512;
1783 rdev->config.evergreen.sx_num_of_sets = 4;
1784 rdev->config.evergreen.sx_max_export_size = 256;
1785 rdev->config.evergreen.sx_max_export_pos_size = 64;
1786 rdev->config.evergreen.sx_max_export_smx_size = 192;
1787 rdev->config.evergreen.max_hw_contexts = 8;
1788 rdev->config.evergreen.sq_num_cf_insts = 2;
1789
1790 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1791 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1792 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1793 break;
1794 case CHIP_TURKS:
1795 rdev->config.evergreen.num_ses = 1;
1796 rdev->config.evergreen.max_pipes = 4;
1797 rdev->config.evergreen.max_tile_pipes = 4;
1798 rdev->config.evergreen.max_simds = 6;
1799 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1800 rdev->config.evergreen.max_gprs = 256;
1801 rdev->config.evergreen.max_threads = 248;
1802 rdev->config.evergreen.max_gs_threads = 32;
1803 rdev->config.evergreen.max_stack_entries = 256;
1804 rdev->config.evergreen.sx_num_of_sets = 4;
1805 rdev->config.evergreen.sx_max_export_size = 256;
1806 rdev->config.evergreen.sx_max_export_pos_size = 64;
1807 rdev->config.evergreen.sx_max_export_smx_size = 192;
1808 rdev->config.evergreen.max_hw_contexts = 8;
1809 rdev->config.evergreen.sq_num_cf_insts = 2;
1810
1811 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1812 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1813 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1814 break;
1815 case CHIP_CAICOS:
1816 rdev->config.evergreen.num_ses = 1;
1817 rdev->config.evergreen.max_pipes = 4;
1818 rdev->config.evergreen.max_tile_pipes = 2;
1819 rdev->config.evergreen.max_simds = 2;
1820 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1821 rdev->config.evergreen.max_gprs = 256;
1822 rdev->config.evergreen.max_threads = 192;
1823 rdev->config.evergreen.max_gs_threads = 16;
1824 rdev->config.evergreen.max_stack_entries = 256;
1825 rdev->config.evergreen.sx_num_of_sets = 4;
1826 rdev->config.evergreen.sx_max_export_size = 128;
1827 rdev->config.evergreen.sx_max_export_pos_size = 32;
1828 rdev->config.evergreen.sx_max_export_smx_size = 96;
1829 rdev->config.evergreen.max_hw_contexts = 4;
1830 rdev->config.evergreen.sq_num_cf_insts = 1;
1831
1832 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1833 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1834 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1835 break;
1836 }
1837
1838 /* Initialize HDP */
1839 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1840 WREG32((0x2c14 + j), 0x00000000);
1841 WREG32((0x2c18 + j), 0x00000000);
1842 WREG32((0x2c1c + j), 0x00000000);
1843 WREG32((0x2c20 + j), 0x00000000);
1844 WREG32((0x2c24 + j), 0x00000000);
1845 }
1846
1847 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1848
1849 evergreen_fix_pci_max_read_req_size(rdev);
1850
1851 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
1852
1853 cc_gc_shader_pipe_config |=
1854 INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
1855 & EVERGREEN_MAX_PIPES_MASK);
1856 cc_gc_shader_pipe_config |=
1857 INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
1858 & EVERGREEN_MAX_SIMDS_MASK);
1859
1860 cc_rb_backend_disable =
1861 BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
1862 & EVERGREEN_MAX_BACKENDS_MASK);
1863
1864
1865 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1866 if (rdev->flags & RADEON_IS_IGP)
1867 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
1868 else
1869 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1870
1871 switch (rdev->config.evergreen.max_tile_pipes) {
1872 case 1:
1873 default:
1874 gb_addr_config |= NUM_PIPES(0);
1875 break;
1876 case 2:
1877 gb_addr_config |= NUM_PIPES(1);
1878 break;
1879 case 4:
1880 gb_addr_config |= NUM_PIPES(2);
1881 break;
1882 case 8:
1883 gb_addr_config |= NUM_PIPES(3);
1884 break;
1885 }
1886
1887 gb_addr_config |= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1888 gb_addr_config |= BANK_INTERLEAVE_SIZE(0);
1889 gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
1890 gb_addr_config |= SHADER_ENGINE_TILE_SIZE(1);
1891 gb_addr_config |= NUM_GPUS(0); /* Hemlock? */
1892 gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
1893
1894 if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
1895 gb_addr_config |= ROW_SIZE(2);
1896 else
1897 gb_addr_config |= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
1898
1899 if (rdev->ddev->pdev->device == 0x689e) {
1900 u32 efuse_straps_4;
1901 u32 efuse_straps_3;
1902 u8 efuse_box_bit_131_124;
1903
1904 WREG32(RCU_IND_INDEX, 0x204);
1905 efuse_straps_4 = RREG32(RCU_IND_DATA);
1906 WREG32(RCU_IND_INDEX, 0x203);
1907 efuse_straps_3 = RREG32(RCU_IND_DATA);
1908 efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28));
1909
1910 switch(efuse_box_bit_131_124) {
1911 case 0x00:
1912 gb_backend_map = 0x76543210;
1913 break;
1914 case 0x55:
1915 gb_backend_map = 0x77553311;
1916 break;
1917 case 0x56:
1918 gb_backend_map = 0x77553300;
1919 break;
1920 case 0x59:
1921 gb_backend_map = 0x77552211;
1922 break;
1923 case 0x66:
1924 gb_backend_map = 0x77443300;
1925 break;
1926 case 0x99:
1927 gb_backend_map = 0x66552211;
1928 break;
1929 case 0x5a:
1930 gb_backend_map = 0x77552200;
1931 break;
1932 case 0xaa:
1933 gb_backend_map = 0x66442200;
1934 break;
1935 case 0x95:
1936 gb_backend_map = 0x66553311;
1937 break;
1938 default:
1939 DRM_ERROR("bad backend map, using default\n");
1940 gb_backend_map =
1941 evergreen_get_tile_pipe_to_backend_map(rdev,
1942 rdev->config.evergreen.max_tile_pipes,
1943 rdev->config.evergreen.max_backends,
1944 ((EVERGREEN_MAX_BACKENDS_MASK <<
1945 rdev->config.evergreen.max_backends) &
1946 EVERGREEN_MAX_BACKENDS_MASK));
1947 break;
1948 }
1949 } else if (rdev->ddev->pdev->device == 0x68b9) {
1950 u32 efuse_straps_3;
1951 u8 efuse_box_bit_127_124;
1952
1953 WREG32(RCU_IND_INDEX, 0x203);
1954 efuse_straps_3 = RREG32(RCU_IND_DATA);
1955 efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
1956
1957 switch(efuse_box_bit_127_124) {
1958 case 0x0:
1959 gb_backend_map = 0x00003210;
1960 break;
1961 case 0x5:
1962 case 0x6:
1963 case 0x9:
1964 case 0xa:
1965 gb_backend_map = 0x00003311;
1966 break;
1967 default:
1968 DRM_ERROR("bad backend map, using default\n");
1969 gb_backend_map =
1970 evergreen_get_tile_pipe_to_backend_map(rdev,
1971 rdev->config.evergreen.max_tile_pipes,
1972 rdev->config.evergreen.max_backends,
1973 ((EVERGREEN_MAX_BACKENDS_MASK <<
1974 rdev->config.evergreen.max_backends) &
1975 EVERGREEN_MAX_BACKENDS_MASK));
1976 break;
1977 }
1978 } else {
1979 switch (rdev->family) {
1980 case CHIP_CYPRESS:
1981 case CHIP_HEMLOCK:
1982 case CHIP_BARTS:
1983 gb_backend_map = 0x66442200;
1984 break;
1985 case CHIP_JUNIPER:
1986 gb_backend_map = 0x00002200;
1987 break;
1988 default:
1989 gb_backend_map =
1990 evergreen_get_tile_pipe_to_backend_map(rdev,
1991 rdev->config.evergreen.max_tile_pipes,
1992 rdev->config.evergreen.max_backends,
1993 ((EVERGREEN_MAX_BACKENDS_MASK <<
1994 rdev->config.evergreen.max_backends) &
1995 EVERGREEN_MAX_BACKENDS_MASK));
1996 }
1997 }
1998
1999 /* setup tiling info dword. gb_addr_config is not adequate since it does
2000 * not have bank info, so create a custom tiling dword.
2001 * bits 3:0 num_pipes
2002 * bits 7:4 num_banks
2003 * bits 11:8 group_size
2004 * bits 15:12 row_size
2005 */
2006 rdev->config.evergreen.tile_config = 0;
2007 switch (rdev->config.evergreen.max_tile_pipes) {
2008 case 1:
2009 default:
2010 rdev->config.evergreen.tile_config |= (0 << 0);
2011 break;
2012 case 2:
2013 rdev->config.evergreen.tile_config |= (1 << 0);
2014 break;
2015 case 4:
2016 rdev->config.evergreen.tile_config |= (2 << 0);
2017 break;
2018 case 8:
2019 rdev->config.evergreen.tile_config |= (3 << 0);
2020 break;
2021 }
2022 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
2023 if (rdev->flags & RADEON_IS_IGP)
2024 rdev->config.evergreen.tile_config |= 1 << 4;
2025 else
2026 rdev->config.evergreen.tile_config |=
2027 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
2028 rdev->config.evergreen.tile_config |=
2029 ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
2030 rdev->config.evergreen.tile_config |=
2031 ((gb_addr_config & 0x30000000) >> 28) << 12;
2032
2033 rdev->config.evergreen.backend_map = gb_backend_map;
2034 WREG32(GB_BACKEND_MAP, gb_backend_map);
2035 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2036 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2037 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2038
2039 num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
2040 grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
2041
2042 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
2043 u32 rb = cc_rb_backend_disable | (0xf0 << 16);
2044 u32 sp = cc_gc_shader_pipe_config;
2045 u32 gfx = grbm_gfx_index | SE_INDEX(i);
2046
2047 if (i == num_shader_engines) {
2048 rb |= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
2049 sp |= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
2050 }
2051
2052 WREG32(GRBM_GFX_INDEX, gfx);
2053 WREG32(RLC_GFX_INDEX, gfx);
2054
2055 WREG32(CC_RB_BACKEND_DISABLE, rb);
2056 WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
2057 WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
2058 WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
2059 }
2060
2061 grbm_gfx_index |= SE_BROADCAST_WRITES;
2062 WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
2063 WREG32(RLC_GFX_INDEX, grbm_gfx_index);
2064
2065 WREG32(CGTS_SYS_TCC_DISABLE, 0);
2066 WREG32(CGTS_TCC_DISABLE, 0);
2067 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2068 WREG32(CGTS_USER_TCC_DISABLE, 0);
2069
2070 /* set HW defaults for 3D engine */
2071 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2072 ROQ_IB2_START(0x2b)));
2073
2074 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2075
2076 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2077 SYNC_GRADIENT |
2078 SYNC_WALKER |
2079 SYNC_ALIGNER));
2080
2081 sx_debug_1 = RREG32(SX_DEBUG_1);
2082 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2083 WREG32(SX_DEBUG_1, sx_debug_1);
2084
2085
2086 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2087 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2088 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2089 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2090
2091 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2092 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2093 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2094
2095 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2096 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2097 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2098
2099 WREG32(VGT_NUM_INSTANCES, 1);
2100 WREG32(SPI_CONFIG_CNTL, 0);
2101 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2102 WREG32(CP_PERFMON_CNTL, 0);
2103
2104 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2105 FETCH_FIFO_HIWATER(0x4) |
2106 DONE_FIFO_HIWATER(0xe0) |
2107 ALU_UPDATE_FIFO_HIWATER(0x8)));
2108
2109 sq_config = RREG32(SQ_CONFIG);
2110 sq_config &= ~(PS_PRIO(3) |
2111 VS_PRIO(3) |
2112 GS_PRIO(3) |
2113 ES_PRIO(3));
2114 sq_config |= (VC_ENABLE |
2115 EXPORT_SRC_C |
2116 PS_PRIO(0) |
2117 VS_PRIO(1) |
2118 GS_PRIO(2) |
2119 ES_PRIO(3));
2120
2121 switch (rdev->family) {
2122 case CHIP_CEDAR:
2123 case CHIP_PALM:
2124 case CHIP_SUMO:
2125 case CHIP_SUMO2:
2126 case CHIP_CAICOS:
2127 /* no vertex cache */
2128 sq_config &= ~VC_ENABLE;
2129 break;
2130 default:
2131 break;
2132 }
2133
2134 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2135
2136 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2137 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2138 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2139 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2140 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2141 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2142 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2143
2144 switch (rdev->family) {
2145 case CHIP_CEDAR:
2146 case CHIP_PALM:
2147 case CHIP_SUMO:
2148 case CHIP_SUMO2:
2149 ps_thread_count = 96;
2150 break;
2151 default:
2152 ps_thread_count = 128;
2153 break;
2154 }
2155
2156 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2157 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2158 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2159 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2160 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2161 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2162
2163 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2164 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2165 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2166 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2167 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2168 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2169
2170 WREG32(SQ_CONFIG, sq_config);
2171 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2172 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2173 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2174 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2175 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2176 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2177 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2178 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2179 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2180 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2181
2182 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2183 FORCE_EOV_MAX_REZ_CNT(255)));
2184
2185 switch (rdev->family) {
2186 case CHIP_CEDAR:
2187 case CHIP_PALM:
2188 case CHIP_SUMO:
2189 case CHIP_SUMO2:
2190 case CHIP_CAICOS:
2191 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2192 break;
2193 default:
2194 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2195 break;
2196 }
2197 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2198 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2199
2200 WREG32(VGT_GS_VERTEX_REUSE, 16);
2201 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2202 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2203
2204 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2205 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2206
2207 WREG32(CB_PERF_CTR0_SEL_0, 0);
2208 WREG32(CB_PERF_CTR0_SEL_1, 0);
2209 WREG32(CB_PERF_CTR1_SEL_0, 0);
2210 WREG32(CB_PERF_CTR1_SEL_1, 0);
2211 WREG32(CB_PERF_CTR2_SEL_0, 0);
2212 WREG32(CB_PERF_CTR2_SEL_1, 0);
2213 WREG32(CB_PERF_CTR3_SEL_0, 0);
2214 WREG32(CB_PERF_CTR3_SEL_1, 0);
2215
2216 /* clear render buffer base addresses */
2217 WREG32(CB_COLOR0_BASE, 0);
2218 WREG32(CB_COLOR1_BASE, 0);
2219 WREG32(CB_COLOR2_BASE, 0);
2220 WREG32(CB_COLOR3_BASE, 0);
2221 WREG32(CB_COLOR4_BASE, 0);
2222 WREG32(CB_COLOR5_BASE, 0);
2223 WREG32(CB_COLOR6_BASE, 0);
2224 WREG32(CB_COLOR7_BASE, 0);
2225 WREG32(CB_COLOR8_BASE, 0);
2226 WREG32(CB_COLOR9_BASE, 0);
2227 WREG32(CB_COLOR10_BASE, 0);
2228 WREG32(CB_COLOR11_BASE, 0);
2229
2230 /* set the shader const cache sizes to 0 */
2231 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2232 WREG32(i, 0);
2233 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2234 WREG32(i, 0);
2235
2236 tmp = RREG32(HDP_MISC_CNTL);
2237 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2238 WREG32(HDP_MISC_CNTL, tmp);
2239
2240 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2241 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2242
2243 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2244
2245 udelay(50);
2246
2247}
2248
2249int evergreen_mc_init(struct radeon_device *rdev)
2250{
2251 u32 tmp;
2252 int chansize, numchan;
2253
2254 /* Get VRAM informations */
2255 rdev->mc.vram_is_ddr = true;
2256 if (rdev->flags & RADEON_IS_IGP)
2257 tmp = RREG32(FUS_MC_ARB_RAMCFG);
2258 else
2259 tmp = RREG32(MC_ARB_RAMCFG);
2260 if (tmp & CHANSIZE_OVERRIDE) {
2261 chansize = 16;
2262 } else if (tmp & CHANSIZE_MASK) {
2263 chansize = 64;
2264 } else {
2265 chansize = 32;
2266 }
2267 tmp = RREG32(MC_SHARED_CHMAP);
2268 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2269 case 0:
2270 default:
2271 numchan = 1;
2272 break;
2273 case 1:
2274 numchan = 2;
2275 break;
2276 case 2:
2277 numchan = 4;
2278 break;
2279 case 3:
2280 numchan = 8;
2281 break;
2282 }
2283 rdev->mc.vram_width = numchan * chansize;
2284 /* Could aper size report 0 ? */
2285 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2286 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
2287 /* Setup GPU memory space */
2288 if (rdev->flags & RADEON_IS_IGP) {
2289 /* size in bytes on fusion */
2290 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2291 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2292 } else {
2293 /* size in MB on evergreen */
2294 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2295 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2296 }
2297 rdev->mc.visible_vram_size = rdev->mc.aper_size;
2298 r700_vram_gtt_location(rdev, &rdev->mc);
2299 radeon_update_bandwidth_info(rdev);
2300
2301 return 0;
2302}
2303
2304bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
2305{
2306 u32 srbm_status;
2307 u32 grbm_status;
2308 u32 grbm_status_se0, grbm_status_se1;
2309 struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
2310 int r;
2311
2312 srbm_status = RREG32(SRBM_STATUS);
2313 grbm_status = RREG32(GRBM_STATUS);
2314 grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2315 grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2316 if (!(grbm_status & GUI_ACTIVE)) {
2317 r100_gpu_lockup_update(lockup, &rdev->cp);
2318 return false;
2319 }
2320 /* force CP activities */
2321 r = radeon_ring_lock(rdev, 2);
2322 if (!r) {
2323 /* PACKET2 NOP */
2324 radeon_ring_write(rdev, 0x80000000);
2325 radeon_ring_write(rdev, 0x80000000);
2326 radeon_ring_unlock_commit(rdev);
2327 }
2328 rdev->cp.rptr = RREG32(CP_RB_RPTR);
2329 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
2330}
2331
2332static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
2333{
2334 struct evergreen_mc_save save;
2335 u32 grbm_reset = 0;
2336
2337 if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
2338 return 0;
2339
2340 dev_info(rdev->dev, "GPU softreset \n");
2341 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
2342 RREG32(GRBM_STATUS));
2343 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2344 RREG32(GRBM_STATUS_SE0));
2345 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2346 RREG32(GRBM_STATUS_SE1));
2347 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
2348 RREG32(SRBM_STATUS));
2349 evergreen_mc_stop(rdev, &save);
2350 if (evergreen_mc_wait_for_idle(rdev)) {
2351 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2352 }
2353 /* Disable CP parsing/prefetching */
2354 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2355
2356 /* reset all the gfx blocks */
2357 grbm_reset = (SOFT_RESET_CP |
2358 SOFT_RESET_CB |
2359 SOFT_RESET_DB |
2360 SOFT_RESET_PA |
2361 SOFT_RESET_SC |
2362 SOFT_RESET_SPI |
2363 SOFT_RESET_SH |
2364 SOFT_RESET_SX |
2365 SOFT_RESET_TC |
2366 SOFT_RESET_TA |
2367 SOFT_RESET_VC |
2368 SOFT_RESET_VGT);
2369
2370 dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
2371 WREG32(GRBM_SOFT_RESET, grbm_reset);
2372 (void)RREG32(GRBM_SOFT_RESET);
2373 udelay(50);
2374 WREG32(GRBM_SOFT_RESET, 0);
2375 (void)RREG32(GRBM_SOFT_RESET);
2376 /* Wait a little for things to settle down */
2377 udelay(50);
2378 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
2379 RREG32(GRBM_STATUS));
2380 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
2381 RREG32(GRBM_STATUS_SE0));
2382 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
2383 RREG32(GRBM_STATUS_SE1));
2384 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
2385 RREG32(SRBM_STATUS));
2386 evergreen_mc_resume(rdev, &save);
2387 return 0;
2388}
2389
2390int evergreen_asic_reset(struct radeon_device *rdev)
2391{
2392 return evergreen_gpu_soft_reset(rdev);
2393}
2394
2395/* Interrupts */
2396
2397u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2398{
2399 switch (crtc) {
2400 case 0:
2401 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
2402 case 1:
2403 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
2404 case 2:
2405 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
2406 case 3:
2407 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
2408 case 4:
2409 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
2410 case 5:
2411 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
2412 default:
2413 return 0;
2414 }
2415}
2416
2417void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2418{
2419 u32 tmp;
2420
2421 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2422 WREG32(GRBM_INT_CNTL, 0);
2423 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2424 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2425 if (rdev->num_crtc >= 4) {
2426 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2427 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2428 }
2429 if (rdev->num_crtc >= 6) {
2430 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2431 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2432 }
2433
2434 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2435 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2436 if (rdev->num_crtc >= 4) {
2437 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2438 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2439 }
2440 if (rdev->num_crtc >= 6) {
2441 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2442 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2443 }
2444
2445 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2446 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2447
2448 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2449 WREG32(DC_HPD1_INT_CONTROL, tmp);
2450 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2451 WREG32(DC_HPD2_INT_CONTROL, tmp);
2452 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2453 WREG32(DC_HPD3_INT_CONTROL, tmp);
2454 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2455 WREG32(DC_HPD4_INT_CONTROL, tmp);
2456 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2457 WREG32(DC_HPD5_INT_CONTROL, tmp);
2458 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2459 WREG32(DC_HPD6_INT_CONTROL, tmp);
2460
2461}
2462
2463int evergreen_irq_set(struct radeon_device *rdev)
2464{
2465 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2466 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2467 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2468 u32 grbm_int_cntl = 0;
2469 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2470
2471 if (!rdev->irq.installed) {
2472 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2473 return -EINVAL;
2474 }
2475 /* don't enable anything if the ih is disabled */
2476 if (!rdev->ih.enabled) {
2477 r600_disable_interrupts(rdev);
2478 /* force the active interrupt state to all disabled */
2479 evergreen_disable_interrupt_state(rdev);
2480 return 0;
2481 }
2482
2483 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2484 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2485 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2486 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2487 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2488 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2489
2490 if (rdev->irq.sw_int) {
2491 DRM_DEBUG("evergreen_irq_set: sw int\n");
2492 cp_int_cntl |= RB_INT_ENABLE;
2493 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2494 }
2495 if (rdev->irq.crtc_vblank_int[0] ||
2496 rdev->irq.pflip[0]) {
2497 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2498 crtc1 |= VBLANK_INT_MASK;
2499 }
2500 if (rdev->irq.crtc_vblank_int[1] ||
2501 rdev->irq.pflip[1]) {
2502 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2503 crtc2 |= VBLANK_INT_MASK;
2504 }
2505 if (rdev->irq.crtc_vblank_int[2] ||
2506 rdev->irq.pflip[2]) {
2507 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2508 crtc3 |= VBLANK_INT_MASK;
2509 }
2510 if (rdev->irq.crtc_vblank_int[3] ||
2511 rdev->irq.pflip[3]) {
2512 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2513 crtc4 |= VBLANK_INT_MASK;
2514 }
2515 if (rdev->irq.crtc_vblank_int[4] ||
2516 rdev->irq.pflip[4]) {
2517 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2518 crtc5 |= VBLANK_INT_MASK;
2519 }
2520 if (rdev->irq.crtc_vblank_int[5] ||
2521 rdev->irq.pflip[5]) {
2522 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2523 crtc6 |= VBLANK_INT_MASK;
2524 }
2525 if (rdev->irq.hpd[0]) {
2526 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2527 hpd1 |= DC_HPDx_INT_EN;
2528 }
2529 if (rdev->irq.hpd[1]) {
2530 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2531 hpd2 |= DC_HPDx_INT_EN;
2532 }
2533 if (rdev->irq.hpd[2]) {
2534 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2535 hpd3 |= DC_HPDx_INT_EN;
2536 }
2537 if (rdev->irq.hpd[3]) {
2538 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2539 hpd4 |= DC_HPDx_INT_EN;
2540 }
2541 if (rdev->irq.hpd[4]) {
2542 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2543 hpd5 |= DC_HPDx_INT_EN;
2544 }
2545 if (rdev->irq.hpd[5]) {
2546 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2547 hpd6 |= DC_HPDx_INT_EN;
2548 }
2549 if (rdev->irq.gui_idle) {
2550 DRM_DEBUG("gui idle\n");
2551 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2552 }
2553
2554 WREG32(CP_INT_CNTL, cp_int_cntl);
2555 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2556
2557 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2558 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2559 if (rdev->num_crtc >= 4) {
2560 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2561 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2562 }
2563 if (rdev->num_crtc >= 6) {
2564 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2565 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2566 }
2567
2568 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2570 if (rdev->num_crtc >= 4) {
2571 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2572 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2573 }
2574 if (rdev->num_crtc >= 6) {
2575 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2576 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2577 }
2578
2579 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2580 WREG32(DC_HPD2_INT_CONTROL, hpd2);
2581 WREG32(DC_HPD3_INT_CONTROL, hpd3);
2582 WREG32(DC_HPD4_INT_CONTROL, hpd4);
2583 WREG32(DC_HPD5_INT_CONTROL, hpd5);
2584 WREG32(DC_HPD6_INT_CONTROL, hpd6);
2585
2586 return 0;
2587}
2588
2589static inline void evergreen_irq_ack(struct radeon_device *rdev)
2590{
2591 u32 tmp;
2592
2593 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2594 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2595 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2596 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2597 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2598 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2599 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2600 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2601 if (rdev->num_crtc >= 4) {
2602 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2603 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2604 }
2605 if (rdev->num_crtc >= 6) {
2606 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2607 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2608 }
2609
2610 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2611 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2612 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2613 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2614 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2615 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2616 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2617 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2618 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2619 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2620 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2621 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2622
2623 if (rdev->num_crtc >= 4) {
2624 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2625 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2626 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2627 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2628 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2629 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2630 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2631 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2632 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2633 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2634 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2635 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2636 }
2637
2638 if (rdev->num_crtc >= 6) {
2639 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2640 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2641 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2642 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2643 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2644 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2645 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2646 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2647 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2648 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2649 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2650 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2651 }
2652
2653 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2654 tmp = RREG32(DC_HPD1_INT_CONTROL);
2655 tmp |= DC_HPDx_INT_ACK;
2656 WREG32(DC_HPD1_INT_CONTROL, tmp);
2657 }
2658 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2659 tmp = RREG32(DC_HPD2_INT_CONTROL);
2660 tmp |= DC_HPDx_INT_ACK;
2661 WREG32(DC_HPD2_INT_CONTROL, tmp);
2662 }
2663 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2664 tmp = RREG32(DC_HPD3_INT_CONTROL);
2665 tmp |= DC_HPDx_INT_ACK;
2666 WREG32(DC_HPD3_INT_CONTROL, tmp);
2667 }
2668 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2669 tmp = RREG32(DC_HPD4_INT_CONTROL);
2670 tmp |= DC_HPDx_INT_ACK;
2671 WREG32(DC_HPD4_INT_CONTROL, tmp);
2672 }
2673 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2674 tmp = RREG32(DC_HPD5_INT_CONTROL);
2675 tmp |= DC_HPDx_INT_ACK;
2676 WREG32(DC_HPD5_INT_CONTROL, tmp);
2677 }
2678 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2679 tmp = RREG32(DC_HPD5_INT_CONTROL);
2680 tmp |= DC_HPDx_INT_ACK;
2681 WREG32(DC_HPD6_INT_CONTROL, tmp);
2682 }
2683}
2684
2685void evergreen_irq_disable(struct radeon_device *rdev)
2686{
2687 r600_disable_interrupts(rdev);
2688 /* Wait and acknowledge irq */
2689 mdelay(1);
2690 evergreen_irq_ack(rdev);
2691 evergreen_disable_interrupt_state(rdev);
2692}
2693
2694void evergreen_irq_suspend(struct radeon_device *rdev)
2695{
2696 evergreen_irq_disable(rdev);
2697 r600_rlc_stop(rdev);
2698}
2699
2700static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
2701{
2702 u32 wptr, tmp;
2703
2704 if (rdev->wb.enabled)
2705 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
2706 else
2707 wptr = RREG32(IH_RB_WPTR);
2708
2709 if (wptr & RB_OVERFLOW) {
2710 /* When a ring buffer overflow happen start parsing interrupt
2711 * from the last not overwritten vector (wptr + 16). Hopefully
2712 * this should allow us to catchup.
2713 */
2714 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2715 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2716 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2717 tmp = RREG32(IH_RB_CNTL);
2718 tmp |= IH_WPTR_OVERFLOW_CLEAR;
2719 WREG32(IH_RB_CNTL, tmp);
2720 }
2721 return (wptr & rdev->ih.ptr_mask);
2722}
2723
2724int evergreen_irq_process(struct radeon_device *rdev)
2725{
2726 u32 wptr;
2727 u32 rptr;
2728 u32 src_id, src_data;
2729 u32 ring_index;
2730 unsigned long flags;
2731 bool queue_hotplug = false;
2732
2733 if (!rdev->ih.enabled || rdev->shutdown)
2734 return IRQ_NONE;
2735
2736 wptr = evergreen_get_ih_wptr(rdev);
2737 rptr = rdev->ih.rptr;
2738 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2739
2740 spin_lock_irqsave(&rdev->ih.lock, flags);
2741 if (rptr == wptr) {
2742 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2743 return IRQ_NONE;
2744 }
2745restart_ih:
2746 /* Order reading of wptr vs. reading of IH ring data */
2747 rmb();
2748
2749 /* display interrupts */
2750 evergreen_irq_ack(rdev);
2751
2752 rdev->ih.wptr = wptr;
2753 while (rptr != wptr) {
2754 /* wptr/rptr are in bytes! */
2755 ring_index = rptr / 4;
2756 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
2757 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
2758
2759 switch (src_id) {
2760 case 1: /* D1 vblank/vline */
2761 switch (src_data) {
2762 case 0: /* D1 vblank */
2763 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
2764 if (rdev->irq.crtc_vblank_int[0]) {
2765 drm_handle_vblank(rdev->ddev, 0);
2766 rdev->pm.vblank_sync = true;
2767 wake_up(&rdev->irq.vblank_queue);
2768 }
2769 if (rdev->irq.pflip[0])
2770 radeon_crtc_handle_flip(rdev, 0);
2771 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2772 DRM_DEBUG("IH: D1 vblank\n");
2773 }
2774 break;
2775 case 1: /* D1 vline */
2776 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
2777 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
2778 DRM_DEBUG("IH: D1 vline\n");
2779 }
2780 break;
2781 default:
2782 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2783 break;
2784 }
2785 break;
2786 case 2: /* D2 vblank/vline */
2787 switch (src_data) {
2788 case 0: /* D2 vblank */
2789 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
2790 if (rdev->irq.crtc_vblank_int[1]) {
2791 drm_handle_vblank(rdev->ddev, 1);
2792 rdev->pm.vblank_sync = true;
2793 wake_up(&rdev->irq.vblank_queue);
2794 }
2795 if (rdev->irq.pflip[1])
2796 radeon_crtc_handle_flip(rdev, 1);
2797 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
2798 DRM_DEBUG("IH: D2 vblank\n");
2799 }
2800 break;
2801 case 1: /* D2 vline */
2802 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
2803 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
2804 DRM_DEBUG("IH: D2 vline\n");
2805 }
2806 break;
2807 default:
2808 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2809 break;
2810 }
2811 break;
2812 case 3: /* D3 vblank/vline */
2813 switch (src_data) {
2814 case 0: /* D3 vblank */
2815 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
2816 if (rdev->irq.crtc_vblank_int[2]) {
2817 drm_handle_vblank(rdev->ddev, 2);
2818 rdev->pm.vblank_sync = true;
2819 wake_up(&rdev->irq.vblank_queue);
2820 }
2821 if (rdev->irq.pflip[2])
2822 radeon_crtc_handle_flip(rdev, 2);
2823 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
2824 DRM_DEBUG("IH: D3 vblank\n");
2825 }
2826 break;
2827 case 1: /* D3 vline */
2828 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
2829 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
2830 DRM_DEBUG("IH: D3 vline\n");
2831 }
2832 break;
2833 default:
2834 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2835 break;
2836 }
2837 break;
2838 case 4: /* D4 vblank/vline */
2839 switch (src_data) {
2840 case 0: /* D4 vblank */
2841 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
2842 if (rdev->irq.crtc_vblank_int[3]) {
2843 drm_handle_vblank(rdev->ddev, 3);
2844 rdev->pm.vblank_sync = true;
2845 wake_up(&rdev->irq.vblank_queue);
2846 }
2847 if (rdev->irq.pflip[3])
2848 radeon_crtc_handle_flip(rdev, 3);
2849 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
2850 DRM_DEBUG("IH: D4 vblank\n");
2851 }
2852 break;
2853 case 1: /* D4 vline */
2854 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
2855 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
2856 DRM_DEBUG("IH: D4 vline\n");
2857 }
2858 break;
2859 default:
2860 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2861 break;
2862 }
2863 break;
2864 case 5: /* D5 vblank/vline */
2865 switch (src_data) {
2866 case 0: /* D5 vblank */
2867 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
2868 if (rdev->irq.crtc_vblank_int[4]) {
2869 drm_handle_vblank(rdev->ddev, 4);
2870 rdev->pm.vblank_sync = true;
2871 wake_up(&rdev->irq.vblank_queue);
2872 }
2873 if (rdev->irq.pflip[4])
2874 radeon_crtc_handle_flip(rdev, 4);
2875 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
2876 DRM_DEBUG("IH: D5 vblank\n");
2877 }
2878 break;
2879 case 1: /* D5 vline */
2880 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
2881 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
2882 DRM_DEBUG("IH: D5 vline\n");
2883 }
2884 break;
2885 default:
2886 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2887 break;
2888 }
2889 break;
2890 case 6: /* D6 vblank/vline */
2891 switch (src_data) {
2892 case 0: /* D6 vblank */
2893 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
2894 if (rdev->irq.crtc_vblank_int[5]) {
2895 drm_handle_vblank(rdev->ddev, 5);
2896 rdev->pm.vblank_sync = true;
2897 wake_up(&rdev->irq.vblank_queue);
2898 }
2899 if (rdev->irq.pflip[5])
2900 radeon_crtc_handle_flip(rdev, 5);
2901 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
2902 DRM_DEBUG("IH: D6 vblank\n");
2903 }
2904 break;
2905 case 1: /* D6 vline */
2906 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
2907 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
2908 DRM_DEBUG("IH: D6 vline\n");
2909 }
2910 break;
2911 default:
2912 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2913 break;
2914 }
2915 break;
2916 case 42: /* HPD hotplug */
2917 switch (src_data) {
2918 case 0:
2919 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2920 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
2921 queue_hotplug = true;
2922 DRM_DEBUG("IH: HPD1\n");
2923 }
2924 break;
2925 case 1:
2926 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2927 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
2928 queue_hotplug = true;
2929 DRM_DEBUG("IH: HPD2\n");
2930 }
2931 break;
2932 case 2:
2933 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2934 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
2935 queue_hotplug = true;
2936 DRM_DEBUG("IH: HPD3\n");
2937 }
2938 break;
2939 case 3:
2940 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2941 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
2942 queue_hotplug = true;
2943 DRM_DEBUG("IH: HPD4\n");
2944 }
2945 break;
2946 case 4:
2947 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2948 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
2949 queue_hotplug = true;
2950 DRM_DEBUG("IH: HPD5\n");
2951 }
2952 break;
2953 case 5:
2954 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2955 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
2956 queue_hotplug = true;
2957 DRM_DEBUG("IH: HPD6\n");
2958 }
2959 break;
2960 default:
2961 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2962 break;
2963 }
2964 break;
2965 case 176: /* CP_INT in ring buffer */
2966 case 177: /* CP_INT in IB1 */
2967 case 178: /* CP_INT in IB2 */
2968 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2969 radeon_fence_process(rdev);
2970 break;
2971 case 181: /* CP EOP event */
2972 DRM_DEBUG("IH: CP EOP\n");
2973 radeon_fence_process(rdev);
2974 break;
2975 case 233: /* GUI IDLE */
2976 DRM_DEBUG("IH: GUI idle\n");
2977 rdev->pm.gui_idle = true;
2978 wake_up(&rdev->irq.idle_queue);
2979 break;
2980 default:
2981 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2982 break;
2983 }
2984
2985 /* wptr/rptr are in bytes! */
2986 rptr += 16;
2987 rptr &= rdev->ih.ptr_mask;
2988 }
2989 /* make sure wptr hasn't changed while processing */
2990 wptr = evergreen_get_ih_wptr(rdev);
2991 if (wptr != rdev->ih.wptr)
2992 goto restart_ih;
2993 if (queue_hotplug)
2994 schedule_work(&rdev->hotplug_work);
2995 rdev->ih.rptr = rptr;
2996 WREG32(IH_RB_RPTR, rdev->ih.rptr);
2997 spin_unlock_irqrestore(&rdev->ih.lock, flags);
2998 return IRQ_HANDLED;
2999}
3000
3001static int evergreen_startup(struct radeon_device *rdev)
3002{
3003 int r;
3004
3005 /* enable pcie gen2 link */
3006 if (!ASIC_IS_DCE5(rdev))
3007 evergreen_pcie_gen2_enable(rdev);
3008
3009 if (ASIC_IS_DCE5(rdev)) {
3010 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3011 r = ni_init_microcode(rdev);
3012 if (r) {
3013 DRM_ERROR("Failed to load firmware!\n");
3014 return r;
3015 }
3016 }
3017 r = ni_mc_load_microcode(rdev);
3018 if (r) {
3019 DRM_ERROR("Failed to load MC firmware!\n");
3020 return r;
3021 }
3022 } else {
3023 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3024 r = r600_init_microcode(rdev);
3025 if (r) {
3026 DRM_ERROR("Failed to load firmware!\n");
3027 return r;
3028 }
3029 }
3030 }
3031
3032 evergreen_mc_program(rdev);
3033 if (rdev->flags & RADEON_IS_AGP) {
3034 evergreen_agp_enable(rdev);
3035 } else {
3036 r = evergreen_pcie_gart_enable(rdev);
3037 if (r)
3038 return r;
3039 }
3040 evergreen_gpu_init(rdev);
3041
3042 r = evergreen_blit_init(rdev);
3043 if (r) {
3044 evergreen_blit_fini(rdev);
3045 rdev->asic->copy = NULL;
3046 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3047 }
3048
3049 /* allocate wb buffer */
3050 r = radeon_wb_init(rdev);
3051 if (r)
3052 return r;
3053
3054 /* Enable IRQ */
3055 r = r600_irq_init(rdev);
3056 if (r) {
3057 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3058 radeon_irq_kms_fini(rdev);
3059 return r;
3060 }
3061 evergreen_irq_set(rdev);
3062
3063 r = radeon_ring_init(rdev, rdev->cp.ring_size);
3064 if (r)
3065 return r;
3066 r = evergreen_cp_load_microcode(rdev);
3067 if (r)
3068 return r;
3069 r = evergreen_cp_resume(rdev);
3070 if (r)
3071 return r;
3072
3073 return 0;
3074}
3075
3076int evergreen_resume(struct radeon_device *rdev)
3077{
3078 int r;
3079
3080 /* reset the asic, the gfx blocks are often in a bad state
3081 * after the driver is unloaded or after a resume
3082 */
3083 if (radeon_asic_reset(rdev))
3084 dev_warn(rdev->dev, "GPU reset failed !\n");
3085 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3086 * posting will perform necessary task to bring back GPU into good
3087 * shape.
3088 */
3089 /* post card */
3090 atom_asic_init(rdev->mode_info.atom_context);
3091
3092 r = evergreen_startup(rdev);
3093 if (r) {
3094 DRM_ERROR("evergreen startup failed on resume\n");
3095 return r;
3096 }
3097
3098 r = r600_ib_test(rdev);
3099 if (r) {
3100 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3101 return r;
3102 }
3103
3104 return r;
3105
3106}
3107
3108int evergreen_suspend(struct radeon_device *rdev)
3109{
3110 int r;
3111
3112 /* FIXME: we should wait for ring to be empty */
3113 r700_cp_stop(rdev);
3114 rdev->cp.ready = false;
3115 evergreen_irq_suspend(rdev);
3116 radeon_wb_disable(rdev);
3117 evergreen_pcie_gart_disable(rdev);
3118
3119 /* unpin shaders bo */
3120 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false);
3121 if (likely(r == 0)) {
3122 radeon_bo_unpin(rdev->r600_blit.shader_obj);
3123 radeon_bo_unreserve(rdev->r600_blit.shader_obj);
3124 }
3125
3126 return 0;
3127}
3128
3129int evergreen_copy_blit(struct radeon_device *rdev,
3130 uint64_t src_offset,
3131 uint64_t dst_offset,
3132 unsigned num_gpu_pages,
3133 struct radeon_fence *fence)
3134{
3135 int r;
3136
3137 mutex_lock(&rdev->r600_blit.mutex);
3138 rdev->r600_blit.vb_ib = NULL;
3139 r = evergreen_blit_prepare_copy(rdev, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
3140 if (r) {
3141 if (rdev->r600_blit.vb_ib)
3142 radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
3143 mutex_unlock(&rdev->r600_blit.mutex);
3144 return r;
3145 }
3146 evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
3147 evergreen_blit_done_copy(rdev, fence);
3148 mutex_unlock(&rdev->r600_blit.mutex);
3149 return 0;
3150}
3151
3152/* Plan is to move initialization in that function and use
3153 * helper function so that radeon_device_init pretty much
3154 * do nothing more than calling asic specific function. This
3155 * should also allow to remove a bunch of callback function
3156 * like vram_info.
3157 */
3158int evergreen_init(struct radeon_device *rdev)
3159{
3160 int r;
3161
3162 /* This don't do much */
3163 r = radeon_gem_init(rdev);
3164 if (r)
3165 return r;
3166 /* Read BIOS */
3167 if (!radeon_get_bios(rdev)) {
3168 if (ASIC_IS_AVIVO(rdev))
3169 return -EINVAL;
3170 }
3171 /* Must be an ATOMBIOS */
3172 if (!rdev->is_atom_bios) {
3173 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
3174 return -EINVAL;
3175 }
3176 r = radeon_atombios_init(rdev);
3177 if (r)
3178 return r;
3179 /* reset the asic, the gfx blocks are often in a bad state
3180 * after the driver is unloaded or after a resume
3181 */
3182 if (radeon_asic_reset(rdev))
3183 dev_warn(rdev->dev, "GPU reset failed !\n");
3184 /* Post card if necessary */
3185 if (!radeon_card_posted(rdev)) {
3186 if (!rdev->bios) {
3187 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3188 return -EINVAL;
3189 }
3190 DRM_INFO("GPU not posted. posting now...\n");
3191 atom_asic_init(rdev->mode_info.atom_context);
3192 }
3193 /* Initialize scratch registers */
3194 r600_scratch_init(rdev);
3195 /* Initialize surface registers */
3196 radeon_surface_init(rdev);
3197 /* Initialize clocks */
3198 radeon_get_clock_info(rdev->ddev);
3199 /* Fence driver */
3200 r = radeon_fence_driver_init(rdev);
3201 if (r)
3202 return r;
3203 /* initialize AGP */
3204 if (rdev->flags & RADEON_IS_AGP) {
3205 r = radeon_agp_init(rdev);
3206 if (r)
3207 radeon_agp_disable(rdev);
3208 }
3209 /* initialize memory controller */
3210 r = evergreen_mc_init(rdev);
3211 if (r)
3212 return r;
3213 /* Memory manager */
3214 r = radeon_bo_init(rdev);
3215 if (r)
3216 return r;
3217
3218 r = radeon_irq_kms_init(rdev);
3219 if (r)
3220 return r;
3221
3222 rdev->cp.ring_obj = NULL;
3223 r600_ring_init(rdev, 1024 * 1024);
3224
3225 rdev->ih.ring_obj = NULL;
3226 r600_ih_ring_init(rdev, 64 * 1024);
3227
3228 r = r600_pcie_gart_init(rdev);
3229 if (r)
3230 return r;
3231
3232 rdev->accel_working = true;
3233 r = evergreen_startup(rdev);
3234 if (r) {
3235 dev_err(rdev->dev, "disabling GPU acceleration\n");
3236 r700_cp_fini(rdev);
3237 r600_irq_fini(rdev);
3238 radeon_wb_fini(rdev);
3239 radeon_irq_kms_fini(rdev);
3240 evergreen_pcie_gart_fini(rdev);
3241 rdev->accel_working = false;
3242 }
3243 if (rdev->accel_working) {
3244 r = radeon_ib_pool_init(rdev);
3245 if (r) {
3246 DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
3247 rdev->accel_working = false;
3248 }
3249 r = r600_ib_test(rdev);
3250 if (r) {
3251 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3252 rdev->accel_working = false;
3253 }
3254 }
3255 return 0;
3256}
3257
3258void evergreen_fini(struct radeon_device *rdev)
3259{
3260 evergreen_blit_fini(rdev);
3261 r700_cp_fini(rdev);
3262 r600_irq_fini(rdev);
3263 radeon_wb_fini(rdev);
3264 radeon_ib_pool_fini(rdev);
3265 radeon_irq_kms_fini(rdev);
3266 evergreen_pcie_gart_fini(rdev);
3267 radeon_gem_fini(rdev);
3268 radeon_fence_driver_fini(rdev);
3269 radeon_agp_fini(rdev);
3270 radeon_bo_fini(rdev);
3271 radeon_atombios_fini(rdev);
3272 kfree(rdev->bios);
3273 rdev->bios = NULL;
3274}
3275
3276static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
3277{
3278 u32 link_width_cntl, speed_cntl;
3279
3280 if (radeon_pcie_gen2 == 0)
3281 return;
3282
3283 if (rdev->flags & RADEON_IS_IGP)
3284 return;
3285
3286 if (!(rdev->flags & RADEON_IS_PCIE))
3287 return;
3288
3289 /* x2 cards have a special sequence */
3290 if (ASIC_IS_X2(rdev))
3291 return;
3292
3293 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3294 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3295 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3296
3297 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3298 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3299 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3300
3301 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3302 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3303 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3304
3305 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3306 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3307 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3308
3309 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3310 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3311 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3312
3313 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3314 speed_cntl |= LC_GEN2_EN_STRAP;
3315 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3316
3317 } else {
3318 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3319 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3320 if (1)
3321 link_width_cntl |= LC_UPCONFIGURE_DIS;
3322 else
3323 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3324 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3325 }
3326}
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24
25#include <linux/firmware.h>
26#include <linux/pci.h>
27#include <linux/slab.h>
28
29#include <drm/drm_edid.h>
30#include <drm/drm_vblank.h>
31#include <drm/radeon_drm.h>
32#include <drm/drm_fourcc.h>
33#include <drm/drm_framebuffer.h>
34
35#include "atom.h"
36#include "avivod.h"
37#include "cik.h"
38#include "ni.h"
39#include "rv770.h"
40#include "evergreen.h"
41#include "evergreen_blit_shaders.h"
42#include "evergreen_reg.h"
43#include "evergreend.h"
44#include "radeon.h"
45#include "radeon_asic.h"
46#include "radeon_audio.h"
47#include "radeon_ucode.h"
48#include "si.h"
49
50#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
51#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
52#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
53
54/*
55 * Indirect registers accessor
56 */
57u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
58{
59 unsigned long flags;
60 u32 r;
61
62 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
63 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
64 r = RREG32(EVERGREEN_CG_IND_DATA);
65 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
66 return r;
67}
68
69void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
70{
71 unsigned long flags;
72
73 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
74 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
75 WREG32(EVERGREEN_CG_IND_DATA, (v));
76 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
77}
78
79u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
80{
81 unsigned long flags;
82 u32 r;
83
84 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
85 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
86 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
87 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
88 return r;
89}
90
91void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
92{
93 unsigned long flags;
94
95 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
96 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
97 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
98 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
99}
100
101u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
102{
103 unsigned long flags;
104 u32 r;
105
106 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
107 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
108 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
109 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
110 return r;
111}
112
113void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
114{
115 unsigned long flags;
116
117 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
118 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
119 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
120 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
121}
122
123static const u32 crtc_offsets[6] =
124{
125 EVERGREEN_CRTC0_REGISTER_OFFSET,
126 EVERGREEN_CRTC1_REGISTER_OFFSET,
127 EVERGREEN_CRTC2_REGISTER_OFFSET,
128 EVERGREEN_CRTC3_REGISTER_OFFSET,
129 EVERGREEN_CRTC4_REGISTER_OFFSET,
130 EVERGREEN_CRTC5_REGISTER_OFFSET
131};
132
133#include "clearstate_evergreen.h"
134
135static const u32 sumo_rlc_save_restore_register_list[] =
136{
137 0x98fc,
138 0x9830,
139 0x9834,
140 0x9838,
141 0x9870,
142 0x9874,
143 0x8a14,
144 0x8b24,
145 0x8bcc,
146 0x8b10,
147 0x8d00,
148 0x8d04,
149 0x8c00,
150 0x8c04,
151 0x8c08,
152 0x8c0c,
153 0x8d8c,
154 0x8c20,
155 0x8c24,
156 0x8c28,
157 0x8c18,
158 0x8c1c,
159 0x8cf0,
160 0x8e2c,
161 0x8e38,
162 0x8c30,
163 0x9508,
164 0x9688,
165 0x9608,
166 0x960c,
167 0x9610,
168 0x9614,
169 0x88c4,
170 0x88d4,
171 0xa008,
172 0x900c,
173 0x9100,
174 0x913c,
175 0x98f8,
176 0x98f4,
177 0x9b7c,
178 0x3f8c,
179 0x8950,
180 0x8954,
181 0x8a18,
182 0x8b28,
183 0x9144,
184 0x9148,
185 0x914c,
186 0x3f90,
187 0x3f94,
188 0x915c,
189 0x9160,
190 0x9178,
191 0x917c,
192 0x9180,
193 0x918c,
194 0x9190,
195 0x9194,
196 0x9198,
197 0x919c,
198 0x91a8,
199 0x91ac,
200 0x91b0,
201 0x91b4,
202 0x91b8,
203 0x91c4,
204 0x91c8,
205 0x91cc,
206 0x91d0,
207 0x91d4,
208 0x91e0,
209 0x91e4,
210 0x91ec,
211 0x91f0,
212 0x91f4,
213 0x9200,
214 0x9204,
215 0x929c,
216 0x9150,
217 0x802c,
218};
219
220static void evergreen_gpu_init(struct radeon_device *rdev);
221void evergreen_fini(struct radeon_device *rdev);
222void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
223void evergreen_program_aspm(struct radeon_device *rdev);
224
225static const u32 evergreen_golden_registers[] =
226{
227 0x3f90, 0xffff0000, 0xff000000,
228 0x9148, 0xffff0000, 0xff000000,
229 0x3f94, 0xffff0000, 0xff000000,
230 0x914c, 0xffff0000, 0xff000000,
231 0x9b7c, 0xffffffff, 0x00000000,
232 0x8a14, 0xffffffff, 0x00000007,
233 0x8b10, 0xffffffff, 0x00000000,
234 0x960c, 0xffffffff, 0x54763210,
235 0x88c4, 0xffffffff, 0x000000c2,
236 0x88d4, 0xffffffff, 0x00000010,
237 0x8974, 0xffffffff, 0x00000000,
238 0xc78, 0x00000080, 0x00000080,
239 0x5eb4, 0xffffffff, 0x00000002,
240 0x5e78, 0xffffffff, 0x001000f0,
241 0x6104, 0x01000300, 0x00000000,
242 0x5bc0, 0x00300000, 0x00000000,
243 0x7030, 0xffffffff, 0x00000011,
244 0x7c30, 0xffffffff, 0x00000011,
245 0x10830, 0xffffffff, 0x00000011,
246 0x11430, 0xffffffff, 0x00000011,
247 0x12030, 0xffffffff, 0x00000011,
248 0x12c30, 0xffffffff, 0x00000011,
249 0xd02c, 0xffffffff, 0x08421000,
250 0x240c, 0xffffffff, 0x00000380,
251 0x8b24, 0xffffffff, 0x00ff0fff,
252 0x28a4c, 0x06000000, 0x06000000,
253 0x10c, 0x00000001, 0x00000001,
254 0x8d00, 0xffffffff, 0x100e4848,
255 0x8d04, 0xffffffff, 0x00164745,
256 0x8c00, 0xffffffff, 0xe4000003,
257 0x8c04, 0xffffffff, 0x40600060,
258 0x8c08, 0xffffffff, 0x001c001c,
259 0x8cf0, 0xffffffff, 0x08e00620,
260 0x8c20, 0xffffffff, 0x00800080,
261 0x8c24, 0xffffffff, 0x00800080,
262 0x8c18, 0xffffffff, 0x20202078,
263 0x8c1c, 0xffffffff, 0x00001010,
264 0x28350, 0xffffffff, 0x00000000,
265 0xa008, 0xffffffff, 0x00010000,
266 0x5c4, 0xffffffff, 0x00000001,
267 0x9508, 0xffffffff, 0x00000002,
268 0x913c, 0x0000000f, 0x0000000a
269};
270
271static const u32 evergreen_golden_registers2[] =
272{
273 0x2f4c, 0xffffffff, 0x00000000,
274 0x54f4, 0xffffffff, 0x00000000,
275 0x54f0, 0xffffffff, 0x00000000,
276 0x5498, 0xffffffff, 0x00000000,
277 0x549c, 0xffffffff, 0x00000000,
278 0x5494, 0xffffffff, 0x00000000,
279 0x53cc, 0xffffffff, 0x00000000,
280 0x53c8, 0xffffffff, 0x00000000,
281 0x53c4, 0xffffffff, 0x00000000,
282 0x53c0, 0xffffffff, 0x00000000,
283 0x53bc, 0xffffffff, 0x00000000,
284 0x53b8, 0xffffffff, 0x00000000,
285 0x53b4, 0xffffffff, 0x00000000,
286 0x53b0, 0xffffffff, 0x00000000
287};
288
289static const u32 cypress_mgcg_init[] =
290{
291 0x802c, 0xffffffff, 0xc0000000,
292 0x5448, 0xffffffff, 0x00000100,
293 0x55e4, 0xffffffff, 0x00000100,
294 0x160c, 0xffffffff, 0x00000100,
295 0x5644, 0xffffffff, 0x00000100,
296 0xc164, 0xffffffff, 0x00000100,
297 0x8a18, 0xffffffff, 0x00000100,
298 0x897c, 0xffffffff, 0x06000100,
299 0x8b28, 0xffffffff, 0x00000100,
300 0x9144, 0xffffffff, 0x00000100,
301 0x9a60, 0xffffffff, 0x00000100,
302 0x9868, 0xffffffff, 0x00000100,
303 0x8d58, 0xffffffff, 0x00000100,
304 0x9510, 0xffffffff, 0x00000100,
305 0x949c, 0xffffffff, 0x00000100,
306 0x9654, 0xffffffff, 0x00000100,
307 0x9030, 0xffffffff, 0x00000100,
308 0x9034, 0xffffffff, 0x00000100,
309 0x9038, 0xffffffff, 0x00000100,
310 0x903c, 0xffffffff, 0x00000100,
311 0x9040, 0xffffffff, 0x00000100,
312 0xa200, 0xffffffff, 0x00000100,
313 0xa204, 0xffffffff, 0x00000100,
314 0xa208, 0xffffffff, 0x00000100,
315 0xa20c, 0xffffffff, 0x00000100,
316 0x971c, 0xffffffff, 0x00000100,
317 0x977c, 0xffffffff, 0x00000100,
318 0x3f80, 0xffffffff, 0x00000100,
319 0xa210, 0xffffffff, 0x00000100,
320 0xa214, 0xffffffff, 0x00000100,
321 0x4d8, 0xffffffff, 0x00000100,
322 0x9784, 0xffffffff, 0x00000100,
323 0x9698, 0xffffffff, 0x00000100,
324 0x4d4, 0xffffffff, 0x00000200,
325 0x30cc, 0xffffffff, 0x00000100,
326 0xd0c0, 0xffffffff, 0xff000100,
327 0x802c, 0xffffffff, 0x40000000,
328 0x915c, 0xffffffff, 0x00010000,
329 0x9160, 0xffffffff, 0x00030002,
330 0x9178, 0xffffffff, 0x00070000,
331 0x917c, 0xffffffff, 0x00030002,
332 0x9180, 0xffffffff, 0x00050004,
333 0x918c, 0xffffffff, 0x00010006,
334 0x9190, 0xffffffff, 0x00090008,
335 0x9194, 0xffffffff, 0x00070000,
336 0x9198, 0xffffffff, 0x00030002,
337 0x919c, 0xffffffff, 0x00050004,
338 0x91a8, 0xffffffff, 0x00010006,
339 0x91ac, 0xffffffff, 0x00090008,
340 0x91b0, 0xffffffff, 0x00070000,
341 0x91b4, 0xffffffff, 0x00030002,
342 0x91b8, 0xffffffff, 0x00050004,
343 0x91c4, 0xffffffff, 0x00010006,
344 0x91c8, 0xffffffff, 0x00090008,
345 0x91cc, 0xffffffff, 0x00070000,
346 0x91d0, 0xffffffff, 0x00030002,
347 0x91d4, 0xffffffff, 0x00050004,
348 0x91e0, 0xffffffff, 0x00010006,
349 0x91e4, 0xffffffff, 0x00090008,
350 0x91e8, 0xffffffff, 0x00000000,
351 0x91ec, 0xffffffff, 0x00070000,
352 0x91f0, 0xffffffff, 0x00030002,
353 0x91f4, 0xffffffff, 0x00050004,
354 0x9200, 0xffffffff, 0x00010006,
355 0x9204, 0xffffffff, 0x00090008,
356 0x9208, 0xffffffff, 0x00070000,
357 0x920c, 0xffffffff, 0x00030002,
358 0x9210, 0xffffffff, 0x00050004,
359 0x921c, 0xffffffff, 0x00010006,
360 0x9220, 0xffffffff, 0x00090008,
361 0x9224, 0xffffffff, 0x00070000,
362 0x9228, 0xffffffff, 0x00030002,
363 0x922c, 0xffffffff, 0x00050004,
364 0x9238, 0xffffffff, 0x00010006,
365 0x923c, 0xffffffff, 0x00090008,
366 0x9240, 0xffffffff, 0x00070000,
367 0x9244, 0xffffffff, 0x00030002,
368 0x9248, 0xffffffff, 0x00050004,
369 0x9254, 0xffffffff, 0x00010006,
370 0x9258, 0xffffffff, 0x00090008,
371 0x925c, 0xffffffff, 0x00070000,
372 0x9260, 0xffffffff, 0x00030002,
373 0x9264, 0xffffffff, 0x00050004,
374 0x9270, 0xffffffff, 0x00010006,
375 0x9274, 0xffffffff, 0x00090008,
376 0x9278, 0xffffffff, 0x00070000,
377 0x927c, 0xffffffff, 0x00030002,
378 0x9280, 0xffffffff, 0x00050004,
379 0x928c, 0xffffffff, 0x00010006,
380 0x9290, 0xffffffff, 0x00090008,
381 0x9294, 0xffffffff, 0x00000000,
382 0x929c, 0xffffffff, 0x00000001,
383 0x802c, 0xffffffff, 0x40010000,
384 0x915c, 0xffffffff, 0x00010000,
385 0x9160, 0xffffffff, 0x00030002,
386 0x9178, 0xffffffff, 0x00070000,
387 0x917c, 0xffffffff, 0x00030002,
388 0x9180, 0xffffffff, 0x00050004,
389 0x918c, 0xffffffff, 0x00010006,
390 0x9190, 0xffffffff, 0x00090008,
391 0x9194, 0xffffffff, 0x00070000,
392 0x9198, 0xffffffff, 0x00030002,
393 0x919c, 0xffffffff, 0x00050004,
394 0x91a8, 0xffffffff, 0x00010006,
395 0x91ac, 0xffffffff, 0x00090008,
396 0x91b0, 0xffffffff, 0x00070000,
397 0x91b4, 0xffffffff, 0x00030002,
398 0x91b8, 0xffffffff, 0x00050004,
399 0x91c4, 0xffffffff, 0x00010006,
400 0x91c8, 0xffffffff, 0x00090008,
401 0x91cc, 0xffffffff, 0x00070000,
402 0x91d0, 0xffffffff, 0x00030002,
403 0x91d4, 0xffffffff, 0x00050004,
404 0x91e0, 0xffffffff, 0x00010006,
405 0x91e4, 0xffffffff, 0x00090008,
406 0x91e8, 0xffffffff, 0x00000000,
407 0x91ec, 0xffffffff, 0x00070000,
408 0x91f0, 0xffffffff, 0x00030002,
409 0x91f4, 0xffffffff, 0x00050004,
410 0x9200, 0xffffffff, 0x00010006,
411 0x9204, 0xffffffff, 0x00090008,
412 0x9208, 0xffffffff, 0x00070000,
413 0x920c, 0xffffffff, 0x00030002,
414 0x9210, 0xffffffff, 0x00050004,
415 0x921c, 0xffffffff, 0x00010006,
416 0x9220, 0xffffffff, 0x00090008,
417 0x9224, 0xffffffff, 0x00070000,
418 0x9228, 0xffffffff, 0x00030002,
419 0x922c, 0xffffffff, 0x00050004,
420 0x9238, 0xffffffff, 0x00010006,
421 0x923c, 0xffffffff, 0x00090008,
422 0x9240, 0xffffffff, 0x00070000,
423 0x9244, 0xffffffff, 0x00030002,
424 0x9248, 0xffffffff, 0x00050004,
425 0x9254, 0xffffffff, 0x00010006,
426 0x9258, 0xffffffff, 0x00090008,
427 0x925c, 0xffffffff, 0x00070000,
428 0x9260, 0xffffffff, 0x00030002,
429 0x9264, 0xffffffff, 0x00050004,
430 0x9270, 0xffffffff, 0x00010006,
431 0x9274, 0xffffffff, 0x00090008,
432 0x9278, 0xffffffff, 0x00070000,
433 0x927c, 0xffffffff, 0x00030002,
434 0x9280, 0xffffffff, 0x00050004,
435 0x928c, 0xffffffff, 0x00010006,
436 0x9290, 0xffffffff, 0x00090008,
437 0x9294, 0xffffffff, 0x00000000,
438 0x929c, 0xffffffff, 0x00000001,
439 0x802c, 0xffffffff, 0xc0000000
440};
441
442static const u32 redwood_mgcg_init[] =
443{
444 0x802c, 0xffffffff, 0xc0000000,
445 0x5448, 0xffffffff, 0x00000100,
446 0x55e4, 0xffffffff, 0x00000100,
447 0x160c, 0xffffffff, 0x00000100,
448 0x5644, 0xffffffff, 0x00000100,
449 0xc164, 0xffffffff, 0x00000100,
450 0x8a18, 0xffffffff, 0x00000100,
451 0x897c, 0xffffffff, 0x06000100,
452 0x8b28, 0xffffffff, 0x00000100,
453 0x9144, 0xffffffff, 0x00000100,
454 0x9a60, 0xffffffff, 0x00000100,
455 0x9868, 0xffffffff, 0x00000100,
456 0x8d58, 0xffffffff, 0x00000100,
457 0x9510, 0xffffffff, 0x00000100,
458 0x949c, 0xffffffff, 0x00000100,
459 0x9654, 0xffffffff, 0x00000100,
460 0x9030, 0xffffffff, 0x00000100,
461 0x9034, 0xffffffff, 0x00000100,
462 0x9038, 0xffffffff, 0x00000100,
463 0x903c, 0xffffffff, 0x00000100,
464 0x9040, 0xffffffff, 0x00000100,
465 0xa200, 0xffffffff, 0x00000100,
466 0xa204, 0xffffffff, 0x00000100,
467 0xa208, 0xffffffff, 0x00000100,
468 0xa20c, 0xffffffff, 0x00000100,
469 0x971c, 0xffffffff, 0x00000100,
470 0x977c, 0xffffffff, 0x00000100,
471 0x3f80, 0xffffffff, 0x00000100,
472 0xa210, 0xffffffff, 0x00000100,
473 0xa214, 0xffffffff, 0x00000100,
474 0x4d8, 0xffffffff, 0x00000100,
475 0x9784, 0xffffffff, 0x00000100,
476 0x9698, 0xffffffff, 0x00000100,
477 0x4d4, 0xffffffff, 0x00000200,
478 0x30cc, 0xffffffff, 0x00000100,
479 0xd0c0, 0xffffffff, 0xff000100,
480 0x802c, 0xffffffff, 0x40000000,
481 0x915c, 0xffffffff, 0x00010000,
482 0x9160, 0xffffffff, 0x00030002,
483 0x9178, 0xffffffff, 0x00070000,
484 0x917c, 0xffffffff, 0x00030002,
485 0x9180, 0xffffffff, 0x00050004,
486 0x918c, 0xffffffff, 0x00010006,
487 0x9190, 0xffffffff, 0x00090008,
488 0x9194, 0xffffffff, 0x00070000,
489 0x9198, 0xffffffff, 0x00030002,
490 0x919c, 0xffffffff, 0x00050004,
491 0x91a8, 0xffffffff, 0x00010006,
492 0x91ac, 0xffffffff, 0x00090008,
493 0x91b0, 0xffffffff, 0x00070000,
494 0x91b4, 0xffffffff, 0x00030002,
495 0x91b8, 0xffffffff, 0x00050004,
496 0x91c4, 0xffffffff, 0x00010006,
497 0x91c8, 0xffffffff, 0x00090008,
498 0x91cc, 0xffffffff, 0x00070000,
499 0x91d0, 0xffffffff, 0x00030002,
500 0x91d4, 0xffffffff, 0x00050004,
501 0x91e0, 0xffffffff, 0x00010006,
502 0x91e4, 0xffffffff, 0x00090008,
503 0x91e8, 0xffffffff, 0x00000000,
504 0x91ec, 0xffffffff, 0x00070000,
505 0x91f0, 0xffffffff, 0x00030002,
506 0x91f4, 0xffffffff, 0x00050004,
507 0x9200, 0xffffffff, 0x00010006,
508 0x9204, 0xffffffff, 0x00090008,
509 0x9294, 0xffffffff, 0x00000000,
510 0x929c, 0xffffffff, 0x00000001,
511 0x802c, 0xffffffff, 0xc0000000
512};
513
514static const u32 cedar_golden_registers[] =
515{
516 0x3f90, 0xffff0000, 0xff000000,
517 0x9148, 0xffff0000, 0xff000000,
518 0x3f94, 0xffff0000, 0xff000000,
519 0x914c, 0xffff0000, 0xff000000,
520 0x9b7c, 0xffffffff, 0x00000000,
521 0x8a14, 0xffffffff, 0x00000007,
522 0x8b10, 0xffffffff, 0x00000000,
523 0x960c, 0xffffffff, 0x54763210,
524 0x88c4, 0xffffffff, 0x000000c2,
525 0x88d4, 0xffffffff, 0x00000000,
526 0x8974, 0xffffffff, 0x00000000,
527 0xc78, 0x00000080, 0x00000080,
528 0x5eb4, 0xffffffff, 0x00000002,
529 0x5e78, 0xffffffff, 0x001000f0,
530 0x6104, 0x01000300, 0x00000000,
531 0x5bc0, 0x00300000, 0x00000000,
532 0x7030, 0xffffffff, 0x00000011,
533 0x7c30, 0xffffffff, 0x00000011,
534 0x10830, 0xffffffff, 0x00000011,
535 0x11430, 0xffffffff, 0x00000011,
536 0xd02c, 0xffffffff, 0x08421000,
537 0x240c, 0xffffffff, 0x00000380,
538 0x8b24, 0xffffffff, 0x00ff0fff,
539 0x28a4c, 0x06000000, 0x06000000,
540 0x10c, 0x00000001, 0x00000001,
541 0x8d00, 0xffffffff, 0x100e4848,
542 0x8d04, 0xffffffff, 0x00164745,
543 0x8c00, 0xffffffff, 0xe4000003,
544 0x8c04, 0xffffffff, 0x40600060,
545 0x8c08, 0xffffffff, 0x001c001c,
546 0x8cf0, 0xffffffff, 0x08e00410,
547 0x8c20, 0xffffffff, 0x00800080,
548 0x8c24, 0xffffffff, 0x00800080,
549 0x8c18, 0xffffffff, 0x20202078,
550 0x8c1c, 0xffffffff, 0x00001010,
551 0x28350, 0xffffffff, 0x00000000,
552 0xa008, 0xffffffff, 0x00010000,
553 0x5c4, 0xffffffff, 0x00000001,
554 0x9508, 0xffffffff, 0x00000002
555};
556
557static const u32 cedar_mgcg_init[] =
558{
559 0x802c, 0xffffffff, 0xc0000000,
560 0x5448, 0xffffffff, 0x00000100,
561 0x55e4, 0xffffffff, 0x00000100,
562 0x160c, 0xffffffff, 0x00000100,
563 0x5644, 0xffffffff, 0x00000100,
564 0xc164, 0xffffffff, 0x00000100,
565 0x8a18, 0xffffffff, 0x00000100,
566 0x897c, 0xffffffff, 0x06000100,
567 0x8b28, 0xffffffff, 0x00000100,
568 0x9144, 0xffffffff, 0x00000100,
569 0x9a60, 0xffffffff, 0x00000100,
570 0x9868, 0xffffffff, 0x00000100,
571 0x8d58, 0xffffffff, 0x00000100,
572 0x9510, 0xffffffff, 0x00000100,
573 0x949c, 0xffffffff, 0x00000100,
574 0x9654, 0xffffffff, 0x00000100,
575 0x9030, 0xffffffff, 0x00000100,
576 0x9034, 0xffffffff, 0x00000100,
577 0x9038, 0xffffffff, 0x00000100,
578 0x903c, 0xffffffff, 0x00000100,
579 0x9040, 0xffffffff, 0x00000100,
580 0xa200, 0xffffffff, 0x00000100,
581 0xa204, 0xffffffff, 0x00000100,
582 0xa208, 0xffffffff, 0x00000100,
583 0xa20c, 0xffffffff, 0x00000100,
584 0x971c, 0xffffffff, 0x00000100,
585 0x977c, 0xffffffff, 0x00000100,
586 0x3f80, 0xffffffff, 0x00000100,
587 0xa210, 0xffffffff, 0x00000100,
588 0xa214, 0xffffffff, 0x00000100,
589 0x4d8, 0xffffffff, 0x00000100,
590 0x9784, 0xffffffff, 0x00000100,
591 0x9698, 0xffffffff, 0x00000100,
592 0x4d4, 0xffffffff, 0x00000200,
593 0x30cc, 0xffffffff, 0x00000100,
594 0xd0c0, 0xffffffff, 0xff000100,
595 0x802c, 0xffffffff, 0x40000000,
596 0x915c, 0xffffffff, 0x00010000,
597 0x9178, 0xffffffff, 0x00050000,
598 0x917c, 0xffffffff, 0x00030002,
599 0x918c, 0xffffffff, 0x00010004,
600 0x9190, 0xffffffff, 0x00070006,
601 0x9194, 0xffffffff, 0x00050000,
602 0x9198, 0xffffffff, 0x00030002,
603 0x91a8, 0xffffffff, 0x00010004,
604 0x91ac, 0xffffffff, 0x00070006,
605 0x91e8, 0xffffffff, 0x00000000,
606 0x9294, 0xffffffff, 0x00000000,
607 0x929c, 0xffffffff, 0x00000001,
608 0x802c, 0xffffffff, 0xc0000000
609};
610
611static const u32 juniper_mgcg_init[] =
612{
613 0x802c, 0xffffffff, 0xc0000000,
614 0x5448, 0xffffffff, 0x00000100,
615 0x55e4, 0xffffffff, 0x00000100,
616 0x160c, 0xffffffff, 0x00000100,
617 0x5644, 0xffffffff, 0x00000100,
618 0xc164, 0xffffffff, 0x00000100,
619 0x8a18, 0xffffffff, 0x00000100,
620 0x897c, 0xffffffff, 0x06000100,
621 0x8b28, 0xffffffff, 0x00000100,
622 0x9144, 0xffffffff, 0x00000100,
623 0x9a60, 0xffffffff, 0x00000100,
624 0x9868, 0xffffffff, 0x00000100,
625 0x8d58, 0xffffffff, 0x00000100,
626 0x9510, 0xffffffff, 0x00000100,
627 0x949c, 0xffffffff, 0x00000100,
628 0x9654, 0xffffffff, 0x00000100,
629 0x9030, 0xffffffff, 0x00000100,
630 0x9034, 0xffffffff, 0x00000100,
631 0x9038, 0xffffffff, 0x00000100,
632 0x903c, 0xffffffff, 0x00000100,
633 0x9040, 0xffffffff, 0x00000100,
634 0xa200, 0xffffffff, 0x00000100,
635 0xa204, 0xffffffff, 0x00000100,
636 0xa208, 0xffffffff, 0x00000100,
637 0xa20c, 0xffffffff, 0x00000100,
638 0x971c, 0xffffffff, 0x00000100,
639 0xd0c0, 0xffffffff, 0xff000100,
640 0x802c, 0xffffffff, 0x40000000,
641 0x915c, 0xffffffff, 0x00010000,
642 0x9160, 0xffffffff, 0x00030002,
643 0x9178, 0xffffffff, 0x00070000,
644 0x917c, 0xffffffff, 0x00030002,
645 0x9180, 0xffffffff, 0x00050004,
646 0x918c, 0xffffffff, 0x00010006,
647 0x9190, 0xffffffff, 0x00090008,
648 0x9194, 0xffffffff, 0x00070000,
649 0x9198, 0xffffffff, 0x00030002,
650 0x919c, 0xffffffff, 0x00050004,
651 0x91a8, 0xffffffff, 0x00010006,
652 0x91ac, 0xffffffff, 0x00090008,
653 0x91b0, 0xffffffff, 0x00070000,
654 0x91b4, 0xffffffff, 0x00030002,
655 0x91b8, 0xffffffff, 0x00050004,
656 0x91c4, 0xffffffff, 0x00010006,
657 0x91c8, 0xffffffff, 0x00090008,
658 0x91cc, 0xffffffff, 0x00070000,
659 0x91d0, 0xffffffff, 0x00030002,
660 0x91d4, 0xffffffff, 0x00050004,
661 0x91e0, 0xffffffff, 0x00010006,
662 0x91e4, 0xffffffff, 0x00090008,
663 0x91e8, 0xffffffff, 0x00000000,
664 0x91ec, 0xffffffff, 0x00070000,
665 0x91f0, 0xffffffff, 0x00030002,
666 0x91f4, 0xffffffff, 0x00050004,
667 0x9200, 0xffffffff, 0x00010006,
668 0x9204, 0xffffffff, 0x00090008,
669 0x9208, 0xffffffff, 0x00070000,
670 0x920c, 0xffffffff, 0x00030002,
671 0x9210, 0xffffffff, 0x00050004,
672 0x921c, 0xffffffff, 0x00010006,
673 0x9220, 0xffffffff, 0x00090008,
674 0x9224, 0xffffffff, 0x00070000,
675 0x9228, 0xffffffff, 0x00030002,
676 0x922c, 0xffffffff, 0x00050004,
677 0x9238, 0xffffffff, 0x00010006,
678 0x923c, 0xffffffff, 0x00090008,
679 0x9240, 0xffffffff, 0x00070000,
680 0x9244, 0xffffffff, 0x00030002,
681 0x9248, 0xffffffff, 0x00050004,
682 0x9254, 0xffffffff, 0x00010006,
683 0x9258, 0xffffffff, 0x00090008,
684 0x925c, 0xffffffff, 0x00070000,
685 0x9260, 0xffffffff, 0x00030002,
686 0x9264, 0xffffffff, 0x00050004,
687 0x9270, 0xffffffff, 0x00010006,
688 0x9274, 0xffffffff, 0x00090008,
689 0x9278, 0xffffffff, 0x00070000,
690 0x927c, 0xffffffff, 0x00030002,
691 0x9280, 0xffffffff, 0x00050004,
692 0x928c, 0xffffffff, 0x00010006,
693 0x9290, 0xffffffff, 0x00090008,
694 0x9294, 0xffffffff, 0x00000000,
695 0x929c, 0xffffffff, 0x00000001,
696 0x802c, 0xffffffff, 0xc0000000,
697 0x977c, 0xffffffff, 0x00000100,
698 0x3f80, 0xffffffff, 0x00000100,
699 0xa210, 0xffffffff, 0x00000100,
700 0xa214, 0xffffffff, 0x00000100,
701 0x4d8, 0xffffffff, 0x00000100,
702 0x9784, 0xffffffff, 0x00000100,
703 0x9698, 0xffffffff, 0x00000100,
704 0x4d4, 0xffffffff, 0x00000200,
705 0x30cc, 0xffffffff, 0x00000100,
706 0x802c, 0xffffffff, 0xc0000000
707};
708
709static const u32 supersumo_golden_registers[] =
710{
711 0x5eb4, 0xffffffff, 0x00000002,
712 0x5c4, 0xffffffff, 0x00000001,
713 0x7030, 0xffffffff, 0x00000011,
714 0x7c30, 0xffffffff, 0x00000011,
715 0x6104, 0x01000300, 0x00000000,
716 0x5bc0, 0x00300000, 0x00000000,
717 0x8c04, 0xffffffff, 0x40600060,
718 0x8c08, 0xffffffff, 0x001c001c,
719 0x8c20, 0xffffffff, 0x00800080,
720 0x8c24, 0xffffffff, 0x00800080,
721 0x8c18, 0xffffffff, 0x20202078,
722 0x8c1c, 0xffffffff, 0x00001010,
723 0x918c, 0xffffffff, 0x00010006,
724 0x91a8, 0xffffffff, 0x00010006,
725 0x91c4, 0xffffffff, 0x00010006,
726 0x91e0, 0xffffffff, 0x00010006,
727 0x9200, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9180, 0xffffffff, 0x00050004,
731 0x9198, 0xffffffff, 0x00030002,
732 0x919c, 0xffffffff, 0x00050004,
733 0x91b4, 0xffffffff, 0x00030002,
734 0x91b8, 0xffffffff, 0x00050004,
735 0x91d0, 0xffffffff, 0x00030002,
736 0x91d4, 0xffffffff, 0x00050004,
737 0x91f0, 0xffffffff, 0x00030002,
738 0x91f4, 0xffffffff, 0x00050004,
739 0x915c, 0xffffffff, 0x00010000,
740 0x9160, 0xffffffff, 0x00030002,
741 0x3f90, 0xffff0000, 0xff000000,
742 0x9178, 0xffffffff, 0x00070000,
743 0x9194, 0xffffffff, 0x00070000,
744 0x91b0, 0xffffffff, 0x00070000,
745 0x91cc, 0xffffffff, 0x00070000,
746 0x91ec, 0xffffffff, 0x00070000,
747 0x9148, 0xffff0000, 0xff000000,
748 0x9190, 0xffffffff, 0x00090008,
749 0x91ac, 0xffffffff, 0x00090008,
750 0x91c8, 0xffffffff, 0x00090008,
751 0x91e4, 0xffffffff, 0x00090008,
752 0x9204, 0xffffffff, 0x00090008,
753 0x3f94, 0xffff0000, 0xff000000,
754 0x914c, 0xffff0000, 0xff000000,
755 0x929c, 0xffffffff, 0x00000001,
756 0x8a18, 0xffffffff, 0x00000100,
757 0x8b28, 0xffffffff, 0x00000100,
758 0x9144, 0xffffffff, 0x00000100,
759 0x5644, 0xffffffff, 0x00000100,
760 0x9b7c, 0xffffffff, 0x00000000,
761 0x8030, 0xffffffff, 0x0000100a,
762 0x8a14, 0xffffffff, 0x00000007,
763 0x8b24, 0xffffffff, 0x00ff0fff,
764 0x8b10, 0xffffffff, 0x00000000,
765 0x28a4c, 0x06000000, 0x06000000,
766 0x4d8, 0xffffffff, 0x00000100,
767 0x913c, 0xffff000f, 0x0100000a,
768 0x960c, 0xffffffff, 0x54763210,
769 0x88c4, 0xffffffff, 0x000000c2,
770 0x88d4, 0xffffffff, 0x00000010,
771 0x8974, 0xffffffff, 0x00000000,
772 0xc78, 0x00000080, 0x00000080,
773 0x5e78, 0xffffffff, 0x001000f0,
774 0xd02c, 0xffffffff, 0x08421000,
775 0xa008, 0xffffffff, 0x00010000,
776 0x8d00, 0xffffffff, 0x100e4848,
777 0x8d04, 0xffffffff, 0x00164745,
778 0x8c00, 0xffffffff, 0xe4000003,
779 0x8cf0, 0x1fffffff, 0x08e00620,
780 0x28350, 0xffffffff, 0x00000000,
781 0x9508, 0xffffffff, 0x00000002
782};
783
784static const u32 sumo_golden_registers[] =
785{
786 0x900c, 0x00ffffff, 0x0017071f,
787 0x8c18, 0xffffffff, 0x10101060,
788 0x8c1c, 0xffffffff, 0x00001010,
789 0x8c30, 0x0000000f, 0x00000005,
790 0x9688, 0x0000000f, 0x00000007
791};
792
793static const u32 wrestler_golden_registers[] =
794{
795 0x5eb4, 0xffffffff, 0x00000002,
796 0x5c4, 0xffffffff, 0x00000001,
797 0x7030, 0xffffffff, 0x00000011,
798 0x7c30, 0xffffffff, 0x00000011,
799 0x6104, 0x01000300, 0x00000000,
800 0x5bc0, 0x00300000, 0x00000000,
801 0x918c, 0xffffffff, 0x00010006,
802 0x91a8, 0xffffffff, 0x00010006,
803 0x9150, 0xffffffff, 0x6e944040,
804 0x917c, 0xffffffff, 0x00030002,
805 0x9198, 0xffffffff, 0x00030002,
806 0x915c, 0xffffffff, 0x00010000,
807 0x3f90, 0xffff0000, 0xff000000,
808 0x9178, 0xffffffff, 0x00070000,
809 0x9194, 0xffffffff, 0x00070000,
810 0x9148, 0xffff0000, 0xff000000,
811 0x9190, 0xffffffff, 0x00090008,
812 0x91ac, 0xffffffff, 0x00090008,
813 0x3f94, 0xffff0000, 0xff000000,
814 0x914c, 0xffff0000, 0xff000000,
815 0x929c, 0xffffffff, 0x00000001,
816 0x8a18, 0xffffffff, 0x00000100,
817 0x8b28, 0xffffffff, 0x00000100,
818 0x9144, 0xffffffff, 0x00000100,
819 0x9b7c, 0xffffffff, 0x00000000,
820 0x8030, 0xffffffff, 0x0000100a,
821 0x8a14, 0xffffffff, 0x00000001,
822 0x8b24, 0xffffffff, 0x00ff0fff,
823 0x8b10, 0xffffffff, 0x00000000,
824 0x28a4c, 0x06000000, 0x06000000,
825 0x4d8, 0xffffffff, 0x00000100,
826 0x913c, 0xffff000f, 0x0100000a,
827 0x960c, 0xffffffff, 0x54763210,
828 0x88c4, 0xffffffff, 0x000000c2,
829 0x88d4, 0xffffffff, 0x00000010,
830 0x8974, 0xffffffff, 0x00000000,
831 0xc78, 0x00000080, 0x00000080,
832 0x5e78, 0xffffffff, 0x001000f0,
833 0xd02c, 0xffffffff, 0x08421000,
834 0xa008, 0xffffffff, 0x00010000,
835 0x8d00, 0xffffffff, 0x100e4848,
836 0x8d04, 0xffffffff, 0x00164745,
837 0x8c00, 0xffffffff, 0xe4000003,
838 0x8cf0, 0x1fffffff, 0x08e00410,
839 0x28350, 0xffffffff, 0x00000000,
840 0x9508, 0xffffffff, 0x00000002,
841 0x900c, 0xffffffff, 0x0017071f,
842 0x8c18, 0xffffffff, 0x10101060,
843 0x8c1c, 0xffffffff, 0x00001010
844};
845
846static const u32 barts_golden_registers[] =
847{
848 0x5eb4, 0xffffffff, 0x00000002,
849 0x5e78, 0x8f311ff1, 0x001000f0,
850 0x3f90, 0xffff0000, 0xff000000,
851 0x9148, 0xffff0000, 0xff000000,
852 0x3f94, 0xffff0000, 0xff000000,
853 0x914c, 0xffff0000, 0xff000000,
854 0xc78, 0x00000080, 0x00000080,
855 0xbd4, 0x70073777, 0x00010001,
856 0xd02c, 0xbfffff1f, 0x08421000,
857 0xd0b8, 0x03773777, 0x02011003,
858 0x5bc0, 0x00200000, 0x50100000,
859 0x98f8, 0x33773777, 0x02011003,
860 0x98fc, 0xffffffff, 0x76543210,
861 0x7030, 0x31000311, 0x00000011,
862 0x2f48, 0x00000007, 0x02011003,
863 0x6b28, 0x00000010, 0x00000012,
864 0x7728, 0x00000010, 0x00000012,
865 0x10328, 0x00000010, 0x00000012,
866 0x10f28, 0x00000010, 0x00000012,
867 0x11b28, 0x00000010, 0x00000012,
868 0x12728, 0x00000010, 0x00000012,
869 0x240c, 0x000007ff, 0x00000380,
870 0x8a14, 0xf000001f, 0x00000007,
871 0x8b24, 0x3fff3fff, 0x00ff0fff,
872 0x8b10, 0x0000ff0f, 0x00000000,
873 0x28a4c, 0x07ffffff, 0x06000000,
874 0x10c, 0x00000001, 0x00010003,
875 0xa02c, 0xffffffff, 0x0000009b,
876 0x913c, 0x0000000f, 0x0100000a,
877 0x8d00, 0xffff7f7f, 0x100e4848,
878 0x8d04, 0x00ffffff, 0x00164745,
879 0x8c00, 0xfffc0003, 0xe4000003,
880 0x8c04, 0xf8ff00ff, 0x40600060,
881 0x8c08, 0x00ff00ff, 0x001c001c,
882 0x8cf0, 0x1fff1fff, 0x08e00620,
883 0x8c20, 0x0fff0fff, 0x00800080,
884 0x8c24, 0x0fff0fff, 0x00800080,
885 0x8c18, 0xffffffff, 0x20202078,
886 0x8c1c, 0x0000ffff, 0x00001010,
887 0x28350, 0x00000f01, 0x00000000,
888 0x9508, 0x3700001f, 0x00000002,
889 0x960c, 0xffffffff, 0x54763210,
890 0x88c4, 0x001f3ae3, 0x000000c2,
891 0x88d4, 0x0000001f, 0x00000010,
892 0x8974, 0xffffffff, 0x00000000
893};
894
895static const u32 turks_golden_registers[] =
896{
897 0x5eb4, 0xffffffff, 0x00000002,
898 0x5e78, 0x8f311ff1, 0x001000f0,
899 0x8c8, 0x00003000, 0x00001070,
900 0x8cc, 0x000fffff, 0x00040035,
901 0x3f90, 0xffff0000, 0xfff00000,
902 0x9148, 0xffff0000, 0xfff00000,
903 0x3f94, 0xffff0000, 0xfff00000,
904 0x914c, 0xffff0000, 0xfff00000,
905 0xc78, 0x00000080, 0x00000080,
906 0xbd4, 0x00073007, 0x00010002,
907 0xd02c, 0xbfffff1f, 0x08421000,
908 0xd0b8, 0x03773777, 0x02010002,
909 0x5bc0, 0x00200000, 0x50100000,
910 0x98f8, 0x33773777, 0x00010002,
911 0x98fc, 0xffffffff, 0x33221100,
912 0x7030, 0x31000311, 0x00000011,
913 0x2f48, 0x33773777, 0x00010002,
914 0x6b28, 0x00000010, 0x00000012,
915 0x7728, 0x00000010, 0x00000012,
916 0x10328, 0x00000010, 0x00000012,
917 0x10f28, 0x00000010, 0x00000012,
918 0x11b28, 0x00000010, 0x00000012,
919 0x12728, 0x00000010, 0x00000012,
920 0x240c, 0x000007ff, 0x00000380,
921 0x8a14, 0xf000001f, 0x00000007,
922 0x8b24, 0x3fff3fff, 0x00ff0fff,
923 0x8b10, 0x0000ff0f, 0x00000000,
924 0x28a4c, 0x07ffffff, 0x06000000,
925 0x10c, 0x00000001, 0x00010003,
926 0xa02c, 0xffffffff, 0x0000009b,
927 0x913c, 0x0000000f, 0x0100000a,
928 0x8d00, 0xffff7f7f, 0x100e4848,
929 0x8d04, 0x00ffffff, 0x00164745,
930 0x8c00, 0xfffc0003, 0xe4000003,
931 0x8c04, 0xf8ff00ff, 0x40600060,
932 0x8c08, 0x00ff00ff, 0x001c001c,
933 0x8cf0, 0x1fff1fff, 0x08e00410,
934 0x8c20, 0x0fff0fff, 0x00800080,
935 0x8c24, 0x0fff0fff, 0x00800080,
936 0x8c18, 0xffffffff, 0x20202078,
937 0x8c1c, 0x0000ffff, 0x00001010,
938 0x28350, 0x00000f01, 0x00000000,
939 0x9508, 0x3700001f, 0x00000002,
940 0x960c, 0xffffffff, 0x54763210,
941 0x88c4, 0x001f3ae3, 0x000000c2,
942 0x88d4, 0x0000001f, 0x00000010,
943 0x8974, 0xffffffff, 0x00000000
944};
945
946static const u32 caicos_golden_registers[] =
947{
948 0x5eb4, 0xffffffff, 0x00000002,
949 0x5e78, 0x8f311ff1, 0x001000f0,
950 0x8c8, 0x00003420, 0x00001450,
951 0x8cc, 0x000fffff, 0x00040035,
952 0x3f90, 0xffff0000, 0xfffc0000,
953 0x9148, 0xffff0000, 0xfffc0000,
954 0x3f94, 0xffff0000, 0xfffc0000,
955 0x914c, 0xffff0000, 0xfffc0000,
956 0xc78, 0x00000080, 0x00000080,
957 0xbd4, 0x00073007, 0x00010001,
958 0xd02c, 0xbfffff1f, 0x08421000,
959 0xd0b8, 0x03773777, 0x02010001,
960 0x5bc0, 0x00200000, 0x50100000,
961 0x98f8, 0x33773777, 0x02010001,
962 0x98fc, 0xffffffff, 0x33221100,
963 0x7030, 0x31000311, 0x00000011,
964 0x2f48, 0x33773777, 0x02010001,
965 0x6b28, 0x00000010, 0x00000012,
966 0x7728, 0x00000010, 0x00000012,
967 0x10328, 0x00000010, 0x00000012,
968 0x10f28, 0x00000010, 0x00000012,
969 0x11b28, 0x00000010, 0x00000012,
970 0x12728, 0x00000010, 0x00000012,
971 0x240c, 0x000007ff, 0x00000380,
972 0x8a14, 0xf000001f, 0x00000001,
973 0x8b24, 0x3fff3fff, 0x00ff0fff,
974 0x8b10, 0x0000ff0f, 0x00000000,
975 0x28a4c, 0x07ffffff, 0x06000000,
976 0x10c, 0x00000001, 0x00010003,
977 0xa02c, 0xffffffff, 0x0000009b,
978 0x913c, 0x0000000f, 0x0100000a,
979 0x8d00, 0xffff7f7f, 0x100e4848,
980 0x8d04, 0x00ffffff, 0x00164745,
981 0x8c00, 0xfffc0003, 0xe4000003,
982 0x8c04, 0xf8ff00ff, 0x40600060,
983 0x8c08, 0x00ff00ff, 0x001c001c,
984 0x8cf0, 0x1fff1fff, 0x08e00410,
985 0x8c20, 0x0fff0fff, 0x00800080,
986 0x8c24, 0x0fff0fff, 0x00800080,
987 0x8c18, 0xffffffff, 0x20202078,
988 0x8c1c, 0x0000ffff, 0x00001010,
989 0x28350, 0x00000f01, 0x00000000,
990 0x9508, 0x3700001f, 0x00000002,
991 0x960c, 0xffffffff, 0x54763210,
992 0x88c4, 0x001f3ae3, 0x000000c2,
993 0x88d4, 0x0000001f, 0x00000010,
994 0x8974, 0xffffffff, 0x00000000
995};
996
997static void evergreen_init_golden_registers(struct radeon_device *rdev)
998{
999 switch (rdev->family) {
1000 case CHIP_CYPRESS:
1001 case CHIP_HEMLOCK:
1002 radeon_program_register_sequence(rdev,
1003 evergreen_golden_registers,
1004 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005 radeon_program_register_sequence(rdev,
1006 evergreen_golden_registers2,
1007 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008 radeon_program_register_sequence(rdev,
1009 cypress_mgcg_init,
1010 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1011 break;
1012 case CHIP_JUNIPER:
1013 radeon_program_register_sequence(rdev,
1014 evergreen_golden_registers,
1015 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016 radeon_program_register_sequence(rdev,
1017 evergreen_golden_registers2,
1018 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019 radeon_program_register_sequence(rdev,
1020 juniper_mgcg_init,
1021 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1022 break;
1023 case CHIP_REDWOOD:
1024 radeon_program_register_sequence(rdev,
1025 evergreen_golden_registers,
1026 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027 radeon_program_register_sequence(rdev,
1028 evergreen_golden_registers2,
1029 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030 radeon_program_register_sequence(rdev,
1031 redwood_mgcg_init,
1032 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1033 break;
1034 case CHIP_CEDAR:
1035 radeon_program_register_sequence(rdev,
1036 cedar_golden_registers,
1037 (const u32)ARRAY_SIZE(cedar_golden_registers));
1038 radeon_program_register_sequence(rdev,
1039 evergreen_golden_registers2,
1040 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041 radeon_program_register_sequence(rdev,
1042 cedar_mgcg_init,
1043 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1044 break;
1045 case CHIP_PALM:
1046 radeon_program_register_sequence(rdev,
1047 wrestler_golden_registers,
1048 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1049 break;
1050 case CHIP_SUMO:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 break;
1055 case CHIP_SUMO2:
1056 radeon_program_register_sequence(rdev,
1057 supersumo_golden_registers,
1058 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059 radeon_program_register_sequence(rdev,
1060 sumo_golden_registers,
1061 (const u32)ARRAY_SIZE(sumo_golden_registers));
1062 break;
1063 case CHIP_BARTS:
1064 radeon_program_register_sequence(rdev,
1065 barts_golden_registers,
1066 (const u32)ARRAY_SIZE(barts_golden_registers));
1067 break;
1068 case CHIP_TURKS:
1069 radeon_program_register_sequence(rdev,
1070 turks_golden_registers,
1071 (const u32)ARRAY_SIZE(turks_golden_registers));
1072 break;
1073 case CHIP_CAICOS:
1074 radeon_program_register_sequence(rdev,
1075 caicos_golden_registers,
1076 (const u32)ARRAY_SIZE(caicos_golden_registers));
1077 break;
1078 default:
1079 break;
1080 }
1081}
1082
1083/**
1084 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1085 *
1086 * @rdev: radeon_device pointer
1087 * @reg: register offset in bytes
1088 * @val: register value
1089 *
1090 * Returns 0 for success or -EINVAL for an invalid register
1091 *
1092 */
1093int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1094 u32 reg, u32 *val)
1095{
1096 switch (reg) {
1097 case GRBM_STATUS:
1098 case GRBM_STATUS_SE0:
1099 case GRBM_STATUS_SE1:
1100 case SRBM_STATUS:
1101 case SRBM_STATUS2:
1102 case DMA_STATUS_REG:
1103 case UVD_STATUS:
1104 *val = RREG32(reg);
1105 return 0;
1106 default:
1107 return -EINVAL;
1108 }
1109}
1110
1111void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112 unsigned *bankh, unsigned *mtaspect,
1113 unsigned *tile_split)
1114{
1115 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1119 switch (*bankw) {
1120 default:
1121 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1125 }
1126 switch (*bankh) {
1127 default:
1128 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1132 }
1133 switch (*mtaspect) {
1134 default:
1135 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1139 }
1140}
1141
1142static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143 u32 cntl_reg, u32 status_reg)
1144{
1145 int r, i;
1146 struct atom_clock_dividers dividers;
1147
1148 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149 clock, false, ÷rs);
1150 if (r)
1151 return r;
1152
1153 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1154
1155 for (i = 0; i < 100; i++) {
1156 if (RREG32(status_reg) & DCLK_STATUS)
1157 break;
1158 mdelay(10);
1159 }
1160 if (i == 100)
1161 return -ETIMEDOUT;
1162
1163 return 0;
1164}
1165
1166int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1167{
1168 int r = 0;
1169 u32 cg_scratch = RREG32(CG_SCRATCH1);
1170
1171 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1172 if (r)
1173 goto done;
1174 cg_scratch &= 0xffff0000;
1175 cg_scratch |= vclk / 100; /* Mhz */
1176
1177 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1178 if (r)
1179 goto done;
1180 cg_scratch &= 0x0000ffff;
1181 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1182
1183done:
1184 WREG32(CG_SCRATCH1, cg_scratch);
1185
1186 return r;
1187}
1188
1189int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1190{
1191 /* start off with something large */
1192 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1193 int r;
1194
1195 /* bypass vclk and dclk with bclk */
1196 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1199
1200 /* put PLL in bypass mode */
1201 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1202
1203 if (!vclk || !dclk) {
1204 /* keep the Bypass mode, put PLL to sleep */
1205 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1206 return 0;
1207 }
1208
1209 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1210 16384, 0x03FFFFFF, 0, 128, 5,
1211 &fb_div, &vclk_div, &dclk_div);
1212 if (r)
1213 return r;
1214
1215 /* set VCO_MODE to 1 */
1216 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1217
1218 /* toggle UPLL_SLEEP to 1 then back to 0 */
1219 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1221
1222 /* deassert UPLL_RESET */
1223 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1224
1225 mdelay(1);
1226
1227 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1228 if (r)
1229 return r;
1230
1231 /* assert UPLL_RESET again */
1232 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1233
1234 /* disable spread spectrum. */
1235 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1236
1237 /* set feedback divider */
1238 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1239
1240 /* set ref divider to 0 */
1241 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1242
1243 if (fb_div < 307200)
1244 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1245 else
1246 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1247
1248 /* set PDIV_A and PDIV_B */
1249 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1252
1253 /* give the PLL some time to settle */
1254 mdelay(15);
1255
1256 /* deassert PLL_RESET */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1258
1259 mdelay(15);
1260
1261 /* switch from bypass mode to normal mode */
1262 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1263
1264 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1265 if (r)
1266 return r;
1267
1268 /* switch VCLK and DCLK selection */
1269 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1272
1273 mdelay(100);
1274
1275 return 0;
1276}
1277
1278void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1279{
1280 int readrq;
1281 u16 v;
1282
1283 readrq = pcie_get_readrq(rdev->pdev);
1284 v = ffs(readrq) - 8;
1285 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286 * to avoid hangs or perfomance issues
1287 */
1288 if ((v == 0) || (v == 6) || (v == 7))
1289 pcie_set_readrq(rdev->pdev, 512);
1290}
1291
1292void dce4_program_fmt(struct drm_encoder *encoder)
1293{
1294 struct drm_device *dev = encoder->dev;
1295 struct radeon_device *rdev = dev->dev_private;
1296 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1299 int bpc = 0;
1300 u32 tmp = 0;
1301 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1302
1303 if (connector) {
1304 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305 bpc = radeon_get_monitor_bpc(connector);
1306 dither = radeon_connector->dither;
1307 }
1308
1309 /* LVDS/eDP FMT is set up by atom */
1310 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1311 return;
1312
1313 /* not needed for analog */
1314 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1316 return;
1317
1318 if (bpc == 0)
1319 return;
1320
1321 switch (bpc) {
1322 case 6:
1323 if (dither == RADEON_FMT_DITHER_ENABLE)
1324 /* XXX sort out optimal dither settings */
1325 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326 FMT_SPATIAL_DITHER_EN);
1327 else
1328 tmp |= FMT_TRUNCATE_EN;
1329 break;
1330 case 8:
1331 if (dither == RADEON_FMT_DITHER_ENABLE)
1332 /* XXX sort out optimal dither settings */
1333 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334 FMT_RGB_RANDOM_ENABLE |
1335 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1336 else
1337 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1338 break;
1339 case 10:
1340 default:
1341 /* not needed */
1342 break;
1343 }
1344
1345 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1346}
1347
1348static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1349{
1350 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1351 return true;
1352 else
1353 return false;
1354}
1355
1356static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1357{
1358 u32 pos1, pos2;
1359
1360 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1362
1363 if (pos1 != pos2)
1364 return true;
1365 else
1366 return false;
1367}
1368
1369/**
1370 * dce4_wait_for_vblank - vblank wait asic callback.
1371 *
1372 * @rdev: radeon_device pointer
1373 * @crtc: crtc to wait for vblank on
1374 *
1375 * Wait for vblank on the requested crtc (evergreen+).
1376 */
1377void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1378{
1379 unsigned i = 0;
1380
1381 if (crtc >= rdev->num_crtc)
1382 return;
1383
1384 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1385 return;
1386
1387 /* depending on when we hit vblank, we may be close to active; if so,
1388 * wait for another frame.
1389 */
1390 while (dce4_is_in_vblank(rdev, crtc)) {
1391 if (i++ % 100 == 0) {
1392 if (!dce4_is_counter_moving(rdev, crtc))
1393 break;
1394 }
1395 }
1396
1397 while (!dce4_is_in_vblank(rdev, crtc)) {
1398 if (i++ % 100 == 0) {
1399 if (!dce4_is_counter_moving(rdev, crtc))
1400 break;
1401 }
1402 }
1403}
1404
1405/**
1406 * evergreen_page_flip - pageflip callback.
1407 *
1408 * @rdev: radeon_device pointer
1409 * @crtc_id: crtc to cleanup pageflip on
1410 * @crtc_base: new address of the crtc (GPU MC address)
1411 * @async: asynchronous flip
1412 *
1413 * Triggers the actual pageflip by updating the primary
1414 * surface base address (evergreen+).
1415 */
1416void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1417 bool async)
1418{
1419 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420 struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1421
1422 /* flip at hsync for async, default is vsync */
1423 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1425 /* update pitch */
1426 WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427 fb->pitches[0] / fb->format->cpp[0]);
1428 /* update the scanout addresses */
1429 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430 upper_32_bits(crtc_base));
1431 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432 (u32)crtc_base);
1433 /* post the write */
1434 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1435}
1436
1437/**
1438 * evergreen_page_flip_pending - check if page flip is still pending
1439 *
1440 * @rdev: radeon_device pointer
1441 * @crtc_id: crtc to check
1442 *
1443 * Returns the current update pending status.
1444 */
1445bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1446{
1447 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1448
1449 /* Return current update_pending status: */
1450 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1452}
1453
1454/* get temperature in millidegrees */
1455int evergreen_get_temp(struct radeon_device *rdev)
1456{
1457 u32 temp, toffset;
1458 int actual_temp = 0;
1459
1460 if (rdev->family == CHIP_JUNIPER) {
1461 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1462 TOFFSET_SHIFT;
1463 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1464 TS0_ADC_DOUT_SHIFT;
1465
1466 if (toffset & 0x100)
1467 actual_temp = temp / 2 - (0x200 - toffset);
1468 else
1469 actual_temp = temp / 2 + toffset;
1470
1471 actual_temp = actual_temp * 1000;
1472
1473 } else {
1474 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1475 ASIC_T_SHIFT;
1476
1477 if (temp & 0x400)
1478 actual_temp = -256;
1479 else if (temp & 0x200)
1480 actual_temp = 255;
1481 else if (temp & 0x100) {
1482 actual_temp = temp & 0x1ff;
1483 actual_temp |= ~0x1ff;
1484 } else
1485 actual_temp = temp & 0xff;
1486
1487 actual_temp = (actual_temp * 1000) / 2;
1488 }
1489
1490 return actual_temp;
1491}
1492
1493int sumo_get_temp(struct radeon_device *rdev)
1494{
1495 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496 int actual_temp = temp - 49;
1497
1498 return actual_temp * 1000;
1499}
1500
1501/**
1502 * sumo_pm_init_profile - Initialize power profiles callback.
1503 *
1504 * @rdev: radeon_device pointer
1505 *
1506 * Initialize the power states used in profile mode
1507 * (sumo, trinity, SI).
1508 * Used for profile mode only.
1509 */
1510void sumo_pm_init_profile(struct radeon_device *rdev)
1511{
1512 int idx;
1513
1514 /* default */
1515 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1519
1520 /* low,mid sh/mh */
1521 if (rdev->flags & RADEON_IS_MOBILITY)
1522 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1523 else
1524 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1525
1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1530
1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1540
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1545
1546 /* high sh/mh */
1547 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1548 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552 rdev->pm.power_state[idx].num_clock_modes - 1;
1553
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558 rdev->pm.power_state[idx].num_clock_modes - 1;
1559}
1560
1561/**
1562 * btc_pm_init_profile - Initialize power profiles callback.
1563 *
1564 * @rdev: radeon_device pointer
1565 *
1566 * Initialize the power states used in profile mode
1567 * (BTC, cayman).
1568 * Used for profile mode only.
1569 */
1570void btc_pm_init_profile(struct radeon_device *rdev)
1571{
1572 int idx;
1573
1574 /* default */
1575 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579 /* starting with BTC, there is one state that is used for both
1580 * MH and SH. Difference is that we always use the high clock index for
1581 * mclk.
1582 */
1583 if (rdev->flags & RADEON_IS_MOBILITY)
1584 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1585 else
1586 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1587 /* low sh */
1588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1592 /* mid sh */
1593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1597 /* high sh */
1598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1602 /* low mh */
1603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1607 /* mid mh */
1608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1612 /* high mh */
1613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1617}
1618
1619/**
1620 * evergreen_pm_misc - set additional pm hw parameters callback.
1621 *
1622 * @rdev: radeon_device pointer
1623 *
1624 * Set non-clock parameters associated with a power state
1625 * (voltage, etc.) (evergreen+).
1626 */
1627void evergreen_pm_misc(struct radeon_device *rdev)
1628{
1629 int req_ps_idx = rdev->pm.requested_power_state_index;
1630 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1633
1634 if (voltage->type == VOLTAGE_SW) {
1635 /* 0xff0x are flags rather then an actual voltage */
1636 if ((voltage->voltage & 0xff00) == 0xff00)
1637 return;
1638 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1639 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1640 rdev->pm.current_vddc = voltage->voltage;
1641 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1642 }
1643
1644 /* starting with BTC, there is one state that is used for both
1645 * MH and SH. Difference is that we always use the high clock index for
1646 * mclk and vddci.
1647 */
1648 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1649 (rdev->family >= CHIP_BARTS) &&
1650 rdev->pm.active_crtc_count &&
1651 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1652 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1653 voltage = &rdev->pm.power_state[req_ps_idx].
1654 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1655
1656 /* 0xff0x are flags rather then an actual voltage */
1657 if ((voltage->vddci & 0xff00) == 0xff00)
1658 return;
1659 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1660 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1661 rdev->pm.current_vddci = voltage->vddci;
1662 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1663 }
1664 }
1665}
1666
1667/**
1668 * evergreen_pm_prepare - pre-power state change callback.
1669 *
1670 * @rdev: radeon_device pointer
1671 *
1672 * Prepare for a power state change (evergreen+).
1673 */
1674void evergreen_pm_prepare(struct radeon_device *rdev)
1675{
1676 struct drm_device *ddev = rdev->ddev;
1677 struct drm_crtc *crtc;
1678 struct radeon_crtc *radeon_crtc;
1679 u32 tmp;
1680
1681 /* disable any active CRTCs */
1682 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1683 radeon_crtc = to_radeon_crtc(crtc);
1684 if (radeon_crtc->enabled) {
1685 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1686 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1687 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1688 }
1689 }
1690}
1691
1692/**
1693 * evergreen_pm_finish - post-power state change callback.
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Clean up after a power state change (evergreen+).
1698 */
1699void evergreen_pm_finish(struct radeon_device *rdev)
1700{
1701 struct drm_device *ddev = rdev->ddev;
1702 struct drm_crtc *crtc;
1703 struct radeon_crtc *radeon_crtc;
1704 u32 tmp;
1705
1706 /* enable any active CRTCs */
1707 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1708 radeon_crtc = to_radeon_crtc(crtc);
1709 if (radeon_crtc->enabled) {
1710 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1711 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1712 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1713 }
1714 }
1715}
1716
1717/**
1718 * evergreen_hpd_sense - hpd sense callback.
1719 *
1720 * @rdev: radeon_device pointer
1721 * @hpd: hpd (hotplug detect) pin
1722 *
1723 * Checks if a digital monitor is connected (evergreen+).
1724 * Returns true if connected, false if not connected.
1725 */
1726bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1727{
1728 if (hpd == RADEON_HPD_NONE)
1729 return false;
1730
1731 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1732}
1733
1734/**
1735 * evergreen_hpd_set_polarity - hpd set polarity callback.
1736 *
1737 * @rdev: radeon_device pointer
1738 * @hpd: hpd (hotplug detect) pin
1739 *
1740 * Set the polarity of the hpd pin (evergreen+).
1741 */
1742void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1743 enum radeon_hpd_id hpd)
1744{
1745 bool connected = evergreen_hpd_sense(rdev, hpd);
1746
1747 if (hpd == RADEON_HPD_NONE)
1748 return;
1749
1750 if (connected)
1751 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1752 else
1753 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1754}
1755
1756/**
1757 * evergreen_hpd_init - hpd setup callback.
1758 *
1759 * @rdev: radeon_device pointer
1760 *
1761 * Setup the hpd pins used by the card (evergreen+).
1762 * Enable the pin, set the polarity, and enable the hpd interrupts.
1763 */
1764void evergreen_hpd_init(struct radeon_device *rdev)
1765{
1766 struct drm_device *dev = rdev->ddev;
1767 struct drm_connector *connector;
1768 unsigned enabled = 0;
1769 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1770 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1771
1772 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773 enum radeon_hpd_id hpd =
1774 to_radeon_connector(connector)->hpd.hpd;
1775
1776 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1777 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1778 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1779 * aux dp channel on imac and help (but not completely fix)
1780 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1781 * also avoid interrupt storms during dpms.
1782 */
1783 continue;
1784 }
1785
1786 if (hpd == RADEON_HPD_NONE)
1787 continue;
1788
1789 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1790 enabled |= 1 << hpd;
1791
1792 radeon_hpd_set_polarity(rdev, hpd);
1793 }
1794 radeon_irq_kms_enable_hpd(rdev, enabled);
1795}
1796
1797/**
1798 * evergreen_hpd_fini - hpd tear down callback.
1799 *
1800 * @rdev: radeon_device pointer
1801 *
1802 * Tear down the hpd pins used by the card (evergreen+).
1803 * Disable the hpd interrupts.
1804 */
1805void evergreen_hpd_fini(struct radeon_device *rdev)
1806{
1807 struct drm_device *dev = rdev->ddev;
1808 struct drm_connector *connector;
1809 unsigned disabled = 0;
1810
1811 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1812 enum radeon_hpd_id hpd =
1813 to_radeon_connector(connector)->hpd.hpd;
1814
1815 if (hpd == RADEON_HPD_NONE)
1816 continue;
1817
1818 WREG32(DC_HPDx_CONTROL(hpd), 0);
1819 disabled |= 1 << hpd;
1820 }
1821 radeon_irq_kms_disable_hpd(rdev, disabled);
1822}
1823
1824/* watermark setup */
1825
1826static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1827 struct radeon_crtc *radeon_crtc,
1828 struct drm_display_mode *mode,
1829 struct drm_display_mode *other_mode)
1830{
1831 u32 tmp, buffer_alloc, i;
1832 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1833 /*
1834 * Line Buffer Setup
1835 * There are 3 line buffers, each one shared by 2 display controllers.
1836 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1837 * the display controllers. The paritioning is done via one of four
1838 * preset allocations specified in bits 2:0:
1839 * first display controller
1840 * 0 - first half of lb (3840 * 2)
1841 * 1 - first 3/4 of lb (5760 * 2)
1842 * 2 - whole lb (7680 * 2), other crtc must be disabled
1843 * 3 - first 1/4 of lb (1920 * 2)
1844 * second display controller
1845 * 4 - second half of lb (3840 * 2)
1846 * 5 - second 3/4 of lb (5760 * 2)
1847 * 6 - whole lb (7680 * 2), other crtc must be disabled
1848 * 7 - last 1/4 of lb (1920 * 2)
1849 */
1850 /* this can get tricky if we have two large displays on a paired group
1851 * of crtcs. Ideally for multiple large displays we'd assign them to
1852 * non-linked crtcs for maximum line buffer allocation.
1853 */
1854 if (radeon_crtc->base.enabled && mode) {
1855 if (other_mode) {
1856 tmp = 0; /* 1/2 */
1857 buffer_alloc = 1;
1858 } else {
1859 tmp = 2; /* whole */
1860 buffer_alloc = 2;
1861 }
1862 } else {
1863 tmp = 0;
1864 buffer_alloc = 0;
1865 }
1866
1867 /* second controller of the pair uses second half of the lb */
1868 if (radeon_crtc->crtc_id % 2)
1869 tmp += 4;
1870 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1871
1872 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1873 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1874 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1875 for (i = 0; i < rdev->usec_timeout; i++) {
1876 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1877 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1878 break;
1879 udelay(1);
1880 }
1881 }
1882
1883 if (radeon_crtc->base.enabled && mode) {
1884 switch (tmp) {
1885 case 0:
1886 case 4:
1887 default:
1888 if (ASIC_IS_DCE5(rdev))
1889 return 4096 * 2;
1890 else
1891 return 3840 * 2;
1892 case 1:
1893 case 5:
1894 if (ASIC_IS_DCE5(rdev))
1895 return 6144 * 2;
1896 else
1897 return 5760 * 2;
1898 case 2:
1899 case 6:
1900 if (ASIC_IS_DCE5(rdev))
1901 return 8192 * 2;
1902 else
1903 return 7680 * 2;
1904 case 3:
1905 case 7:
1906 if (ASIC_IS_DCE5(rdev))
1907 return 2048 * 2;
1908 else
1909 return 1920 * 2;
1910 }
1911 }
1912
1913 /* controller not enabled, so no lb used */
1914 return 0;
1915}
1916
1917u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1918{
1919 u32 tmp = RREG32(MC_SHARED_CHMAP);
1920
1921 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1922 case 0:
1923 default:
1924 return 1;
1925 case 1:
1926 return 2;
1927 case 2:
1928 return 4;
1929 case 3:
1930 return 8;
1931 }
1932}
1933
1934struct evergreen_wm_params {
1935 u32 dram_channels; /* number of dram channels */
1936 u32 yclk; /* bandwidth per dram data pin in kHz */
1937 u32 sclk; /* engine clock in kHz */
1938 u32 disp_clk; /* display clock in kHz */
1939 u32 src_width; /* viewport width */
1940 u32 active_time; /* active display time in ns */
1941 u32 blank_time; /* blank time in ns */
1942 bool interlaced; /* mode is interlaced */
1943 fixed20_12 vsc; /* vertical scale ratio */
1944 u32 num_heads; /* number of active crtcs */
1945 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1946 u32 lb_size; /* line buffer allocated to pipe */
1947 u32 vtaps; /* vertical scaler taps */
1948};
1949
1950static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1951{
1952 /* Calculate DRAM Bandwidth and the part allocated to display. */
1953 fixed20_12 dram_efficiency; /* 0.7 */
1954 fixed20_12 yclk, dram_channels, bandwidth;
1955 fixed20_12 a;
1956
1957 a.full = dfixed_const(1000);
1958 yclk.full = dfixed_const(wm->yclk);
1959 yclk.full = dfixed_div(yclk, a);
1960 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1961 a.full = dfixed_const(10);
1962 dram_efficiency.full = dfixed_const(7);
1963 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1964 bandwidth.full = dfixed_mul(dram_channels, yclk);
1965 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1966
1967 return dfixed_trunc(bandwidth);
1968}
1969
1970static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1971{
1972 /* Calculate DRAM Bandwidth and the part allocated to display. */
1973 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1974 fixed20_12 yclk, dram_channels, bandwidth;
1975 fixed20_12 a;
1976
1977 a.full = dfixed_const(1000);
1978 yclk.full = dfixed_const(wm->yclk);
1979 yclk.full = dfixed_div(yclk, a);
1980 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981 a.full = dfixed_const(10);
1982 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1983 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1984 bandwidth.full = dfixed_mul(dram_channels, yclk);
1985 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1986
1987 return dfixed_trunc(bandwidth);
1988}
1989
1990static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1991{
1992 /* Calculate the display Data return Bandwidth */
1993 fixed20_12 return_efficiency; /* 0.8 */
1994 fixed20_12 sclk, bandwidth;
1995 fixed20_12 a;
1996
1997 a.full = dfixed_const(1000);
1998 sclk.full = dfixed_const(wm->sclk);
1999 sclk.full = dfixed_div(sclk, a);
2000 a.full = dfixed_const(10);
2001 return_efficiency.full = dfixed_const(8);
2002 return_efficiency.full = dfixed_div(return_efficiency, a);
2003 a.full = dfixed_const(32);
2004 bandwidth.full = dfixed_mul(a, sclk);
2005 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2006
2007 return dfixed_trunc(bandwidth);
2008}
2009
2010static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2011{
2012 /* Calculate the DMIF Request Bandwidth */
2013 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2014 fixed20_12 disp_clk, bandwidth;
2015 fixed20_12 a;
2016
2017 a.full = dfixed_const(1000);
2018 disp_clk.full = dfixed_const(wm->disp_clk);
2019 disp_clk.full = dfixed_div(disp_clk, a);
2020 a.full = dfixed_const(10);
2021 disp_clk_request_efficiency.full = dfixed_const(8);
2022 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2023 a.full = dfixed_const(32);
2024 bandwidth.full = dfixed_mul(a, disp_clk);
2025 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2026
2027 return dfixed_trunc(bandwidth);
2028}
2029
2030static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2031{
2032 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2033 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2034 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2035 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2036
2037 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2038}
2039
2040static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2041{
2042 /* Calculate the display mode Average Bandwidth
2043 * DisplayMode should contain the source and destination dimensions,
2044 * timing, etc.
2045 */
2046 fixed20_12 bpp;
2047 fixed20_12 line_time;
2048 fixed20_12 src_width;
2049 fixed20_12 bandwidth;
2050 fixed20_12 a;
2051
2052 a.full = dfixed_const(1000);
2053 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2054 line_time.full = dfixed_div(line_time, a);
2055 bpp.full = dfixed_const(wm->bytes_per_pixel);
2056 src_width.full = dfixed_const(wm->src_width);
2057 bandwidth.full = dfixed_mul(src_width, bpp);
2058 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2059 bandwidth.full = dfixed_div(bandwidth, line_time);
2060
2061 return dfixed_trunc(bandwidth);
2062}
2063
2064static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2065{
2066 /* First calcualte the latency in ns */
2067 u32 mc_latency = 2000; /* 2000 ns. */
2068 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2069 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2070 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2071 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2072 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2073 (wm->num_heads * cursor_line_pair_return_time);
2074 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2075 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2076 fixed20_12 a, b, c;
2077
2078 if (wm->num_heads == 0)
2079 return 0;
2080
2081 a.full = dfixed_const(2);
2082 b.full = dfixed_const(1);
2083 if ((wm->vsc.full > a.full) ||
2084 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2085 (wm->vtaps >= 5) ||
2086 ((wm->vsc.full >= a.full) && wm->interlaced))
2087 max_src_lines_per_dst_line = 4;
2088 else
2089 max_src_lines_per_dst_line = 2;
2090
2091 a.full = dfixed_const(available_bandwidth);
2092 b.full = dfixed_const(wm->num_heads);
2093 a.full = dfixed_div(a, b);
2094
2095 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2096
2097 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2098 b.full = dfixed_const(1000);
2099 c.full = dfixed_const(lb_fill_bw);
2100 b.full = dfixed_div(c, b);
2101 a.full = dfixed_div(a, b);
2102 line_fill_time = dfixed_trunc(a);
2103
2104 if (line_fill_time < wm->active_time)
2105 return latency;
2106 else
2107 return latency + (line_fill_time - wm->active_time);
2108
2109}
2110
2111static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2112{
2113 if (evergreen_average_bandwidth(wm) <=
2114 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2115 return true;
2116 else
2117 return false;
2118};
2119
2120static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2121{
2122 if (evergreen_average_bandwidth(wm) <=
2123 (evergreen_available_bandwidth(wm) / wm->num_heads))
2124 return true;
2125 else
2126 return false;
2127};
2128
2129static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2130{
2131 u32 lb_partitions = wm->lb_size / wm->src_width;
2132 u32 line_time = wm->active_time + wm->blank_time;
2133 u32 latency_tolerant_lines;
2134 u32 latency_hiding;
2135 fixed20_12 a;
2136
2137 a.full = dfixed_const(1);
2138 if (wm->vsc.full > a.full)
2139 latency_tolerant_lines = 1;
2140 else {
2141 if (lb_partitions <= (wm->vtaps + 1))
2142 latency_tolerant_lines = 1;
2143 else
2144 latency_tolerant_lines = 2;
2145 }
2146
2147 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2148
2149 if (evergreen_latency_watermark(wm) <= latency_hiding)
2150 return true;
2151 else
2152 return false;
2153}
2154
2155static void evergreen_program_watermarks(struct radeon_device *rdev,
2156 struct radeon_crtc *radeon_crtc,
2157 u32 lb_size, u32 num_heads)
2158{
2159 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2160 struct evergreen_wm_params wm_low, wm_high;
2161 u32 dram_channels;
2162 u32 active_time;
2163 u32 line_time = 0;
2164 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2165 u32 priority_a_mark = 0, priority_b_mark = 0;
2166 u32 priority_a_cnt = PRIORITY_OFF;
2167 u32 priority_b_cnt = PRIORITY_OFF;
2168 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2169 u32 tmp, arb_control3;
2170 fixed20_12 a, b, c;
2171
2172 if (radeon_crtc->base.enabled && num_heads && mode) {
2173 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2174 (u32)mode->clock);
2175 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2176 (u32)mode->clock);
2177 line_time = min(line_time, (u32)65535);
2178 priority_a_cnt = 0;
2179 priority_b_cnt = 0;
2180 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2181
2182 /* watermark for high clocks */
2183 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2184 wm_high.yclk =
2185 radeon_dpm_get_mclk(rdev, false) * 10;
2186 wm_high.sclk =
2187 radeon_dpm_get_sclk(rdev, false) * 10;
2188 } else {
2189 wm_high.yclk = rdev->pm.current_mclk * 10;
2190 wm_high.sclk = rdev->pm.current_sclk * 10;
2191 }
2192
2193 wm_high.disp_clk = mode->clock;
2194 wm_high.src_width = mode->crtc_hdisplay;
2195 wm_high.active_time = active_time;
2196 wm_high.blank_time = line_time - wm_high.active_time;
2197 wm_high.interlaced = false;
2198 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2199 wm_high.interlaced = true;
2200 wm_high.vsc = radeon_crtc->vsc;
2201 wm_high.vtaps = 1;
2202 if (radeon_crtc->rmx_type != RMX_OFF)
2203 wm_high.vtaps = 2;
2204 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2205 wm_high.lb_size = lb_size;
2206 wm_high.dram_channels = dram_channels;
2207 wm_high.num_heads = num_heads;
2208
2209 /* watermark for low clocks */
2210 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2211 wm_low.yclk =
2212 radeon_dpm_get_mclk(rdev, true) * 10;
2213 wm_low.sclk =
2214 radeon_dpm_get_sclk(rdev, true) * 10;
2215 } else {
2216 wm_low.yclk = rdev->pm.current_mclk * 10;
2217 wm_low.sclk = rdev->pm.current_sclk * 10;
2218 }
2219
2220 wm_low.disp_clk = mode->clock;
2221 wm_low.src_width = mode->crtc_hdisplay;
2222 wm_low.active_time = active_time;
2223 wm_low.blank_time = line_time - wm_low.active_time;
2224 wm_low.interlaced = false;
2225 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2226 wm_low.interlaced = true;
2227 wm_low.vsc = radeon_crtc->vsc;
2228 wm_low.vtaps = 1;
2229 if (radeon_crtc->rmx_type != RMX_OFF)
2230 wm_low.vtaps = 2;
2231 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2232 wm_low.lb_size = lb_size;
2233 wm_low.dram_channels = dram_channels;
2234 wm_low.num_heads = num_heads;
2235
2236 /* set for high clocks */
2237 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2238 /* set for low clocks */
2239 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2240
2241 /* possibly force display priority to high */
2242 /* should really do this at mode validation time... */
2243 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2244 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2245 !evergreen_check_latency_hiding(&wm_high) ||
2246 (rdev->disp_priority == 2)) {
2247 DRM_DEBUG_KMS("force priority a to high\n");
2248 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2249 }
2250 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2251 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2252 !evergreen_check_latency_hiding(&wm_low) ||
2253 (rdev->disp_priority == 2)) {
2254 DRM_DEBUG_KMS("force priority b to high\n");
2255 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2256 }
2257
2258 a.full = dfixed_const(1000);
2259 b.full = dfixed_const(mode->clock);
2260 b.full = dfixed_div(b, a);
2261 c.full = dfixed_const(latency_watermark_a);
2262 c.full = dfixed_mul(c, b);
2263 c.full = dfixed_mul(c, radeon_crtc->hsc);
2264 c.full = dfixed_div(c, a);
2265 a.full = dfixed_const(16);
2266 c.full = dfixed_div(c, a);
2267 priority_a_mark = dfixed_trunc(c);
2268 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2269
2270 a.full = dfixed_const(1000);
2271 b.full = dfixed_const(mode->clock);
2272 b.full = dfixed_div(b, a);
2273 c.full = dfixed_const(latency_watermark_b);
2274 c.full = dfixed_mul(c, b);
2275 c.full = dfixed_mul(c, radeon_crtc->hsc);
2276 c.full = dfixed_div(c, a);
2277 a.full = dfixed_const(16);
2278 c.full = dfixed_div(c, a);
2279 priority_b_mark = dfixed_trunc(c);
2280 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2281
2282 /* Save number of lines the linebuffer leads before the scanout */
2283 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2284 }
2285
2286 /* select wm A */
2287 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2288 tmp = arb_control3;
2289 tmp &= ~LATENCY_WATERMARK_MASK(3);
2290 tmp |= LATENCY_WATERMARK_MASK(1);
2291 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2294 LATENCY_HIGH_WATERMARK(line_time)));
2295 /* select wm B */
2296 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2297 tmp &= ~LATENCY_WATERMARK_MASK(3);
2298 tmp |= LATENCY_WATERMARK_MASK(2);
2299 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2300 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2301 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2302 LATENCY_HIGH_WATERMARK(line_time)));
2303 /* restore original selection */
2304 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2305
2306 /* write the priority marks */
2307 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2308 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2309
2310 /* save values for DPM */
2311 radeon_crtc->line_time = line_time;
2312 radeon_crtc->wm_high = latency_watermark_a;
2313 radeon_crtc->wm_low = latency_watermark_b;
2314}
2315
2316/**
2317 * evergreen_bandwidth_update - update display watermarks callback.
2318 *
2319 * @rdev: radeon_device pointer
2320 *
2321 * Update the display watermarks based on the requested mode(s)
2322 * (evergreen+).
2323 */
2324void evergreen_bandwidth_update(struct radeon_device *rdev)
2325{
2326 struct drm_display_mode *mode0 = NULL;
2327 struct drm_display_mode *mode1 = NULL;
2328 u32 num_heads = 0, lb_size;
2329 int i;
2330
2331 if (!rdev->mode_info.mode_config_initialized)
2332 return;
2333
2334 radeon_update_display_priority(rdev);
2335
2336 for (i = 0; i < rdev->num_crtc; i++) {
2337 if (rdev->mode_info.crtcs[i]->base.enabled)
2338 num_heads++;
2339 }
2340 for (i = 0; i < rdev->num_crtc; i += 2) {
2341 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2342 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2343 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2344 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2345 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2346 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2347 }
2348}
2349
2350/**
2351 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2352 *
2353 * @rdev: radeon_device pointer
2354 *
2355 * Wait for the MC (memory controller) to be idle.
2356 * (evergreen+).
2357 * Returns 0 if the MC is idle, -1 if not.
2358 */
2359int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2360{
2361 unsigned i;
2362 u32 tmp;
2363
2364 for (i = 0; i < rdev->usec_timeout; i++) {
2365 /* read MC_STATUS */
2366 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2367 if (!tmp)
2368 return 0;
2369 udelay(1);
2370 }
2371 return -1;
2372}
2373
2374/*
2375 * GART
2376 */
2377void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2378{
2379 unsigned i;
2380 u32 tmp;
2381
2382 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2383
2384 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2385 for (i = 0; i < rdev->usec_timeout; i++) {
2386 /* read MC_STATUS */
2387 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2388 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2389 if (tmp == 2) {
2390 pr_warn("[drm] r600 flush TLB failed\n");
2391 return;
2392 }
2393 if (tmp) {
2394 return;
2395 }
2396 udelay(1);
2397 }
2398}
2399
2400static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2401{
2402 u32 tmp;
2403 int r;
2404
2405 if (rdev->gart.robj == NULL) {
2406 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2407 return -EINVAL;
2408 }
2409 r = radeon_gart_table_vram_pin(rdev);
2410 if (r)
2411 return r;
2412 /* Setup L2 cache */
2413 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2414 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2415 EFFECTIVE_L2_QUEUE_SIZE(7));
2416 WREG32(VM_L2_CNTL2, 0);
2417 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2418 /* Setup TLB control */
2419 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2420 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2421 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2422 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2423 if (rdev->flags & RADEON_IS_IGP) {
2424 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2425 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2426 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2427 } else {
2428 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2429 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2430 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2431 if ((rdev->family == CHIP_JUNIPER) ||
2432 (rdev->family == CHIP_CYPRESS) ||
2433 (rdev->family == CHIP_HEMLOCK) ||
2434 (rdev->family == CHIP_BARTS))
2435 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2436 }
2437 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2439 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2440 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2441 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2442 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2443 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2444 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2445 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2446 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2447 (u32)(rdev->dummy_page.addr >> 12));
2448 WREG32(VM_CONTEXT1_CNTL, 0);
2449
2450 evergreen_pcie_gart_tlb_flush(rdev);
2451 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2452 (unsigned)(rdev->mc.gtt_size >> 20),
2453 (unsigned long long)rdev->gart.table_addr);
2454 rdev->gart.ready = true;
2455 return 0;
2456}
2457
2458static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2459{
2460 u32 tmp;
2461
2462 /* Disable all tables */
2463 WREG32(VM_CONTEXT0_CNTL, 0);
2464 WREG32(VM_CONTEXT1_CNTL, 0);
2465
2466 /* Setup L2 cache */
2467 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2468 EFFECTIVE_L2_QUEUE_SIZE(7));
2469 WREG32(VM_L2_CNTL2, 0);
2470 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2471 /* Setup TLB control */
2472 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2473 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2474 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2475 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2476 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2477 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2478 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2479 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2480 radeon_gart_table_vram_unpin(rdev);
2481}
2482
2483static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2484{
2485 evergreen_pcie_gart_disable(rdev);
2486 radeon_gart_table_vram_free(rdev);
2487 radeon_gart_fini(rdev);
2488}
2489
2490
2491static void evergreen_agp_enable(struct radeon_device *rdev)
2492{
2493 u32 tmp;
2494
2495 /* Setup L2 cache */
2496 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2497 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2498 EFFECTIVE_L2_QUEUE_SIZE(7));
2499 WREG32(VM_L2_CNTL2, 0);
2500 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2501 /* Setup TLB control */
2502 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2503 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2504 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2505 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2506 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2507 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2508 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2509 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2510 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2511 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2512 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2513 WREG32(VM_CONTEXT0_CNTL, 0);
2514 WREG32(VM_CONTEXT1_CNTL, 0);
2515}
2516
2517static const unsigned ni_dig_offsets[] =
2518{
2519 NI_DIG0_REGISTER_OFFSET,
2520 NI_DIG1_REGISTER_OFFSET,
2521 NI_DIG2_REGISTER_OFFSET,
2522 NI_DIG3_REGISTER_OFFSET,
2523 NI_DIG4_REGISTER_OFFSET,
2524 NI_DIG5_REGISTER_OFFSET
2525};
2526
2527static const unsigned ni_tx_offsets[] =
2528{
2529 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2530 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2531 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2532 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2533 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2534 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2535};
2536
2537static const unsigned evergreen_dp_offsets[] =
2538{
2539 EVERGREEN_DP0_REGISTER_OFFSET,
2540 EVERGREEN_DP1_REGISTER_OFFSET,
2541 EVERGREEN_DP2_REGISTER_OFFSET,
2542 EVERGREEN_DP3_REGISTER_OFFSET,
2543 EVERGREEN_DP4_REGISTER_OFFSET,
2544 EVERGREEN_DP5_REGISTER_OFFSET
2545};
2546
2547static const unsigned evergreen_disp_int_status[] =
2548{
2549 DISP_INTERRUPT_STATUS,
2550 DISP_INTERRUPT_STATUS_CONTINUE,
2551 DISP_INTERRUPT_STATUS_CONTINUE2,
2552 DISP_INTERRUPT_STATUS_CONTINUE3,
2553 DISP_INTERRUPT_STATUS_CONTINUE4,
2554 DISP_INTERRUPT_STATUS_CONTINUE5
2555};
2556
2557/*
2558 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2559 * We go from crtc to connector and it is not relible since it
2560 * should be an opposite direction .If crtc is enable then
2561 * find the dig_fe which selects this crtc and insure that it enable.
2562 * if such dig_fe is found then find dig_be which selects found dig_be and
2563 * insure that it enable and in DP_SST mode.
2564 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2565 * from dp symbols clocks .
2566 */
2567static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2568 unsigned crtc_id, unsigned *ret_dig_fe)
2569{
2570 unsigned i;
2571 unsigned dig_fe;
2572 unsigned dig_be;
2573 unsigned dig_en_be;
2574 unsigned uniphy_pll;
2575 unsigned digs_fe_selected;
2576 unsigned dig_be_mode;
2577 unsigned dig_fe_mask;
2578 bool is_enabled = false;
2579 bool found_crtc = false;
2580
2581 /* loop through all running dig_fe to find selected crtc */
2582 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2583 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2584 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2585 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2586 /* found running pipe */
2587 found_crtc = true;
2588 dig_fe_mask = 1 << i;
2589 dig_fe = i;
2590 break;
2591 }
2592 }
2593
2594 if (found_crtc) {
2595 /* loop through all running dig_be to find selected dig_fe */
2596 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2597 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2598 /* if dig_fe_selected by dig_be? */
2599 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2600 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2601 if (dig_fe_mask & digs_fe_selected &&
2602 /* if dig_be in sst mode? */
2603 dig_be_mode == NI_DIG_BE_DPSST) {
2604 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2605 ni_dig_offsets[i]);
2606 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2607 ni_tx_offsets[i]);
2608 /* dig_be enable and tx is running */
2609 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2610 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2611 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2612 is_enabled = true;
2613 *ret_dig_fe = dig_fe;
2614 break;
2615 }
2616 }
2617 }
2618 }
2619
2620 return is_enabled;
2621}
2622
2623/*
2624 * Blank dig when in dp sst mode
2625 * Dig ignores crtc timing
2626 */
2627static void evergreen_blank_dp_output(struct radeon_device *rdev,
2628 unsigned dig_fe)
2629{
2630 unsigned stream_ctrl;
2631 unsigned fifo_ctrl;
2632 unsigned counter = 0;
2633
2634 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2635 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2636 return;
2637 }
2638
2639 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2640 evergreen_dp_offsets[dig_fe]);
2641 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2642 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2643 return;
2644 }
2645
2646 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2647 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648 evergreen_dp_offsets[dig_fe], stream_ctrl);
2649
2650 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2651 evergreen_dp_offsets[dig_fe]);
2652 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2653 msleep(1);
2654 counter++;
2655 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2656 evergreen_dp_offsets[dig_fe]);
2657 }
2658 if (counter >= 32 )
2659 DRM_ERROR("counter exceeds %d\n", counter);
2660
2661 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2662 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2663 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2664
2665}
2666
2667void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2668{
2669 u32 crtc_enabled, tmp, frame_count, blackout;
2670 int i, j;
2671 unsigned dig_fe;
2672
2673 if (!ASIC_IS_NODCE(rdev)) {
2674 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2675 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2676
2677 /* disable VGA render */
2678 WREG32(VGA_RENDER_CONTROL, 0);
2679 }
2680 /* blank the display controllers */
2681 for (i = 0; i < rdev->num_crtc; i++) {
2682 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2683 if (crtc_enabled) {
2684 save->crtc_enabled[i] = true;
2685 if (ASIC_IS_DCE6(rdev)) {
2686 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2687 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2688 radeon_wait_for_vblank(rdev, i);
2689 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2690 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2691 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2692 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2693 }
2694 } else {
2695 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2696 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2697 radeon_wait_for_vblank(rdev, i);
2698 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2699 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2700 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2701 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2702 }
2703 }
2704 /* wait for the next frame */
2705 frame_count = radeon_get_vblank_counter(rdev, i);
2706 for (j = 0; j < rdev->usec_timeout; j++) {
2707 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2708 break;
2709 udelay(1);
2710 }
2711 /*we should disable dig if it drives dp sst*/
2712 /*but we are in radeon_device_init and the topology is unknown*/
2713 /*and it is available after radeon_modeset_init*/
2714 /*the following method radeon_atom_encoder_dpms_dig*/
2715 /*does the job if we initialize it properly*/
2716 /*for now we do it this manually*/
2717 /**/
2718 if (ASIC_IS_DCE5(rdev) &&
2719 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2720 evergreen_blank_dp_output(rdev, dig_fe);
2721 /*we could remove 6 lines below*/
2722 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2723 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2724 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2725 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2726 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2727 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2728 save->crtc_enabled[i] = false;
2729 /* ***** */
2730 } else {
2731 save->crtc_enabled[i] = false;
2732 }
2733 }
2734
2735 radeon_mc_wait_for_idle(rdev);
2736
2737 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2738 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2739 /* Block CPU access */
2740 WREG32(BIF_FB_EN, 0);
2741 /* blackout the MC */
2742 blackout &= ~BLACKOUT_MODE_MASK;
2743 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2744 }
2745 /* wait for the MC to settle */
2746 udelay(100);
2747
2748 /* lock double buffered regs */
2749 for (i = 0; i < rdev->num_crtc; i++) {
2750 if (save->crtc_enabled[i]) {
2751 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2752 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2753 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2754 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2755 }
2756 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2757 if (!(tmp & 1)) {
2758 tmp |= 1;
2759 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2760 }
2761 }
2762 }
2763}
2764
2765void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2766{
2767 u32 tmp, frame_count;
2768 int i, j;
2769
2770 /* update crtc base addresses */
2771 for (i = 0; i < rdev->num_crtc; i++) {
2772 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2773 upper_32_bits(rdev->mc.vram_start));
2774 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2775 upper_32_bits(rdev->mc.vram_start));
2776 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2777 (u32)rdev->mc.vram_start);
2778 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2779 (u32)rdev->mc.vram_start);
2780 }
2781
2782 if (!ASIC_IS_NODCE(rdev)) {
2783 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2784 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2785 }
2786
2787 /* unlock regs and wait for update */
2788 for (i = 0; i < rdev->num_crtc; i++) {
2789 if (save->crtc_enabled[i]) {
2790 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2791 if ((tmp & 0x7) != 0) {
2792 tmp &= ~0x7;
2793 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2794 }
2795 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2796 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2797 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2798 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2799 }
2800 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2801 if (tmp & 1) {
2802 tmp &= ~1;
2803 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2804 }
2805 for (j = 0; j < rdev->usec_timeout; j++) {
2806 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2807 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2808 break;
2809 udelay(1);
2810 }
2811 }
2812 }
2813
2814 /* unblackout the MC */
2815 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2816 tmp &= ~BLACKOUT_MODE_MASK;
2817 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2818 /* allow CPU access */
2819 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2820
2821 for (i = 0; i < rdev->num_crtc; i++) {
2822 if (save->crtc_enabled[i]) {
2823 if (ASIC_IS_DCE6(rdev)) {
2824 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2825 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2826 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2827 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2828 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2829 } else {
2830 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2831 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2832 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2833 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2834 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2835 }
2836 /* wait for the next frame */
2837 frame_count = radeon_get_vblank_counter(rdev, i);
2838 for (j = 0; j < rdev->usec_timeout; j++) {
2839 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2840 break;
2841 udelay(1);
2842 }
2843 }
2844 }
2845 if (!ASIC_IS_NODCE(rdev)) {
2846 /* Unlock vga access */
2847 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2848 mdelay(1);
2849 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2850 }
2851}
2852
2853void evergreen_mc_program(struct radeon_device *rdev)
2854{
2855 struct evergreen_mc_save save;
2856 u32 tmp;
2857 int i, j;
2858
2859 /* Initialize HDP */
2860 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2861 WREG32((0x2c14 + j), 0x00000000);
2862 WREG32((0x2c18 + j), 0x00000000);
2863 WREG32((0x2c1c + j), 0x00000000);
2864 WREG32((0x2c20 + j), 0x00000000);
2865 WREG32((0x2c24 + j), 0x00000000);
2866 }
2867 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2868
2869 evergreen_mc_stop(rdev, &save);
2870 if (evergreen_mc_wait_for_idle(rdev)) {
2871 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2872 }
2873 /* Lockout access through VGA aperture*/
2874 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2875 /* Update configuration */
2876 if (rdev->flags & RADEON_IS_AGP) {
2877 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2878 /* VRAM before AGP */
2879 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2880 rdev->mc.vram_start >> 12);
2881 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2882 rdev->mc.gtt_end >> 12);
2883 } else {
2884 /* VRAM after AGP */
2885 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2886 rdev->mc.gtt_start >> 12);
2887 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2888 rdev->mc.vram_end >> 12);
2889 }
2890 } else {
2891 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2892 rdev->mc.vram_start >> 12);
2893 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2894 rdev->mc.vram_end >> 12);
2895 }
2896 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2897 /* llano/ontario only */
2898 if ((rdev->family == CHIP_PALM) ||
2899 (rdev->family == CHIP_SUMO) ||
2900 (rdev->family == CHIP_SUMO2)) {
2901 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2902 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2903 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2904 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2905 }
2906 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2907 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2908 WREG32(MC_VM_FB_LOCATION, tmp);
2909 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2910 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2911 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2912 if (rdev->flags & RADEON_IS_AGP) {
2913 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2914 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2915 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2916 } else {
2917 WREG32(MC_VM_AGP_BASE, 0);
2918 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2919 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2920 }
2921 if (evergreen_mc_wait_for_idle(rdev)) {
2922 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2923 }
2924 evergreen_mc_resume(rdev, &save);
2925 /* we need to own VRAM, so turn off the VGA renderer here
2926 * to stop it overwriting our objects */
2927 rv515_vga_render_disable(rdev);
2928}
2929
2930/*
2931 * CP.
2932 */
2933void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2934{
2935 struct radeon_ring *ring = &rdev->ring[ib->ring];
2936 u32 next_rptr;
2937
2938 /* set to DX10/11 mode */
2939 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2940 radeon_ring_write(ring, 1);
2941
2942 if (ring->rptr_save_reg) {
2943 next_rptr = ring->wptr + 3 + 4;
2944 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2945 radeon_ring_write(ring, ((ring->rptr_save_reg -
2946 PACKET3_SET_CONFIG_REG_START) >> 2));
2947 radeon_ring_write(ring, next_rptr);
2948 } else if (rdev->wb.enabled) {
2949 next_rptr = ring->wptr + 5 + 4;
2950 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2951 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2952 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2953 radeon_ring_write(ring, next_rptr);
2954 radeon_ring_write(ring, 0);
2955 }
2956
2957 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2958 radeon_ring_write(ring,
2959#ifdef __BIG_ENDIAN
2960 (2 << 0) |
2961#endif
2962 (ib->gpu_addr & 0xFFFFFFFC));
2963 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2964 radeon_ring_write(ring, ib->length_dw);
2965}
2966
2967
2968static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2969{
2970 const __be32 *fw_data;
2971 int i;
2972
2973 if (!rdev->me_fw || !rdev->pfp_fw)
2974 return -EINVAL;
2975
2976 r700_cp_stop(rdev);
2977 WREG32(CP_RB_CNTL,
2978#ifdef __BIG_ENDIAN
2979 BUF_SWAP_32BIT |
2980#endif
2981 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2982
2983 fw_data = (const __be32 *)rdev->pfp_fw->data;
2984 WREG32(CP_PFP_UCODE_ADDR, 0);
2985 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2986 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2987 WREG32(CP_PFP_UCODE_ADDR, 0);
2988
2989 fw_data = (const __be32 *)rdev->me_fw->data;
2990 WREG32(CP_ME_RAM_WADDR, 0);
2991 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2992 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2993
2994 WREG32(CP_PFP_UCODE_ADDR, 0);
2995 WREG32(CP_ME_RAM_WADDR, 0);
2996 WREG32(CP_ME_RAM_RADDR, 0);
2997 return 0;
2998}
2999
3000static int evergreen_cp_start(struct radeon_device *rdev)
3001{
3002 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3003 int r, i;
3004 uint32_t cp_me;
3005
3006 r = radeon_ring_lock(rdev, ring, 7);
3007 if (r) {
3008 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3009 return r;
3010 }
3011 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3012 radeon_ring_write(ring, 0x1);
3013 radeon_ring_write(ring, 0x0);
3014 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3015 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3016 radeon_ring_write(ring, 0);
3017 radeon_ring_write(ring, 0);
3018 radeon_ring_unlock_commit(rdev, ring, false);
3019
3020 cp_me = 0xff;
3021 WREG32(CP_ME_CNTL, cp_me);
3022
3023 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3024 if (r) {
3025 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3026 return r;
3027 }
3028
3029 /* setup clear context state */
3030 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3031 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3032
3033 for (i = 0; i < evergreen_default_size; i++)
3034 radeon_ring_write(ring, evergreen_default_state[i]);
3035
3036 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3037 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3038
3039 /* set clear context state */
3040 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3041 radeon_ring_write(ring, 0);
3042
3043 /* SQ_VTX_BASE_VTX_LOC */
3044 radeon_ring_write(ring, 0xc0026f00);
3045 radeon_ring_write(ring, 0x00000000);
3046 radeon_ring_write(ring, 0x00000000);
3047 radeon_ring_write(ring, 0x00000000);
3048
3049 /* Clear consts */
3050 radeon_ring_write(ring, 0xc0036f00);
3051 radeon_ring_write(ring, 0x00000bc4);
3052 radeon_ring_write(ring, 0xffffffff);
3053 radeon_ring_write(ring, 0xffffffff);
3054 radeon_ring_write(ring, 0xffffffff);
3055
3056 radeon_ring_write(ring, 0xc0026900);
3057 radeon_ring_write(ring, 0x00000316);
3058 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3059 radeon_ring_write(ring, 0x00000010); /* */
3060
3061 radeon_ring_unlock_commit(rdev, ring, false);
3062
3063 return 0;
3064}
3065
3066static int evergreen_cp_resume(struct radeon_device *rdev)
3067{
3068 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3069 u32 tmp;
3070 u32 rb_bufsz;
3071 int r;
3072
3073 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3074 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3075 SOFT_RESET_PA |
3076 SOFT_RESET_SH |
3077 SOFT_RESET_VGT |
3078 SOFT_RESET_SPI |
3079 SOFT_RESET_SX));
3080 RREG32(GRBM_SOFT_RESET);
3081 mdelay(15);
3082 WREG32(GRBM_SOFT_RESET, 0);
3083 RREG32(GRBM_SOFT_RESET);
3084
3085 /* Set ring buffer size */
3086 rb_bufsz = order_base_2(ring->ring_size / 8);
3087 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3088#ifdef __BIG_ENDIAN
3089 tmp |= BUF_SWAP_32BIT;
3090#endif
3091 WREG32(CP_RB_CNTL, tmp);
3092 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3093 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3094
3095 /* Set the write pointer delay */
3096 WREG32(CP_RB_WPTR_DELAY, 0);
3097
3098 /* Initialize the ring buffer's read and write pointers */
3099 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3100 WREG32(CP_RB_RPTR_WR, 0);
3101 ring->wptr = 0;
3102 WREG32(CP_RB_WPTR, ring->wptr);
3103
3104 /* set the wb address whether it's enabled or not */
3105 WREG32(CP_RB_RPTR_ADDR,
3106 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3107 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3108 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3109
3110 if (rdev->wb.enabled)
3111 WREG32(SCRATCH_UMSK, 0xff);
3112 else {
3113 tmp |= RB_NO_UPDATE;
3114 WREG32(SCRATCH_UMSK, 0);
3115 }
3116
3117 mdelay(1);
3118 WREG32(CP_RB_CNTL, tmp);
3119
3120 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3121 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3122
3123 evergreen_cp_start(rdev);
3124 ring->ready = true;
3125 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3126 if (r) {
3127 ring->ready = false;
3128 return r;
3129 }
3130 return 0;
3131}
3132
3133/*
3134 * Core functions
3135 */
3136static void evergreen_gpu_init(struct radeon_device *rdev)
3137{
3138 u32 gb_addr_config;
3139 u32 mc_arb_ramcfg;
3140 u32 sx_debug_1;
3141 u32 smx_dc_ctl0;
3142 u32 sq_config;
3143 u32 sq_lds_resource_mgmt;
3144 u32 sq_gpr_resource_mgmt_1;
3145 u32 sq_gpr_resource_mgmt_2;
3146 u32 sq_gpr_resource_mgmt_3;
3147 u32 sq_thread_resource_mgmt;
3148 u32 sq_thread_resource_mgmt_2;
3149 u32 sq_stack_resource_mgmt_1;
3150 u32 sq_stack_resource_mgmt_2;
3151 u32 sq_stack_resource_mgmt_3;
3152 u32 vgt_cache_invalidation;
3153 u32 hdp_host_path_cntl, tmp;
3154 u32 disabled_rb_mask;
3155 int i, j, ps_thread_count;
3156
3157 switch (rdev->family) {
3158 case CHIP_CYPRESS:
3159 case CHIP_HEMLOCK:
3160 rdev->config.evergreen.num_ses = 2;
3161 rdev->config.evergreen.max_pipes = 4;
3162 rdev->config.evergreen.max_tile_pipes = 8;
3163 rdev->config.evergreen.max_simds = 10;
3164 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3165 rdev->config.evergreen.max_gprs = 256;
3166 rdev->config.evergreen.max_threads = 248;
3167 rdev->config.evergreen.max_gs_threads = 32;
3168 rdev->config.evergreen.max_stack_entries = 512;
3169 rdev->config.evergreen.sx_num_of_sets = 4;
3170 rdev->config.evergreen.sx_max_export_size = 256;
3171 rdev->config.evergreen.sx_max_export_pos_size = 64;
3172 rdev->config.evergreen.sx_max_export_smx_size = 192;
3173 rdev->config.evergreen.max_hw_contexts = 8;
3174 rdev->config.evergreen.sq_num_cf_insts = 2;
3175
3176 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3177 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3178 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3179 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3180 break;
3181 case CHIP_JUNIPER:
3182 rdev->config.evergreen.num_ses = 1;
3183 rdev->config.evergreen.max_pipes = 4;
3184 rdev->config.evergreen.max_tile_pipes = 4;
3185 rdev->config.evergreen.max_simds = 10;
3186 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3187 rdev->config.evergreen.max_gprs = 256;
3188 rdev->config.evergreen.max_threads = 248;
3189 rdev->config.evergreen.max_gs_threads = 32;
3190 rdev->config.evergreen.max_stack_entries = 512;
3191 rdev->config.evergreen.sx_num_of_sets = 4;
3192 rdev->config.evergreen.sx_max_export_size = 256;
3193 rdev->config.evergreen.sx_max_export_pos_size = 64;
3194 rdev->config.evergreen.sx_max_export_smx_size = 192;
3195 rdev->config.evergreen.max_hw_contexts = 8;
3196 rdev->config.evergreen.sq_num_cf_insts = 2;
3197
3198 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3199 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3200 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3201 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3202 break;
3203 case CHIP_REDWOOD:
3204 rdev->config.evergreen.num_ses = 1;
3205 rdev->config.evergreen.max_pipes = 4;
3206 rdev->config.evergreen.max_tile_pipes = 4;
3207 rdev->config.evergreen.max_simds = 5;
3208 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3209 rdev->config.evergreen.max_gprs = 256;
3210 rdev->config.evergreen.max_threads = 248;
3211 rdev->config.evergreen.max_gs_threads = 32;
3212 rdev->config.evergreen.max_stack_entries = 256;
3213 rdev->config.evergreen.sx_num_of_sets = 4;
3214 rdev->config.evergreen.sx_max_export_size = 256;
3215 rdev->config.evergreen.sx_max_export_pos_size = 64;
3216 rdev->config.evergreen.sx_max_export_smx_size = 192;
3217 rdev->config.evergreen.max_hw_contexts = 8;
3218 rdev->config.evergreen.sq_num_cf_insts = 2;
3219
3220 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3221 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3222 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3223 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3224 break;
3225 case CHIP_CEDAR:
3226 default:
3227 rdev->config.evergreen.num_ses = 1;
3228 rdev->config.evergreen.max_pipes = 2;
3229 rdev->config.evergreen.max_tile_pipes = 2;
3230 rdev->config.evergreen.max_simds = 2;
3231 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3232 rdev->config.evergreen.max_gprs = 256;
3233 rdev->config.evergreen.max_threads = 192;
3234 rdev->config.evergreen.max_gs_threads = 16;
3235 rdev->config.evergreen.max_stack_entries = 256;
3236 rdev->config.evergreen.sx_num_of_sets = 4;
3237 rdev->config.evergreen.sx_max_export_size = 128;
3238 rdev->config.evergreen.sx_max_export_pos_size = 32;
3239 rdev->config.evergreen.sx_max_export_smx_size = 96;
3240 rdev->config.evergreen.max_hw_contexts = 4;
3241 rdev->config.evergreen.sq_num_cf_insts = 1;
3242
3243 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3244 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3245 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3246 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3247 break;
3248 case CHIP_PALM:
3249 rdev->config.evergreen.num_ses = 1;
3250 rdev->config.evergreen.max_pipes = 2;
3251 rdev->config.evergreen.max_tile_pipes = 2;
3252 rdev->config.evergreen.max_simds = 2;
3253 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3254 rdev->config.evergreen.max_gprs = 256;
3255 rdev->config.evergreen.max_threads = 192;
3256 rdev->config.evergreen.max_gs_threads = 16;
3257 rdev->config.evergreen.max_stack_entries = 256;
3258 rdev->config.evergreen.sx_num_of_sets = 4;
3259 rdev->config.evergreen.sx_max_export_size = 128;
3260 rdev->config.evergreen.sx_max_export_pos_size = 32;
3261 rdev->config.evergreen.sx_max_export_smx_size = 96;
3262 rdev->config.evergreen.max_hw_contexts = 4;
3263 rdev->config.evergreen.sq_num_cf_insts = 1;
3264
3265 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3266 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3267 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3268 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3269 break;
3270 case CHIP_SUMO:
3271 rdev->config.evergreen.num_ses = 1;
3272 rdev->config.evergreen.max_pipes = 4;
3273 rdev->config.evergreen.max_tile_pipes = 4;
3274 if (rdev->pdev->device == 0x9648)
3275 rdev->config.evergreen.max_simds = 3;
3276 else if ((rdev->pdev->device == 0x9647) ||
3277 (rdev->pdev->device == 0x964a))
3278 rdev->config.evergreen.max_simds = 4;
3279 else
3280 rdev->config.evergreen.max_simds = 5;
3281 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3282 rdev->config.evergreen.max_gprs = 256;
3283 rdev->config.evergreen.max_threads = 248;
3284 rdev->config.evergreen.max_gs_threads = 32;
3285 rdev->config.evergreen.max_stack_entries = 256;
3286 rdev->config.evergreen.sx_num_of_sets = 4;
3287 rdev->config.evergreen.sx_max_export_size = 256;
3288 rdev->config.evergreen.sx_max_export_pos_size = 64;
3289 rdev->config.evergreen.sx_max_export_smx_size = 192;
3290 rdev->config.evergreen.max_hw_contexts = 8;
3291 rdev->config.evergreen.sq_num_cf_insts = 2;
3292
3293 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3294 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3295 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3296 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3297 break;
3298 case CHIP_SUMO2:
3299 rdev->config.evergreen.num_ses = 1;
3300 rdev->config.evergreen.max_pipes = 4;
3301 rdev->config.evergreen.max_tile_pipes = 4;
3302 rdev->config.evergreen.max_simds = 2;
3303 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3304 rdev->config.evergreen.max_gprs = 256;
3305 rdev->config.evergreen.max_threads = 248;
3306 rdev->config.evergreen.max_gs_threads = 32;
3307 rdev->config.evergreen.max_stack_entries = 512;
3308 rdev->config.evergreen.sx_num_of_sets = 4;
3309 rdev->config.evergreen.sx_max_export_size = 256;
3310 rdev->config.evergreen.sx_max_export_pos_size = 64;
3311 rdev->config.evergreen.sx_max_export_smx_size = 192;
3312 rdev->config.evergreen.max_hw_contexts = 4;
3313 rdev->config.evergreen.sq_num_cf_insts = 2;
3314
3315 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3316 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3317 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3318 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3319 break;
3320 case CHIP_BARTS:
3321 rdev->config.evergreen.num_ses = 2;
3322 rdev->config.evergreen.max_pipes = 4;
3323 rdev->config.evergreen.max_tile_pipes = 8;
3324 rdev->config.evergreen.max_simds = 7;
3325 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3326 rdev->config.evergreen.max_gprs = 256;
3327 rdev->config.evergreen.max_threads = 248;
3328 rdev->config.evergreen.max_gs_threads = 32;
3329 rdev->config.evergreen.max_stack_entries = 512;
3330 rdev->config.evergreen.sx_num_of_sets = 4;
3331 rdev->config.evergreen.sx_max_export_size = 256;
3332 rdev->config.evergreen.sx_max_export_pos_size = 64;
3333 rdev->config.evergreen.sx_max_export_smx_size = 192;
3334 rdev->config.evergreen.max_hw_contexts = 8;
3335 rdev->config.evergreen.sq_num_cf_insts = 2;
3336
3337 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3338 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3339 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3340 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3341 break;
3342 case CHIP_TURKS:
3343 rdev->config.evergreen.num_ses = 1;
3344 rdev->config.evergreen.max_pipes = 4;
3345 rdev->config.evergreen.max_tile_pipes = 4;
3346 rdev->config.evergreen.max_simds = 6;
3347 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3348 rdev->config.evergreen.max_gprs = 256;
3349 rdev->config.evergreen.max_threads = 248;
3350 rdev->config.evergreen.max_gs_threads = 32;
3351 rdev->config.evergreen.max_stack_entries = 256;
3352 rdev->config.evergreen.sx_num_of_sets = 4;
3353 rdev->config.evergreen.sx_max_export_size = 256;
3354 rdev->config.evergreen.sx_max_export_pos_size = 64;
3355 rdev->config.evergreen.sx_max_export_smx_size = 192;
3356 rdev->config.evergreen.max_hw_contexts = 8;
3357 rdev->config.evergreen.sq_num_cf_insts = 2;
3358
3359 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3360 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3361 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3362 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3363 break;
3364 case CHIP_CAICOS:
3365 rdev->config.evergreen.num_ses = 1;
3366 rdev->config.evergreen.max_pipes = 2;
3367 rdev->config.evergreen.max_tile_pipes = 2;
3368 rdev->config.evergreen.max_simds = 2;
3369 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3370 rdev->config.evergreen.max_gprs = 256;
3371 rdev->config.evergreen.max_threads = 192;
3372 rdev->config.evergreen.max_gs_threads = 16;
3373 rdev->config.evergreen.max_stack_entries = 256;
3374 rdev->config.evergreen.sx_num_of_sets = 4;
3375 rdev->config.evergreen.sx_max_export_size = 128;
3376 rdev->config.evergreen.sx_max_export_pos_size = 32;
3377 rdev->config.evergreen.sx_max_export_smx_size = 96;
3378 rdev->config.evergreen.max_hw_contexts = 4;
3379 rdev->config.evergreen.sq_num_cf_insts = 1;
3380
3381 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3382 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3383 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3384 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3385 break;
3386 }
3387
3388 /* Initialize HDP */
3389 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3390 WREG32((0x2c14 + j), 0x00000000);
3391 WREG32((0x2c18 + j), 0x00000000);
3392 WREG32((0x2c1c + j), 0x00000000);
3393 WREG32((0x2c20 + j), 0x00000000);
3394 WREG32((0x2c24 + j), 0x00000000);
3395 }
3396
3397 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3398 WREG32(SRBM_INT_CNTL, 0x1);
3399 WREG32(SRBM_INT_ACK, 0x1);
3400
3401 evergreen_fix_pci_max_read_req_size(rdev);
3402
3403 RREG32(MC_SHARED_CHMAP);
3404 if ((rdev->family == CHIP_PALM) ||
3405 (rdev->family == CHIP_SUMO) ||
3406 (rdev->family == CHIP_SUMO2))
3407 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3408 else
3409 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3410
3411 /* setup tiling info dword. gb_addr_config is not adequate since it does
3412 * not have bank info, so create a custom tiling dword.
3413 * bits 3:0 num_pipes
3414 * bits 7:4 num_banks
3415 * bits 11:8 group_size
3416 * bits 15:12 row_size
3417 */
3418 rdev->config.evergreen.tile_config = 0;
3419 switch (rdev->config.evergreen.max_tile_pipes) {
3420 case 1:
3421 default:
3422 rdev->config.evergreen.tile_config |= (0 << 0);
3423 break;
3424 case 2:
3425 rdev->config.evergreen.tile_config |= (1 << 0);
3426 break;
3427 case 4:
3428 rdev->config.evergreen.tile_config |= (2 << 0);
3429 break;
3430 case 8:
3431 rdev->config.evergreen.tile_config |= (3 << 0);
3432 break;
3433 }
3434 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3435 if (rdev->flags & RADEON_IS_IGP)
3436 rdev->config.evergreen.tile_config |= 1 << 4;
3437 else {
3438 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3439 case 0: /* four banks */
3440 rdev->config.evergreen.tile_config |= 0 << 4;
3441 break;
3442 case 1: /* eight banks */
3443 rdev->config.evergreen.tile_config |= 1 << 4;
3444 break;
3445 case 2: /* sixteen banks */
3446 default:
3447 rdev->config.evergreen.tile_config |= 2 << 4;
3448 break;
3449 }
3450 }
3451 rdev->config.evergreen.tile_config |= 0 << 8;
3452 rdev->config.evergreen.tile_config |=
3453 ((gb_addr_config & 0x30000000) >> 28) << 12;
3454
3455 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3456 u32 efuse_straps_4;
3457 u32 efuse_straps_3;
3458
3459 efuse_straps_4 = RREG32_RCU(0x204);
3460 efuse_straps_3 = RREG32_RCU(0x203);
3461 tmp = (((efuse_straps_4 & 0xf) << 4) |
3462 ((efuse_straps_3 & 0xf0000000) >> 28));
3463 } else {
3464 tmp = 0;
3465 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3466 u32 rb_disable_bitmap;
3467
3468 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3469 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3470 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3471 tmp <<= 4;
3472 tmp |= rb_disable_bitmap;
3473 }
3474 }
3475 /* enabled rb are just the one not disabled :) */
3476 disabled_rb_mask = tmp;
3477 tmp = 0;
3478 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3479 tmp |= (1 << i);
3480 /* if all the backends are disabled, fix it up here */
3481 if ((disabled_rb_mask & tmp) == tmp) {
3482 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3483 disabled_rb_mask &= ~(1 << i);
3484 }
3485
3486 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3487 u32 simd_disable_bitmap;
3488
3489 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3490 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3491 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3492 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3493 tmp <<= 16;
3494 tmp |= simd_disable_bitmap;
3495 }
3496 rdev->config.evergreen.active_simds = hweight32(~tmp);
3497
3498 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3499 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3500
3501 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3502 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3503 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3504 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3505 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3506 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3507 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3508
3509 if ((rdev->config.evergreen.max_backends == 1) &&
3510 (rdev->flags & RADEON_IS_IGP)) {
3511 if ((disabled_rb_mask & 3) == 1) {
3512 /* RB0 disabled, RB1 enabled */
3513 tmp = 0x11111111;
3514 } else {
3515 /* RB1 disabled, RB0 enabled */
3516 tmp = 0x00000000;
3517 }
3518 } else {
3519 tmp = gb_addr_config & NUM_PIPES_MASK;
3520 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3521 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3522 }
3523 rdev->config.evergreen.backend_map = tmp;
3524 WREG32(GB_BACKEND_MAP, tmp);
3525
3526 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3527 WREG32(CGTS_TCC_DISABLE, 0);
3528 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3529 WREG32(CGTS_USER_TCC_DISABLE, 0);
3530
3531 /* set HW defaults for 3D engine */
3532 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3533 ROQ_IB2_START(0x2b)));
3534
3535 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3536
3537 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3538 SYNC_GRADIENT |
3539 SYNC_WALKER |
3540 SYNC_ALIGNER));
3541
3542 sx_debug_1 = RREG32(SX_DEBUG_1);
3543 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3544 WREG32(SX_DEBUG_1, sx_debug_1);
3545
3546
3547 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3548 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3549 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3550 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3551
3552 if (rdev->family <= CHIP_SUMO2)
3553 WREG32(SMX_SAR_CTL0, 0x00010000);
3554
3555 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3556 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3557 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3558
3559 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3560 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3561 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3562
3563 WREG32(VGT_NUM_INSTANCES, 1);
3564 WREG32(SPI_CONFIG_CNTL, 0);
3565 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3566 WREG32(CP_PERFMON_CNTL, 0);
3567
3568 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3569 FETCH_FIFO_HIWATER(0x4) |
3570 DONE_FIFO_HIWATER(0xe0) |
3571 ALU_UPDATE_FIFO_HIWATER(0x8)));
3572
3573 sq_config = RREG32(SQ_CONFIG);
3574 sq_config &= ~(PS_PRIO(3) |
3575 VS_PRIO(3) |
3576 GS_PRIO(3) |
3577 ES_PRIO(3));
3578 sq_config |= (VC_ENABLE |
3579 EXPORT_SRC_C |
3580 PS_PRIO(0) |
3581 VS_PRIO(1) |
3582 GS_PRIO(2) |
3583 ES_PRIO(3));
3584
3585 switch (rdev->family) {
3586 case CHIP_CEDAR:
3587 case CHIP_PALM:
3588 case CHIP_SUMO:
3589 case CHIP_SUMO2:
3590 case CHIP_CAICOS:
3591 /* no vertex cache */
3592 sq_config &= ~VC_ENABLE;
3593 break;
3594 default:
3595 break;
3596 }
3597
3598 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3599
3600 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3601 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3602 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3603 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3604 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3605 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3606 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3607
3608 switch (rdev->family) {
3609 case CHIP_CEDAR:
3610 case CHIP_PALM:
3611 case CHIP_SUMO:
3612 case CHIP_SUMO2:
3613 ps_thread_count = 96;
3614 break;
3615 default:
3616 ps_thread_count = 128;
3617 break;
3618 }
3619
3620 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3621 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3622 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3624 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3625 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3626
3627 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3631 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3632 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3633
3634 WREG32(SQ_CONFIG, sq_config);
3635 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3636 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3637 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3638 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3639 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3640 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3641 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3642 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3643 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3644 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3645
3646 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3647 FORCE_EOV_MAX_REZ_CNT(255)));
3648
3649 switch (rdev->family) {
3650 case CHIP_CEDAR:
3651 case CHIP_PALM:
3652 case CHIP_SUMO:
3653 case CHIP_SUMO2:
3654 case CHIP_CAICOS:
3655 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3656 break;
3657 default:
3658 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3659 break;
3660 }
3661 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3662 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3663
3664 WREG32(VGT_GS_VERTEX_REUSE, 16);
3665 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3666 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3667
3668 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3669 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3670
3671 WREG32(CB_PERF_CTR0_SEL_0, 0);
3672 WREG32(CB_PERF_CTR0_SEL_1, 0);
3673 WREG32(CB_PERF_CTR1_SEL_0, 0);
3674 WREG32(CB_PERF_CTR1_SEL_1, 0);
3675 WREG32(CB_PERF_CTR2_SEL_0, 0);
3676 WREG32(CB_PERF_CTR2_SEL_1, 0);
3677 WREG32(CB_PERF_CTR3_SEL_0, 0);
3678 WREG32(CB_PERF_CTR3_SEL_1, 0);
3679
3680 /* clear render buffer base addresses */
3681 WREG32(CB_COLOR0_BASE, 0);
3682 WREG32(CB_COLOR1_BASE, 0);
3683 WREG32(CB_COLOR2_BASE, 0);
3684 WREG32(CB_COLOR3_BASE, 0);
3685 WREG32(CB_COLOR4_BASE, 0);
3686 WREG32(CB_COLOR5_BASE, 0);
3687 WREG32(CB_COLOR6_BASE, 0);
3688 WREG32(CB_COLOR7_BASE, 0);
3689 WREG32(CB_COLOR8_BASE, 0);
3690 WREG32(CB_COLOR9_BASE, 0);
3691 WREG32(CB_COLOR10_BASE, 0);
3692 WREG32(CB_COLOR11_BASE, 0);
3693
3694 /* set the shader const cache sizes to 0 */
3695 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3696 WREG32(i, 0);
3697 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3698 WREG32(i, 0);
3699
3700 tmp = RREG32(HDP_MISC_CNTL);
3701 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3702 WREG32(HDP_MISC_CNTL, tmp);
3703
3704 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3705 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3706
3707 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3708
3709 udelay(50);
3710
3711}
3712
3713int evergreen_mc_init(struct radeon_device *rdev)
3714{
3715 u32 tmp;
3716 int chansize, numchan;
3717
3718 /* Get VRAM informations */
3719 rdev->mc.vram_is_ddr = true;
3720 if ((rdev->family == CHIP_PALM) ||
3721 (rdev->family == CHIP_SUMO) ||
3722 (rdev->family == CHIP_SUMO2))
3723 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3724 else
3725 tmp = RREG32(MC_ARB_RAMCFG);
3726 if (tmp & CHANSIZE_OVERRIDE) {
3727 chansize = 16;
3728 } else if (tmp & CHANSIZE_MASK) {
3729 chansize = 64;
3730 } else {
3731 chansize = 32;
3732 }
3733 tmp = RREG32(MC_SHARED_CHMAP);
3734 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3735 case 0:
3736 default:
3737 numchan = 1;
3738 break;
3739 case 1:
3740 numchan = 2;
3741 break;
3742 case 2:
3743 numchan = 4;
3744 break;
3745 case 3:
3746 numchan = 8;
3747 break;
3748 }
3749 rdev->mc.vram_width = numchan * chansize;
3750 /* Could aper size report 0 ? */
3751 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3752 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3753 /* Setup GPU memory space */
3754 if ((rdev->family == CHIP_PALM) ||
3755 (rdev->family == CHIP_SUMO) ||
3756 (rdev->family == CHIP_SUMO2)) {
3757 /* size in bytes on fusion */
3758 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3759 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3760 } else {
3761 /* size in MB on evergreen/cayman/tn */
3762 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3763 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3764 }
3765 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3766 r700_vram_gtt_location(rdev, &rdev->mc);
3767 radeon_update_bandwidth_info(rdev);
3768
3769 return 0;
3770}
3771
3772void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3773{
3774 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3775 RREG32(GRBM_STATUS));
3776 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3777 RREG32(GRBM_STATUS_SE0));
3778 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3779 RREG32(GRBM_STATUS_SE1));
3780 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3781 RREG32(SRBM_STATUS));
3782 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3783 RREG32(SRBM_STATUS2));
3784 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3785 RREG32(CP_STALLED_STAT1));
3786 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3787 RREG32(CP_STALLED_STAT2));
3788 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3789 RREG32(CP_BUSY_STAT));
3790 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3791 RREG32(CP_STAT));
3792 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3793 RREG32(DMA_STATUS_REG));
3794 if (rdev->family >= CHIP_CAYMAN) {
3795 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3796 RREG32(DMA_STATUS_REG + 0x800));
3797 }
3798}
3799
3800bool evergreen_is_display_hung(struct radeon_device *rdev)
3801{
3802 u32 crtc_hung = 0;
3803 u32 crtc_status[6];
3804 u32 i, j, tmp;
3805
3806 for (i = 0; i < rdev->num_crtc; i++) {
3807 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3808 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3809 crtc_hung |= (1 << i);
3810 }
3811 }
3812
3813 for (j = 0; j < 10; j++) {
3814 for (i = 0; i < rdev->num_crtc; i++) {
3815 if (crtc_hung & (1 << i)) {
3816 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3817 if (tmp != crtc_status[i])
3818 crtc_hung &= ~(1 << i);
3819 }
3820 }
3821 if (crtc_hung == 0)
3822 return false;
3823 udelay(100);
3824 }
3825
3826 return true;
3827}
3828
3829u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3830{
3831 u32 reset_mask = 0;
3832 u32 tmp;
3833
3834 /* GRBM_STATUS */
3835 tmp = RREG32(GRBM_STATUS);
3836 if (tmp & (PA_BUSY | SC_BUSY |
3837 SH_BUSY | SX_BUSY |
3838 TA_BUSY | VGT_BUSY |
3839 DB_BUSY | CB_BUSY |
3840 SPI_BUSY | VGT_BUSY_NO_DMA))
3841 reset_mask |= RADEON_RESET_GFX;
3842
3843 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3844 CP_BUSY | CP_COHERENCY_BUSY))
3845 reset_mask |= RADEON_RESET_CP;
3846
3847 if (tmp & GRBM_EE_BUSY)
3848 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3849
3850 /* DMA_STATUS_REG */
3851 tmp = RREG32(DMA_STATUS_REG);
3852 if (!(tmp & DMA_IDLE))
3853 reset_mask |= RADEON_RESET_DMA;
3854
3855 /* SRBM_STATUS2 */
3856 tmp = RREG32(SRBM_STATUS2);
3857 if (tmp & DMA_BUSY)
3858 reset_mask |= RADEON_RESET_DMA;
3859
3860 /* SRBM_STATUS */
3861 tmp = RREG32(SRBM_STATUS);
3862 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3863 reset_mask |= RADEON_RESET_RLC;
3864
3865 if (tmp & IH_BUSY)
3866 reset_mask |= RADEON_RESET_IH;
3867
3868 if (tmp & SEM_BUSY)
3869 reset_mask |= RADEON_RESET_SEM;
3870
3871 if (tmp & GRBM_RQ_PENDING)
3872 reset_mask |= RADEON_RESET_GRBM;
3873
3874 if (tmp & VMC_BUSY)
3875 reset_mask |= RADEON_RESET_VMC;
3876
3877 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3878 MCC_BUSY | MCD_BUSY))
3879 reset_mask |= RADEON_RESET_MC;
3880
3881 if (evergreen_is_display_hung(rdev))
3882 reset_mask |= RADEON_RESET_DISPLAY;
3883
3884 /* VM_L2_STATUS */
3885 tmp = RREG32(VM_L2_STATUS);
3886 if (tmp & L2_BUSY)
3887 reset_mask |= RADEON_RESET_VMC;
3888
3889 /* Skip MC reset as it's mostly likely not hung, just busy */
3890 if (reset_mask & RADEON_RESET_MC) {
3891 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3892 reset_mask &= ~RADEON_RESET_MC;
3893 }
3894
3895 return reset_mask;
3896}
3897
3898static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3899{
3900 struct evergreen_mc_save save;
3901 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3902 u32 tmp;
3903
3904 if (reset_mask == 0)
3905 return;
3906
3907 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3908
3909 evergreen_print_gpu_status_regs(rdev);
3910
3911 /* Disable CP parsing/prefetching */
3912 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3913
3914 if (reset_mask & RADEON_RESET_DMA) {
3915 /* Disable DMA */
3916 tmp = RREG32(DMA_RB_CNTL);
3917 tmp &= ~DMA_RB_ENABLE;
3918 WREG32(DMA_RB_CNTL, tmp);
3919 }
3920
3921 udelay(50);
3922
3923 evergreen_mc_stop(rdev, &save);
3924 if (evergreen_mc_wait_for_idle(rdev)) {
3925 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3926 }
3927
3928 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3929 grbm_soft_reset |= SOFT_RESET_DB |
3930 SOFT_RESET_CB |
3931 SOFT_RESET_PA |
3932 SOFT_RESET_SC |
3933 SOFT_RESET_SPI |
3934 SOFT_RESET_SX |
3935 SOFT_RESET_SH |
3936 SOFT_RESET_TC |
3937 SOFT_RESET_TA |
3938 SOFT_RESET_VC |
3939 SOFT_RESET_VGT;
3940 }
3941
3942 if (reset_mask & RADEON_RESET_CP) {
3943 grbm_soft_reset |= SOFT_RESET_CP |
3944 SOFT_RESET_VGT;
3945
3946 srbm_soft_reset |= SOFT_RESET_GRBM;
3947 }
3948
3949 if (reset_mask & RADEON_RESET_DMA)
3950 srbm_soft_reset |= SOFT_RESET_DMA;
3951
3952 if (reset_mask & RADEON_RESET_DISPLAY)
3953 srbm_soft_reset |= SOFT_RESET_DC;
3954
3955 if (reset_mask & RADEON_RESET_RLC)
3956 srbm_soft_reset |= SOFT_RESET_RLC;
3957
3958 if (reset_mask & RADEON_RESET_SEM)
3959 srbm_soft_reset |= SOFT_RESET_SEM;
3960
3961 if (reset_mask & RADEON_RESET_IH)
3962 srbm_soft_reset |= SOFT_RESET_IH;
3963
3964 if (reset_mask & RADEON_RESET_GRBM)
3965 srbm_soft_reset |= SOFT_RESET_GRBM;
3966
3967 if (reset_mask & RADEON_RESET_VMC)
3968 srbm_soft_reset |= SOFT_RESET_VMC;
3969
3970 if (!(rdev->flags & RADEON_IS_IGP)) {
3971 if (reset_mask & RADEON_RESET_MC)
3972 srbm_soft_reset |= SOFT_RESET_MC;
3973 }
3974
3975 if (grbm_soft_reset) {
3976 tmp = RREG32(GRBM_SOFT_RESET);
3977 tmp |= grbm_soft_reset;
3978 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3979 WREG32(GRBM_SOFT_RESET, tmp);
3980 tmp = RREG32(GRBM_SOFT_RESET);
3981
3982 udelay(50);
3983
3984 tmp &= ~grbm_soft_reset;
3985 WREG32(GRBM_SOFT_RESET, tmp);
3986 tmp = RREG32(GRBM_SOFT_RESET);
3987 }
3988
3989 if (srbm_soft_reset) {
3990 tmp = RREG32(SRBM_SOFT_RESET);
3991 tmp |= srbm_soft_reset;
3992 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3993 WREG32(SRBM_SOFT_RESET, tmp);
3994 tmp = RREG32(SRBM_SOFT_RESET);
3995
3996 udelay(50);
3997
3998 tmp &= ~srbm_soft_reset;
3999 WREG32(SRBM_SOFT_RESET, tmp);
4000 tmp = RREG32(SRBM_SOFT_RESET);
4001 }
4002
4003 /* Wait a little for things to settle down */
4004 udelay(50);
4005
4006 evergreen_mc_resume(rdev, &save);
4007 udelay(50);
4008
4009 evergreen_print_gpu_status_regs(rdev);
4010}
4011
4012void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4013{
4014 struct evergreen_mc_save save;
4015 u32 tmp, i;
4016
4017 dev_info(rdev->dev, "GPU pci config reset\n");
4018
4019 /* disable dpm? */
4020
4021 /* Disable CP parsing/prefetching */
4022 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4023 udelay(50);
4024 /* Disable DMA */
4025 tmp = RREG32(DMA_RB_CNTL);
4026 tmp &= ~DMA_RB_ENABLE;
4027 WREG32(DMA_RB_CNTL, tmp);
4028 /* XXX other engines? */
4029
4030 /* halt the rlc */
4031 r600_rlc_stop(rdev);
4032
4033 udelay(50);
4034
4035 /* set mclk/sclk to bypass */
4036 rv770_set_clk_bypass_mode(rdev);
4037 /* disable BM */
4038 pci_clear_master(rdev->pdev);
4039 /* disable mem access */
4040 evergreen_mc_stop(rdev, &save);
4041 if (evergreen_mc_wait_for_idle(rdev)) {
4042 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4043 }
4044 /* reset */
4045 radeon_pci_config_reset(rdev);
4046 /* wait for asic to come out of reset */
4047 for (i = 0; i < rdev->usec_timeout; i++) {
4048 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4049 break;
4050 udelay(1);
4051 }
4052}
4053
4054int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4055{
4056 u32 reset_mask;
4057
4058 if (hard) {
4059 evergreen_gpu_pci_config_reset(rdev);
4060 return 0;
4061 }
4062
4063 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4064
4065 if (reset_mask)
4066 r600_set_bios_scratch_engine_hung(rdev, true);
4067
4068 /* try soft reset */
4069 evergreen_gpu_soft_reset(rdev, reset_mask);
4070
4071 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4072
4073 /* try pci config reset */
4074 if (reset_mask && radeon_hard_reset)
4075 evergreen_gpu_pci_config_reset(rdev);
4076
4077 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4078
4079 if (!reset_mask)
4080 r600_set_bios_scratch_engine_hung(rdev, false);
4081
4082 return 0;
4083}
4084
4085/**
4086 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4087 *
4088 * @rdev: radeon_device pointer
4089 * @ring: radeon_ring structure holding ring information
4090 *
4091 * Check if the GFX engine is locked up.
4092 * Returns true if the engine appears to be locked up, false if not.
4093 */
4094bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4095{
4096 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4097
4098 if (!(reset_mask & (RADEON_RESET_GFX |
4099 RADEON_RESET_COMPUTE |
4100 RADEON_RESET_CP))) {
4101 radeon_ring_lockup_update(rdev, ring);
4102 return false;
4103 }
4104 return radeon_ring_test_lockup(rdev, ring);
4105}
4106
4107/*
4108 * RLC
4109 */
4110#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4111#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4112
4113void sumo_rlc_fini(struct radeon_device *rdev)
4114{
4115 int r;
4116
4117 /* save restore block */
4118 if (rdev->rlc.save_restore_obj) {
4119 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4120 if (unlikely(r != 0))
4121 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4122 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4123 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4124
4125 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4126 rdev->rlc.save_restore_obj = NULL;
4127 }
4128
4129 /* clear state block */
4130 if (rdev->rlc.clear_state_obj) {
4131 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4132 if (unlikely(r != 0))
4133 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4134 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4135 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4136
4137 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4138 rdev->rlc.clear_state_obj = NULL;
4139 }
4140
4141 /* clear state block */
4142 if (rdev->rlc.cp_table_obj) {
4143 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4144 if (unlikely(r != 0))
4145 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4146 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4147 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4148
4149 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4150 rdev->rlc.cp_table_obj = NULL;
4151 }
4152}
4153
4154#define CP_ME_TABLE_SIZE 96
4155
4156int sumo_rlc_init(struct radeon_device *rdev)
4157{
4158 const u32 *src_ptr;
4159 volatile u32 *dst_ptr;
4160 u32 dws, data, i, j, k, reg_num;
4161 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4162 u64 reg_list_mc_addr;
4163 const struct cs_section_def *cs_data;
4164 int r;
4165
4166 src_ptr = rdev->rlc.reg_list;
4167 dws = rdev->rlc.reg_list_size;
4168 if (rdev->family >= CHIP_BONAIRE) {
4169 dws += (5 * 16) + 48 + 48 + 64;
4170 }
4171 cs_data = rdev->rlc.cs_data;
4172
4173 if (src_ptr) {
4174 /* save restore block */
4175 if (rdev->rlc.save_restore_obj == NULL) {
4176 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4177 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4178 NULL, &rdev->rlc.save_restore_obj);
4179 if (r) {
4180 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4181 return r;
4182 }
4183 }
4184
4185 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4186 if (unlikely(r != 0)) {
4187 sumo_rlc_fini(rdev);
4188 return r;
4189 }
4190 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4191 &rdev->rlc.save_restore_gpu_addr);
4192 if (r) {
4193 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4194 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4195 sumo_rlc_fini(rdev);
4196 return r;
4197 }
4198
4199 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4200 if (r) {
4201 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4202 sumo_rlc_fini(rdev);
4203 return r;
4204 }
4205 /* write the sr buffer */
4206 dst_ptr = rdev->rlc.sr_ptr;
4207 if (rdev->family >= CHIP_TAHITI) {
4208 /* SI */
4209 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4210 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4211 } else {
4212 /* ON/LN/TN */
4213 /* format:
4214 * dw0: (reg2 << 16) | reg1
4215 * dw1: reg1 save space
4216 * dw2: reg2 save space
4217 */
4218 for (i = 0; i < dws; i++) {
4219 data = src_ptr[i] >> 2;
4220 i++;
4221 if (i < dws)
4222 data |= (src_ptr[i] >> 2) << 16;
4223 j = (((i - 1) * 3) / 2);
4224 dst_ptr[j] = cpu_to_le32(data);
4225 }
4226 j = ((i * 3) / 2);
4227 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4228 }
4229 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4230 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4231 }
4232
4233 if (cs_data) {
4234 /* clear state block */
4235 if (rdev->family >= CHIP_BONAIRE) {
4236 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4237 } else if (rdev->family >= CHIP_TAHITI) {
4238 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4239 dws = rdev->rlc.clear_state_size + (256 / 4);
4240 } else {
4241 reg_list_num = 0;
4242 dws = 0;
4243 for (i = 0; cs_data[i].section != NULL; i++) {
4244 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4245 reg_list_num++;
4246 dws += cs_data[i].section[j].reg_count;
4247 }
4248 }
4249 reg_list_blk_index = (3 * reg_list_num + 2);
4250 dws += reg_list_blk_index;
4251 rdev->rlc.clear_state_size = dws;
4252 }
4253
4254 if (rdev->rlc.clear_state_obj == NULL) {
4255 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4256 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4257 NULL, &rdev->rlc.clear_state_obj);
4258 if (r) {
4259 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4260 sumo_rlc_fini(rdev);
4261 return r;
4262 }
4263 }
4264 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4265 if (unlikely(r != 0)) {
4266 sumo_rlc_fini(rdev);
4267 return r;
4268 }
4269 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4270 &rdev->rlc.clear_state_gpu_addr);
4271 if (r) {
4272 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4273 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4274 sumo_rlc_fini(rdev);
4275 return r;
4276 }
4277
4278 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4279 if (r) {
4280 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4281 sumo_rlc_fini(rdev);
4282 return r;
4283 }
4284 /* set up the cs buffer */
4285 dst_ptr = rdev->rlc.cs_ptr;
4286 if (rdev->family >= CHIP_BONAIRE) {
4287 cik_get_csb_buffer(rdev, dst_ptr);
4288 } else if (rdev->family >= CHIP_TAHITI) {
4289 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4290 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4291 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4292 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4293 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4294 } else {
4295 reg_list_hdr_blk_index = 0;
4296 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4297 data = upper_32_bits(reg_list_mc_addr);
4298 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4299 reg_list_hdr_blk_index++;
4300 for (i = 0; cs_data[i].section != NULL; i++) {
4301 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4302 reg_num = cs_data[i].section[j].reg_count;
4303 data = reg_list_mc_addr & 0xffffffff;
4304 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305 reg_list_hdr_blk_index++;
4306
4307 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4308 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309 reg_list_hdr_blk_index++;
4310
4311 data = 0x08000000 | (reg_num * 4);
4312 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4313 reg_list_hdr_blk_index++;
4314
4315 for (k = 0; k < reg_num; k++) {
4316 data = cs_data[i].section[j].extent[k];
4317 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4318 }
4319 reg_list_mc_addr += reg_num * 4;
4320 reg_list_blk_index += reg_num;
4321 }
4322 }
4323 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4324 }
4325 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4326 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4327 }
4328
4329 if (rdev->rlc.cp_table_size) {
4330 if (rdev->rlc.cp_table_obj == NULL) {
4331 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4332 PAGE_SIZE, true,
4333 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4334 NULL, &rdev->rlc.cp_table_obj);
4335 if (r) {
4336 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4337 sumo_rlc_fini(rdev);
4338 return r;
4339 }
4340 }
4341
4342 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4343 if (unlikely(r != 0)) {
4344 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4345 sumo_rlc_fini(rdev);
4346 return r;
4347 }
4348 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4349 &rdev->rlc.cp_table_gpu_addr);
4350 if (r) {
4351 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4352 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4353 sumo_rlc_fini(rdev);
4354 return r;
4355 }
4356 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4357 if (r) {
4358 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4359 sumo_rlc_fini(rdev);
4360 return r;
4361 }
4362
4363 cik_init_cp_pg_table(rdev);
4364
4365 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4366 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4367
4368 }
4369
4370 return 0;
4371}
4372
4373static void evergreen_rlc_start(struct radeon_device *rdev)
4374{
4375 u32 mask = RLC_ENABLE;
4376
4377 if (rdev->flags & RADEON_IS_IGP) {
4378 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4379 }
4380
4381 WREG32(RLC_CNTL, mask);
4382}
4383
4384int evergreen_rlc_resume(struct radeon_device *rdev)
4385{
4386 u32 i;
4387 const __be32 *fw_data;
4388
4389 if (!rdev->rlc_fw)
4390 return -EINVAL;
4391
4392 r600_rlc_stop(rdev);
4393
4394 WREG32(RLC_HB_CNTL, 0);
4395
4396 if (rdev->flags & RADEON_IS_IGP) {
4397 if (rdev->family == CHIP_ARUBA) {
4398 u32 always_on_bitmap =
4399 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4400 /* find out the number of active simds */
4401 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4402 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4403 tmp = hweight32(~tmp);
4404 if (tmp == rdev->config.cayman.max_simds_per_se) {
4405 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4406 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4407 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4408 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4409 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4410 }
4411 } else {
4412 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4413 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4414 }
4415 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4416 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4417 } else {
4418 WREG32(RLC_HB_BASE, 0);
4419 WREG32(RLC_HB_RPTR, 0);
4420 WREG32(RLC_HB_WPTR, 0);
4421 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4422 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4423 }
4424 WREG32(RLC_MC_CNTL, 0);
4425 WREG32(RLC_UCODE_CNTL, 0);
4426
4427 fw_data = (const __be32 *)rdev->rlc_fw->data;
4428 if (rdev->family >= CHIP_ARUBA) {
4429 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4430 WREG32(RLC_UCODE_ADDR, i);
4431 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4432 }
4433 } else if (rdev->family >= CHIP_CAYMAN) {
4434 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4435 WREG32(RLC_UCODE_ADDR, i);
4436 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4437 }
4438 } else {
4439 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4440 WREG32(RLC_UCODE_ADDR, i);
4441 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4442 }
4443 }
4444 WREG32(RLC_UCODE_ADDR, 0);
4445
4446 evergreen_rlc_start(rdev);
4447
4448 return 0;
4449}
4450
4451/* Interrupts */
4452
4453u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4454{
4455 if (crtc >= rdev->num_crtc)
4456 return 0;
4457 else
4458 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4459}
4460
4461void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4462{
4463 int i;
4464 u32 tmp;
4465
4466 if (rdev->family >= CHIP_CAYMAN) {
4467 cayman_cp_int_cntl_setup(rdev, 0,
4468 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4469 cayman_cp_int_cntl_setup(rdev, 1, 0);
4470 cayman_cp_int_cntl_setup(rdev, 2, 0);
4471 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4472 WREG32(CAYMAN_DMA1_CNTL, tmp);
4473 } else
4474 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4475 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4476 WREG32(DMA_CNTL, tmp);
4477 WREG32(GRBM_INT_CNTL, 0);
4478 WREG32(SRBM_INT_CNTL, 0);
4479 for (i = 0; i < rdev->num_crtc; i++)
4480 WREG32(INT_MASK + crtc_offsets[i], 0);
4481 for (i = 0; i < rdev->num_crtc; i++)
4482 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4483
4484 /* only one DAC on DCE5 */
4485 if (!ASIC_IS_DCE5(rdev))
4486 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4487 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4488
4489 for (i = 0; i < 6; i++)
4490 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4491}
4492
4493/* Note that the order we write back regs here is important */
4494int evergreen_irq_set(struct radeon_device *rdev)
4495{
4496 int i;
4497 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4498 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4499 u32 grbm_int_cntl = 0;
4500 u32 dma_cntl, dma_cntl1 = 0;
4501 u32 thermal_int = 0;
4502
4503 if (!rdev->irq.installed) {
4504 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4505 return -EINVAL;
4506 }
4507 /* don't enable anything if the ih is disabled */
4508 if (!rdev->ih.enabled) {
4509 r600_disable_interrupts(rdev);
4510 /* force the active interrupt state to all disabled */
4511 evergreen_disable_interrupt_state(rdev);
4512 return 0;
4513 }
4514
4515 if (rdev->family == CHIP_ARUBA)
4516 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4517 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518 else
4519 thermal_int = RREG32(CG_THERMAL_INT) &
4520 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4521
4522 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4523
4524 if (rdev->family >= CHIP_CAYMAN) {
4525 /* enable CP interrupts on all rings */
4526 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4527 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4528 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4529 }
4530 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4531 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4532 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4533 }
4534 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4535 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4536 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4537 }
4538 } else {
4539 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4540 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4541 cp_int_cntl |= RB_INT_ENABLE;
4542 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4543 }
4544 }
4545
4546 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4547 DRM_DEBUG("r600_irq_set: sw int dma\n");
4548 dma_cntl |= TRAP_ENABLE;
4549 }
4550
4551 if (rdev->family >= CHIP_CAYMAN) {
4552 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4553 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4554 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4555 dma_cntl1 |= TRAP_ENABLE;
4556 }
4557 }
4558
4559 if (rdev->irq.dpm_thermal) {
4560 DRM_DEBUG("dpm thermal\n");
4561 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4562 }
4563
4564 if (rdev->family >= CHIP_CAYMAN) {
4565 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4566 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4567 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4568 } else
4569 WREG32(CP_INT_CNTL, cp_int_cntl);
4570
4571 WREG32(DMA_CNTL, dma_cntl);
4572
4573 if (rdev->family >= CHIP_CAYMAN)
4574 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4575
4576 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4577
4578 for (i = 0; i < rdev->num_crtc; i++) {
4579 radeon_irq_kms_set_irq_n_enabled(
4580 rdev, INT_MASK + crtc_offsets[i],
4581 VBLANK_INT_MASK,
4582 rdev->irq.crtc_vblank_int[i] ||
4583 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4584 }
4585
4586 for (i = 0; i < rdev->num_crtc; i++)
4587 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4588
4589 for (i = 0; i < 6; i++) {
4590 radeon_irq_kms_set_irq_n_enabled(
4591 rdev, DC_HPDx_INT_CONTROL(i),
4592 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4593 rdev->irq.hpd[i], "HPD", i);
4594 }
4595
4596 if (rdev->family == CHIP_ARUBA)
4597 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4598 else
4599 WREG32(CG_THERMAL_INT, thermal_int);
4600
4601 for (i = 0; i < 6; i++) {
4602 radeon_irq_kms_set_irq_n_enabled(
4603 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4604 AFMT_AZ_FORMAT_WTRIG_MASK,
4605 rdev->irq.afmt[i], "HDMI", i);
4606 }
4607
4608 /* posting read */
4609 RREG32(SRBM_STATUS);
4610
4611 return 0;
4612}
4613
4614/* Note that the order we write back regs here is important */
4615static void evergreen_irq_ack(struct radeon_device *rdev)
4616{
4617 int i, j;
4618 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4619 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4620 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4621
4622 for (i = 0; i < 6; i++) {
4623 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4624 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4625 if (i < rdev->num_crtc)
4626 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4627 }
4628
4629 /* We write back each interrupt register in pairs of two */
4630 for (i = 0; i < rdev->num_crtc; i += 2) {
4631 for (j = i; j < (i + 2); j++) {
4632 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4633 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4634 GRPH_PFLIP_INT_CLEAR);
4635 }
4636
4637 for (j = i; j < (i + 2); j++) {
4638 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4639 WREG32(VBLANK_STATUS + crtc_offsets[j],
4640 VBLANK_ACK);
4641 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4642 WREG32(VLINE_STATUS + crtc_offsets[j],
4643 VLINE_ACK);
4644 }
4645 }
4646
4647 for (i = 0; i < 6; i++) {
4648 if (disp_int[i] & DC_HPD1_INTERRUPT)
4649 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4650 }
4651
4652 for (i = 0; i < 6; i++) {
4653 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4654 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4655 }
4656
4657 for (i = 0; i < 6; i++) {
4658 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4659 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4660 AFMT_AZ_FORMAT_WTRIG_ACK);
4661 }
4662}
4663
4664static void evergreen_irq_disable(struct radeon_device *rdev)
4665{
4666 r600_disable_interrupts(rdev);
4667 /* Wait and acknowledge irq */
4668 mdelay(1);
4669 evergreen_irq_ack(rdev);
4670 evergreen_disable_interrupt_state(rdev);
4671}
4672
4673void evergreen_irq_suspend(struct radeon_device *rdev)
4674{
4675 evergreen_irq_disable(rdev);
4676 r600_rlc_stop(rdev);
4677}
4678
4679static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4680{
4681 u32 wptr, tmp;
4682
4683 if (rdev->wb.enabled)
4684 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4685 else
4686 wptr = RREG32(IH_RB_WPTR);
4687
4688 if (wptr & RB_OVERFLOW) {
4689 wptr &= ~RB_OVERFLOW;
4690 /* When a ring buffer overflow happen start parsing interrupt
4691 * from the last not overwritten vector (wptr + 16). Hopefully
4692 * this should allow us to catchup.
4693 */
4694 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4695 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4696 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4697 tmp = RREG32(IH_RB_CNTL);
4698 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4699 WREG32(IH_RB_CNTL, tmp);
4700 }
4701 return (wptr & rdev->ih.ptr_mask);
4702}
4703
4704int evergreen_irq_process(struct radeon_device *rdev)
4705{
4706 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4707 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4708 u32 crtc_idx, hpd_idx, afmt_idx;
4709 u32 mask;
4710 u32 wptr;
4711 u32 rptr;
4712 u32 src_id, src_data;
4713 u32 ring_index;
4714 bool queue_hotplug = false;
4715 bool queue_hdmi = false;
4716 bool queue_dp = false;
4717 bool queue_thermal = false;
4718 u32 status, addr;
4719 const char *event_name;
4720
4721 if (!rdev->ih.enabled || rdev->shutdown)
4722 return IRQ_NONE;
4723
4724 wptr = evergreen_get_ih_wptr(rdev);
4725
4726restart_ih:
4727 /* is somebody else already processing irqs? */
4728 if (atomic_xchg(&rdev->ih.lock, 1))
4729 return IRQ_NONE;
4730
4731 rptr = rdev->ih.rptr;
4732 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4733
4734 /* Order reading of wptr vs. reading of IH ring data */
4735 rmb();
4736
4737 /* display interrupts */
4738 evergreen_irq_ack(rdev);
4739
4740 while (rptr != wptr) {
4741 /* wptr/rptr are in bytes! */
4742 ring_index = rptr / 4;
4743 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4744 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4745
4746 switch (src_id) {
4747 case 1: /* D1 vblank/vline */
4748 case 2: /* D2 vblank/vline */
4749 case 3: /* D3 vblank/vline */
4750 case 4: /* D4 vblank/vline */
4751 case 5: /* D5 vblank/vline */
4752 case 6: /* D6 vblank/vline */
4753 crtc_idx = src_id - 1;
4754
4755 if (src_data == 0) { /* vblank */
4756 mask = LB_D1_VBLANK_INTERRUPT;
4757 event_name = "vblank";
4758
4759 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4760 drm_handle_vblank(rdev->ddev, crtc_idx);
4761 rdev->pm.vblank_sync = true;
4762 wake_up(&rdev->irq.vblank_queue);
4763 }
4764 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4765 radeon_crtc_handle_vblank(rdev,
4766 crtc_idx);
4767 }
4768
4769 } else if (src_data == 1) { /* vline */
4770 mask = LB_D1_VLINE_INTERRUPT;
4771 event_name = "vline";
4772 } else {
4773 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4774 src_id, src_data);
4775 break;
4776 }
4777
4778 if (!(disp_int[crtc_idx] & mask)) {
4779 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4780 crtc_idx + 1, event_name);
4781 }
4782
4783 disp_int[crtc_idx] &= ~mask;
4784 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4785
4786 break;
4787 case 8: /* D1 page flip */
4788 case 10: /* D2 page flip */
4789 case 12: /* D3 page flip */
4790 case 14: /* D4 page flip */
4791 case 16: /* D5 page flip */
4792 case 18: /* D6 page flip */
4793 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4794 if (radeon_use_pflipirq > 0)
4795 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4796 break;
4797 case 42: /* HPD hotplug */
4798 if (src_data <= 5) {
4799 hpd_idx = src_data;
4800 mask = DC_HPD1_INTERRUPT;
4801 queue_hotplug = true;
4802 event_name = "HPD";
4803
4804 } else if (src_data <= 11) {
4805 hpd_idx = src_data - 6;
4806 mask = DC_HPD1_RX_INTERRUPT;
4807 queue_dp = true;
4808 event_name = "HPD_RX";
4809
4810 } else {
4811 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4812 src_id, src_data);
4813 break;
4814 }
4815
4816 if (!(disp_int[hpd_idx] & mask))
4817 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4818
4819 disp_int[hpd_idx] &= ~mask;
4820 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4821
4822 break;
4823 case 44: /* hdmi */
4824 afmt_idx = src_data;
4825 if (afmt_idx > 5) {
4826 DRM_ERROR("Unhandled interrupt: %d %d\n",
4827 src_id, src_data);
4828 break;
4829 }
4830
4831 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4832 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4833
4834 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4835 queue_hdmi = true;
4836 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4837 break;
4838 case 96:
4839 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4840 WREG32(SRBM_INT_ACK, 0x1);
4841 break;
4842 case 124: /* UVD */
4843 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4844 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4845 break;
4846 case 146:
4847 case 147:
4848 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4849 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4850 /* reset addr and status */
4851 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4852 if (addr == 0x0 && status == 0x0)
4853 break;
4854 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4855 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4856 addr);
4857 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4858 status);
4859 cayman_vm_decode_fault(rdev, status, addr);
4860 break;
4861 case 176: /* CP_INT in ring buffer */
4862 case 177: /* CP_INT in IB1 */
4863 case 178: /* CP_INT in IB2 */
4864 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4865 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4866 break;
4867 case 181: /* CP EOP event */
4868 DRM_DEBUG("IH: CP EOP\n");
4869 if (rdev->family >= CHIP_CAYMAN) {
4870 switch (src_data) {
4871 case 0:
4872 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4873 break;
4874 case 1:
4875 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4876 break;
4877 case 2:
4878 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4879 break;
4880 }
4881 } else
4882 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4883 break;
4884 case 224: /* DMA trap event */
4885 DRM_DEBUG("IH: DMA trap\n");
4886 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4887 break;
4888 case 230: /* thermal low to high */
4889 DRM_DEBUG("IH: thermal low to high\n");
4890 rdev->pm.dpm.thermal.high_to_low = false;
4891 queue_thermal = true;
4892 break;
4893 case 231: /* thermal high to low */
4894 DRM_DEBUG("IH: thermal high to low\n");
4895 rdev->pm.dpm.thermal.high_to_low = true;
4896 queue_thermal = true;
4897 break;
4898 case 233: /* GUI IDLE */
4899 DRM_DEBUG("IH: GUI idle\n");
4900 break;
4901 case 244: /* DMA trap event */
4902 if (rdev->family >= CHIP_CAYMAN) {
4903 DRM_DEBUG("IH: DMA1 trap\n");
4904 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4905 }
4906 break;
4907 default:
4908 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4909 break;
4910 }
4911
4912 /* wptr/rptr are in bytes! */
4913 rptr += 16;
4914 rptr &= rdev->ih.ptr_mask;
4915 WREG32(IH_RB_RPTR, rptr);
4916 }
4917 if (queue_dp)
4918 schedule_work(&rdev->dp_work);
4919 if (queue_hotplug)
4920 schedule_delayed_work(&rdev->hotplug_work, 0);
4921 if (queue_hdmi)
4922 schedule_work(&rdev->audio_work);
4923 if (queue_thermal && rdev->pm.dpm_enabled)
4924 schedule_work(&rdev->pm.dpm.thermal.work);
4925 rdev->ih.rptr = rptr;
4926 atomic_set(&rdev->ih.lock, 0);
4927
4928 /* make sure wptr hasn't changed while processing */
4929 wptr = evergreen_get_ih_wptr(rdev);
4930 if (wptr != rptr)
4931 goto restart_ih;
4932
4933 return IRQ_HANDLED;
4934}
4935
4936static void evergreen_uvd_init(struct radeon_device *rdev)
4937{
4938 int r;
4939
4940 if (!rdev->has_uvd)
4941 return;
4942
4943 r = radeon_uvd_init(rdev);
4944 if (r) {
4945 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4946 /*
4947 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4948 * to early fails uvd_v2_2_resume() and thus nothing happens
4949 * there. So it is pointless to try to go through that code
4950 * hence why we disable uvd here.
4951 */
4952 rdev->has_uvd = false;
4953 return;
4954 }
4955 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4956 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4957}
4958
4959static void evergreen_uvd_start(struct radeon_device *rdev)
4960{
4961 int r;
4962
4963 if (!rdev->has_uvd)
4964 return;
4965
4966 r = uvd_v2_2_resume(rdev);
4967 if (r) {
4968 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4969 goto error;
4970 }
4971 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4972 if (r) {
4973 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4974 goto error;
4975 }
4976 return;
4977
4978error:
4979 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4980}
4981
4982static void evergreen_uvd_resume(struct radeon_device *rdev)
4983{
4984 struct radeon_ring *ring;
4985 int r;
4986
4987 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4988 return;
4989
4990 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4991 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4992 if (r) {
4993 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4994 return;
4995 }
4996 r = uvd_v1_0_init(rdev);
4997 if (r) {
4998 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4999 return;
5000 }
5001}
5002
5003static int evergreen_startup(struct radeon_device *rdev)
5004{
5005 struct radeon_ring *ring;
5006 int r;
5007
5008 /* enable pcie gen2 link */
5009 evergreen_pcie_gen2_enable(rdev);
5010 /* enable aspm */
5011 evergreen_program_aspm(rdev);
5012
5013 /* scratch needs to be initialized before MC */
5014 r = r600_vram_scratch_init(rdev);
5015 if (r)
5016 return r;
5017
5018 evergreen_mc_program(rdev);
5019
5020 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5021 r = ni_mc_load_microcode(rdev);
5022 if (r) {
5023 DRM_ERROR("Failed to load MC firmware!\n");
5024 return r;
5025 }
5026 }
5027
5028 if (rdev->flags & RADEON_IS_AGP) {
5029 evergreen_agp_enable(rdev);
5030 } else {
5031 r = evergreen_pcie_gart_enable(rdev);
5032 if (r)
5033 return r;
5034 }
5035 evergreen_gpu_init(rdev);
5036
5037 /* allocate rlc buffers */
5038 if (rdev->flags & RADEON_IS_IGP) {
5039 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5040 rdev->rlc.reg_list_size =
5041 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5042 rdev->rlc.cs_data = evergreen_cs_data;
5043 r = sumo_rlc_init(rdev);
5044 if (r) {
5045 DRM_ERROR("Failed to init rlc BOs!\n");
5046 return r;
5047 }
5048 }
5049
5050 /* allocate wb buffer */
5051 r = radeon_wb_init(rdev);
5052 if (r)
5053 return r;
5054
5055 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5056 if (r) {
5057 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5058 return r;
5059 }
5060
5061 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5062 if (r) {
5063 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5064 return r;
5065 }
5066
5067 evergreen_uvd_start(rdev);
5068
5069 /* Enable IRQ */
5070 if (!rdev->irq.installed) {
5071 r = radeon_irq_kms_init(rdev);
5072 if (r)
5073 return r;
5074 }
5075
5076 r = r600_irq_init(rdev);
5077 if (r) {
5078 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5079 radeon_irq_kms_fini(rdev);
5080 return r;
5081 }
5082 evergreen_irq_set(rdev);
5083
5084 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5085 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5086 RADEON_CP_PACKET2);
5087 if (r)
5088 return r;
5089
5090 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5091 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5092 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5093 if (r)
5094 return r;
5095
5096 r = evergreen_cp_load_microcode(rdev);
5097 if (r)
5098 return r;
5099 r = evergreen_cp_resume(rdev);
5100 if (r)
5101 return r;
5102 r = r600_dma_resume(rdev);
5103 if (r)
5104 return r;
5105
5106 evergreen_uvd_resume(rdev);
5107
5108 r = radeon_ib_pool_init(rdev);
5109 if (r) {
5110 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5111 return r;
5112 }
5113
5114 r = radeon_audio_init(rdev);
5115 if (r) {
5116 DRM_ERROR("radeon: audio init failed\n");
5117 return r;
5118 }
5119
5120 return 0;
5121}
5122
5123int evergreen_resume(struct radeon_device *rdev)
5124{
5125 int r;
5126
5127 /* reset the asic, the gfx blocks are often in a bad state
5128 * after the driver is unloaded or after a resume
5129 */
5130 if (radeon_asic_reset(rdev))
5131 dev_warn(rdev->dev, "GPU reset failed !\n");
5132 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5133 * posting will perform necessary task to bring back GPU into good
5134 * shape.
5135 */
5136 /* post card */
5137 atom_asic_init(rdev->mode_info.atom_context);
5138
5139 /* init golden registers */
5140 evergreen_init_golden_registers(rdev);
5141
5142 if (rdev->pm.pm_method == PM_METHOD_DPM)
5143 radeon_pm_resume(rdev);
5144
5145 rdev->accel_working = true;
5146 r = evergreen_startup(rdev);
5147 if (r) {
5148 DRM_ERROR("evergreen startup failed on resume\n");
5149 rdev->accel_working = false;
5150 return r;
5151 }
5152
5153 return r;
5154
5155}
5156
5157int evergreen_suspend(struct radeon_device *rdev)
5158{
5159 radeon_pm_suspend(rdev);
5160 radeon_audio_fini(rdev);
5161 if (rdev->has_uvd) {
5162 radeon_uvd_suspend(rdev);
5163 uvd_v1_0_fini(rdev);
5164 }
5165 r700_cp_stop(rdev);
5166 r600_dma_stop(rdev);
5167 evergreen_irq_suspend(rdev);
5168 radeon_wb_disable(rdev);
5169 evergreen_pcie_gart_disable(rdev);
5170
5171 return 0;
5172}
5173
5174/* Plan is to move initialization in that function and use
5175 * helper function so that radeon_device_init pretty much
5176 * do nothing more than calling asic specific function. This
5177 * should also allow to remove a bunch of callback function
5178 * like vram_info.
5179 */
5180int evergreen_init(struct radeon_device *rdev)
5181{
5182 int r;
5183
5184 /* Read BIOS */
5185 if (!radeon_get_bios(rdev)) {
5186 if (ASIC_IS_AVIVO(rdev))
5187 return -EINVAL;
5188 }
5189 /* Must be an ATOMBIOS */
5190 if (!rdev->is_atom_bios) {
5191 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5192 return -EINVAL;
5193 }
5194 r = radeon_atombios_init(rdev);
5195 if (r)
5196 return r;
5197 /* reset the asic, the gfx blocks are often in a bad state
5198 * after the driver is unloaded or after a resume
5199 */
5200 if (radeon_asic_reset(rdev))
5201 dev_warn(rdev->dev, "GPU reset failed !\n");
5202 /* Post card if necessary */
5203 if (!radeon_card_posted(rdev)) {
5204 if (!rdev->bios) {
5205 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5206 return -EINVAL;
5207 }
5208 DRM_INFO("GPU not posted. posting now...\n");
5209 atom_asic_init(rdev->mode_info.atom_context);
5210 }
5211 /* init golden registers */
5212 evergreen_init_golden_registers(rdev);
5213 /* Initialize scratch registers */
5214 r600_scratch_init(rdev);
5215 /* Initialize surface registers */
5216 radeon_surface_init(rdev);
5217 /* Initialize clocks */
5218 radeon_get_clock_info(rdev->ddev);
5219 /* Fence driver */
5220 radeon_fence_driver_init(rdev);
5221 /* initialize AGP */
5222 if (rdev->flags & RADEON_IS_AGP) {
5223 r = radeon_agp_init(rdev);
5224 if (r)
5225 radeon_agp_disable(rdev);
5226 }
5227 /* initialize memory controller */
5228 r = evergreen_mc_init(rdev);
5229 if (r)
5230 return r;
5231 /* Memory manager */
5232 r = radeon_bo_init(rdev);
5233 if (r)
5234 return r;
5235
5236 if (ASIC_IS_DCE5(rdev)) {
5237 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5238 r = ni_init_microcode(rdev);
5239 if (r) {
5240 DRM_ERROR("Failed to load firmware!\n");
5241 return r;
5242 }
5243 }
5244 } else {
5245 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5246 r = r600_init_microcode(rdev);
5247 if (r) {
5248 DRM_ERROR("Failed to load firmware!\n");
5249 return r;
5250 }
5251 }
5252 }
5253
5254 /* Initialize power management */
5255 radeon_pm_init(rdev);
5256
5257 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5258 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5259
5260 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5261 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5262
5263 evergreen_uvd_init(rdev);
5264
5265 rdev->ih.ring_obj = NULL;
5266 r600_ih_ring_init(rdev, 64 * 1024);
5267
5268 r = r600_pcie_gart_init(rdev);
5269 if (r)
5270 return r;
5271
5272 rdev->accel_working = true;
5273 r = evergreen_startup(rdev);
5274 if (r) {
5275 dev_err(rdev->dev, "disabling GPU acceleration\n");
5276 r700_cp_fini(rdev);
5277 r600_dma_fini(rdev);
5278 r600_irq_fini(rdev);
5279 if (rdev->flags & RADEON_IS_IGP)
5280 sumo_rlc_fini(rdev);
5281 radeon_wb_fini(rdev);
5282 radeon_ib_pool_fini(rdev);
5283 radeon_irq_kms_fini(rdev);
5284 evergreen_pcie_gart_fini(rdev);
5285 rdev->accel_working = false;
5286 }
5287
5288 /* Don't start up if the MC ucode is missing on BTC parts.
5289 * The default clocks and voltages before the MC ucode
5290 * is loaded are not suffient for advanced operations.
5291 */
5292 if (ASIC_IS_DCE5(rdev)) {
5293 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5294 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5295 return -EINVAL;
5296 }
5297 }
5298
5299 return 0;
5300}
5301
5302void evergreen_fini(struct radeon_device *rdev)
5303{
5304 radeon_pm_fini(rdev);
5305 radeon_audio_fini(rdev);
5306 r700_cp_fini(rdev);
5307 r600_dma_fini(rdev);
5308 r600_irq_fini(rdev);
5309 if (rdev->flags & RADEON_IS_IGP)
5310 sumo_rlc_fini(rdev);
5311 radeon_wb_fini(rdev);
5312 radeon_ib_pool_fini(rdev);
5313 radeon_irq_kms_fini(rdev);
5314 uvd_v1_0_fini(rdev);
5315 radeon_uvd_fini(rdev);
5316 evergreen_pcie_gart_fini(rdev);
5317 r600_vram_scratch_fini(rdev);
5318 radeon_gem_fini(rdev);
5319 radeon_fence_driver_fini(rdev);
5320 radeon_agp_fini(rdev);
5321 radeon_bo_fini(rdev);
5322 radeon_atombios_fini(rdev);
5323 kfree(rdev->bios);
5324 rdev->bios = NULL;
5325}
5326
5327void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5328{
5329 u32 link_width_cntl, speed_cntl;
5330
5331 if (radeon_pcie_gen2 == 0)
5332 return;
5333
5334 if (rdev->flags & RADEON_IS_IGP)
5335 return;
5336
5337 if (!(rdev->flags & RADEON_IS_PCIE))
5338 return;
5339
5340 /* x2 cards have a special sequence */
5341 if (ASIC_IS_X2(rdev))
5342 return;
5343
5344 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5345 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5346 return;
5347
5348 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5349 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5350 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5351 return;
5352 }
5353
5354 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5355
5356 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5357 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5358
5359 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5360 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5361 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5362
5363 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5364 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5365 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5366
5367 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5368 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5369 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5370
5371 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5373 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5374
5375 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5376 speed_cntl |= LC_GEN2_EN_STRAP;
5377 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5378
5379 } else {
5380 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5381 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5382 if (1)
5383 link_width_cntl |= LC_UPCONFIGURE_DIS;
5384 else
5385 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5386 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5387 }
5388}
5389
5390void evergreen_program_aspm(struct radeon_device *rdev)
5391{
5392 u32 data, orig;
5393 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5394 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5395 /* fusion_platform = true
5396 * if the system is a fusion system
5397 * (APU or DGPU in a fusion system).
5398 * todo: check if the system is a fusion platform.
5399 */
5400 bool fusion_platform = false;
5401
5402 if (radeon_aspm == 0)
5403 return;
5404
5405 if (!(rdev->flags & RADEON_IS_PCIE))
5406 return;
5407
5408 switch (rdev->family) {
5409 case CHIP_CYPRESS:
5410 case CHIP_HEMLOCK:
5411 case CHIP_JUNIPER:
5412 case CHIP_REDWOOD:
5413 case CHIP_CEDAR:
5414 case CHIP_SUMO:
5415 case CHIP_SUMO2:
5416 case CHIP_PALM:
5417 case CHIP_ARUBA:
5418 disable_l0s = true;
5419 break;
5420 default:
5421 disable_l0s = false;
5422 break;
5423 }
5424
5425 if (rdev->flags & RADEON_IS_IGP)
5426 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5427
5428 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5429 if (fusion_platform)
5430 data &= ~MULTI_PIF;
5431 else
5432 data |= MULTI_PIF;
5433 if (data != orig)
5434 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5435
5436 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5437 if (fusion_platform)
5438 data &= ~MULTI_PIF;
5439 else
5440 data |= MULTI_PIF;
5441 if (data != orig)
5442 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5443
5444 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5445 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5446 if (!disable_l0s) {
5447 if (rdev->family >= CHIP_BARTS)
5448 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5449 else
5450 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5451 }
5452
5453 if (!disable_l1) {
5454 if (rdev->family >= CHIP_BARTS)
5455 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5456 else
5457 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5458
5459 if (!disable_plloff_in_l1) {
5460 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5461 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5462 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5463 if (data != orig)
5464 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5465
5466 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5467 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5468 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5469 if (data != orig)
5470 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5471
5472 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5473 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5474 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5475 if (data != orig)
5476 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5477
5478 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5479 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5480 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5481 if (data != orig)
5482 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5483
5484 if (rdev->family >= CHIP_BARTS) {
5485 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5486 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5487 data |= PLL_RAMP_UP_TIME_0(4);
5488 if (data != orig)
5489 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5490
5491 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5492 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5493 data |= PLL_RAMP_UP_TIME_1(4);
5494 if (data != orig)
5495 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5496
5497 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5498 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5499 data |= PLL_RAMP_UP_TIME_0(4);
5500 if (data != orig)
5501 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5502
5503 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5504 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5505 data |= PLL_RAMP_UP_TIME_1(4);
5506 if (data != orig)
5507 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5508 }
5509
5510 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5511 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5512 data |= LC_DYN_LANES_PWR_STATE(3);
5513 if (data != orig)
5514 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5515
5516 if (rdev->family >= CHIP_BARTS) {
5517 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5518 data &= ~LS2_EXIT_TIME_MASK;
5519 data |= LS2_EXIT_TIME(1);
5520 if (data != orig)
5521 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5522
5523 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5524 data &= ~LS2_EXIT_TIME_MASK;
5525 data |= LS2_EXIT_TIME(1);
5526 if (data != orig)
5527 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5528 }
5529 }
5530 }
5531
5532 /* evergreen parts only */
5533 if (rdev->family < CHIP_BARTS)
5534 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5535
5536 if (pcie_lc_cntl != pcie_lc_cntl_old)
5537 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5538}