1/*
  2 * Copyright 2008 Advanced Micro Devices, Inc.
  3 * Copyright 2008 Red Hat Inc.
  4 * Copyright 2009 Jerome Glisse.
  5 *
  6 * Permission is hereby granted, free of charge, to any person obtaining a
  7 * copy of this software and associated documentation files (the "Software"),
  8 * to deal in the Software without restriction, including without limitation
  9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 10 * and/or sell copies of the Software, and to permit persons to whom the
 11 * Software is furnished to do so, subject to the following conditions:
 12 *
 13 * The above copyright notice and this permission notice shall be included in
 14 * all copies or substantial portions of the Software.
 15 *
 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 22 * OTHER DEALINGS IN THE SOFTWARE.
 23 *
 24 * Authors: Dave Airlie
 25 *          Alex Deucher
 26 *          Jerome Glisse
 27 */
 28#include "drmP.h"
 29#include "radeon_drm.h"
 30#include "radeon_reg.h"
 31#include "radeon.h"
 32#include "atom.h"
 33
 34/* 10 khz */
 35uint32_t radeon_legacy_get_engine_clock(struct radeon_device *rdev)
 36{
 37	struct radeon_pll *spll = &rdev->clock.spll;
 38	uint32_t fb_div, ref_div, post_div, sclk;
 39
 40	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
 41	fb_div = (fb_div >> RADEON_SPLL_FB_DIV_SHIFT) & RADEON_SPLL_FB_DIV_MASK;
 42	fb_div <<= 1;
 43	fb_div *= spll->reference_freq;
 44
 45	ref_div =
 46	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
 47
 48	if (ref_div == 0)
 49		return 0;
 50
 51	sclk = fb_div / ref_div;
 52
 53	post_div = RREG32_PLL(RADEON_SCLK_CNTL) & RADEON_SCLK_SRC_SEL_MASK;
 54	if (post_div == 2)
 55		sclk >>= 1;
 56	else if (post_div == 3)
 57		sclk >>= 2;
 58	else if (post_div == 4)
 59		sclk >>= 3;
 60
 61	return sclk;
 62}
 63
 64/* 10 khz */
 65uint32_t radeon_legacy_get_memory_clock(struct radeon_device *rdev)
 66{
 67	struct radeon_pll *mpll = &rdev->clock.mpll;
 68	uint32_t fb_div, ref_div, post_div, mclk;
 69
 70	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
 71	fb_div = (fb_div >> RADEON_MPLL_FB_DIV_SHIFT) & RADEON_MPLL_FB_DIV_MASK;
 72	fb_div <<= 1;
 73	fb_div *= mpll->reference_freq;
 74
 75	ref_div =
 76	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
 77
 78	if (ref_div == 0)
 79		return 0;
 80
 81	mclk = fb_div / ref_div;
 82
 83	post_div = RREG32_PLL(RADEON_MCLK_CNTL) & 0x7;
 84	if (post_div == 2)
 85		mclk >>= 1;
 86	else if (post_div == 3)
 87		mclk >>= 2;
 88	else if (post_div == 4)
 89		mclk >>= 3;
 90
 91	return mclk;
 92}
 93
 94#ifdef CONFIG_OF
 95/*
 96 * Read XTAL (ref clock), SCLK and MCLK from Open Firmware device
 97 * tree. Hopefully, ATI OF driver is kind enough to fill these
 98 */
 99static bool radeon_read_clocks_OF(struct drm_device *dev)
100{
101	struct radeon_device *rdev = dev->dev_private;
102	struct device_node *dp = rdev->pdev->dev.of_node;
103	const u32 *val;
104	struct radeon_pll *p1pll = &rdev->clock.p1pll;
105	struct radeon_pll *p2pll = &rdev->clock.p2pll;
106	struct radeon_pll *spll = &rdev->clock.spll;
107	struct radeon_pll *mpll = &rdev->clock.mpll;
108
109	if (dp == NULL)
110		return false;
111	val = of_get_property(dp, "ATY,RefCLK", NULL);
112	if (!val || !*val) {
113		printk(KERN_WARNING "radeonfb: No ATY,RefCLK property !\n");
114		return false;
115	}
116	p1pll->reference_freq = p2pll->reference_freq = (*val) / 10;
117	p1pll->reference_div = RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
118	if (p1pll->reference_div < 2)
119		p1pll->reference_div = 12;
120	p2pll->reference_div = p1pll->reference_div;
121
122	/* These aren't in the device-tree */
123	if (rdev->family >= CHIP_R420) {
124		p1pll->pll_in_min = 100;
125		p1pll->pll_in_max = 1350;
126		p1pll->pll_out_min = 20000;
127		p1pll->pll_out_max = 50000;
128		p2pll->pll_in_min = 100;
129		p2pll->pll_in_max = 1350;
130		p2pll->pll_out_min = 20000;
131		p2pll->pll_out_max = 50000;
132	} else {
133		p1pll->pll_in_min = 40;
134		p1pll->pll_in_max = 500;
135		p1pll->pll_out_min = 12500;
136		p1pll->pll_out_max = 35000;
137		p2pll->pll_in_min = 40;
138		p2pll->pll_in_max = 500;
139		p2pll->pll_out_min = 12500;
140		p2pll->pll_out_max = 35000;
141	}
142	/* not sure what the max should be in all cases */
143	rdev->clock.max_pixel_clock = 35000;
144
145	spll->reference_freq = mpll->reference_freq = p1pll->reference_freq;
146	spll->reference_div = mpll->reference_div =
147		RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
148			    RADEON_M_SPLL_REF_DIV_MASK;
149
150	val = of_get_property(dp, "ATY,SCLK", NULL);
151	if (val && *val)
152		rdev->clock.default_sclk = (*val) / 10;
153	else
154		rdev->clock.default_sclk =
155			radeon_legacy_get_engine_clock(rdev);
156
157	val = of_get_property(dp, "ATY,MCLK", NULL);
158	if (val && *val)
159		rdev->clock.default_mclk = (*val) / 10;
160	else
161		rdev->clock.default_mclk =
162			radeon_legacy_get_memory_clock(rdev);
163
164	DRM_INFO("Using device-tree clock info\n");
165
166	return true;
167}
168#else
169static bool radeon_read_clocks_OF(struct drm_device *dev)
170{
171	return false;
172}
173#endif /* CONFIG_OF */
174
175void radeon_get_clock_info(struct drm_device *dev)
176{
177	struct radeon_device *rdev = dev->dev_private;
178	struct radeon_pll *p1pll = &rdev->clock.p1pll;
179	struct radeon_pll *p2pll = &rdev->clock.p2pll;
180	struct radeon_pll *dcpll = &rdev->clock.dcpll;
181	struct radeon_pll *spll = &rdev->clock.spll;
182	struct radeon_pll *mpll = &rdev->clock.mpll;
183	int ret;
184
185	if (rdev->is_atom_bios)
186		ret = radeon_atom_get_clock_info(dev);
187	else
188		ret = radeon_combios_get_clock_info(dev);
189	if (!ret)
190		ret = radeon_read_clocks_OF(dev);
191
192	if (ret) {
193		if (p1pll->reference_div < 2) {
194			if (!ASIC_IS_AVIVO(rdev)) {
195				u32 tmp = RREG32_PLL(RADEON_PPLL_REF_DIV);
196				if (ASIC_IS_R300(rdev))
197					p1pll->reference_div =
198						(tmp & R300_PPLL_REF_DIV_ACC_MASK) >> R300_PPLL_REF_DIV_ACC_SHIFT;
199				else
200					p1pll->reference_div = tmp & RADEON_PPLL_REF_DIV_MASK;
201				if (p1pll->reference_div < 2)
202					p1pll->reference_div = 12;
203			} else
204				p1pll->reference_div = 12;
205		}
206		if (p2pll->reference_div < 2)
207			p2pll->reference_div = 12;
208		if (rdev->family < CHIP_RS600) {
209			if (spll->reference_div < 2)
210				spll->reference_div =
211					RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
212					RADEON_M_SPLL_REF_DIV_MASK;
213		}
214		if (mpll->reference_div < 2)
215			mpll->reference_div = spll->reference_div;
216	} else {
217		if (ASIC_IS_AVIVO(rdev)) {
218			/* TODO FALLBACK */
219		} else {
220			DRM_INFO("Using generic clock info\n");
221
222			/* may need to be per card */
223			rdev->clock.max_pixel_clock = 35000;
224
225			if (rdev->flags & RADEON_IS_IGP) {
226				p1pll->reference_freq = 1432;
227				p2pll->reference_freq = 1432;
228				spll->reference_freq = 1432;
229				mpll->reference_freq = 1432;
230			} else {
231				p1pll->reference_freq = 2700;
232				p2pll->reference_freq = 2700;
233				spll->reference_freq = 2700;
234				mpll->reference_freq = 2700;
235			}
236			p1pll->reference_div =
237			    RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
238			if (p1pll->reference_div < 2)
239				p1pll->reference_div = 12;
240			p2pll->reference_div = p1pll->reference_div;
241
242			if (rdev->family >= CHIP_R420) {
243				p1pll->pll_in_min = 100;
244				p1pll->pll_in_max = 1350;
245				p1pll->pll_out_min = 20000;
246				p1pll->pll_out_max = 50000;
247				p2pll->pll_in_min = 100;
248				p2pll->pll_in_max = 1350;
249				p2pll->pll_out_min = 20000;
250				p2pll->pll_out_max = 50000;
251			} else {
252				p1pll->pll_in_min = 40;
253				p1pll->pll_in_max = 500;
254				p1pll->pll_out_min = 12500;
255				p1pll->pll_out_max = 35000;
256				p2pll->pll_in_min = 40;
257				p2pll->pll_in_max = 500;
258				p2pll->pll_out_min = 12500;
259				p2pll->pll_out_max = 35000;
260			}
261
262			spll->reference_div =
263			    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
264			    RADEON_M_SPLL_REF_DIV_MASK;
265			mpll->reference_div = spll->reference_div;
266			rdev->clock.default_sclk =
267			    radeon_legacy_get_engine_clock(rdev);
268			rdev->clock.default_mclk =
269			    radeon_legacy_get_memory_clock(rdev);
270		}
271	}
272
273	/* pixel clocks */
274	if (ASIC_IS_AVIVO(rdev)) {
275		p1pll->min_post_div = 2;
276		p1pll->max_post_div = 0x7f;
277		p1pll->min_frac_feedback_div = 0;
278		p1pll->max_frac_feedback_div = 9;
279		p2pll->min_post_div = 2;
280		p2pll->max_post_div = 0x7f;
281		p2pll->min_frac_feedback_div = 0;
282		p2pll->max_frac_feedback_div = 9;
283	} else {
284		p1pll->min_post_div = 1;
285		p1pll->max_post_div = 16;
286		p1pll->min_frac_feedback_div = 0;
287		p1pll->max_frac_feedback_div = 0;
288		p2pll->min_post_div = 1;
289		p2pll->max_post_div = 12;
290		p2pll->min_frac_feedback_div = 0;
291		p2pll->max_frac_feedback_div = 0;
292	}
293
294	/* dcpll is DCE4 only */
295	dcpll->min_post_div = 2;
296	dcpll->max_post_div = 0x7f;
297	dcpll->min_frac_feedback_div = 0;
298	dcpll->max_frac_feedback_div = 9;
299	dcpll->min_ref_div = 2;
300	dcpll->max_ref_div = 0x3ff;
301	dcpll->min_feedback_div = 4;
302	dcpll->max_feedback_div = 0xfff;
303	dcpll->best_vco = 0;
304
305	p1pll->min_ref_div = 2;
306	p1pll->max_ref_div = 0x3ff;
307	p1pll->min_feedback_div = 4;
308	p1pll->max_feedback_div = 0x7ff;
309	p1pll->best_vco = 0;
310
311	p2pll->min_ref_div = 2;
312	p2pll->max_ref_div = 0x3ff;
313	p2pll->min_feedback_div = 4;
314	p2pll->max_feedback_div = 0x7ff;
315	p2pll->best_vco = 0;
316
317	/* system clock */
318	spll->min_post_div = 1;
319	spll->max_post_div = 1;
320	spll->min_ref_div = 2;
321	spll->max_ref_div = 0xff;
322	spll->min_feedback_div = 4;
323	spll->max_feedback_div = 0xff;
324	spll->best_vco = 0;
325
326	/* memory clock */
327	mpll->min_post_div = 1;
328	mpll->max_post_div = 1;
329	mpll->min_ref_div = 2;
330	mpll->max_ref_div = 0xff;
331	mpll->min_feedback_div = 4;
332	mpll->max_feedback_div = 0xff;
333	mpll->best_vco = 0;
334
335	if (!rdev->clock.default_sclk)
336		rdev->clock.default_sclk = radeon_get_engine_clock(rdev);
337	if ((!rdev->clock.default_mclk) && rdev->asic->get_memory_clock)
338		rdev->clock.default_mclk = radeon_get_memory_clock(rdev);
339
340	rdev->pm.current_sclk = rdev->clock.default_sclk;
341	rdev->pm.current_mclk = rdev->clock.default_mclk;
342
343}
344
345/* 10 khz */
346static uint32_t calc_eng_mem_clock(struct radeon_device *rdev,
347				   uint32_t req_clock,
348				   int *fb_div, int *post_div)
349{
350	struct radeon_pll *spll = &rdev->clock.spll;
351	int ref_div = spll->reference_div;
352
353	if (!ref_div)
354		ref_div =
355		    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
356		    RADEON_M_SPLL_REF_DIV_MASK;
357
358	if (req_clock < 15000) {
359		*post_div = 8;
360		req_clock *= 8;
361	} else if (req_clock < 30000) {
362		*post_div = 4;
363		req_clock *= 4;
364	} else if (req_clock < 60000) {
365		*post_div = 2;
366		req_clock *= 2;
367	} else
368		*post_div = 1;
369
370	req_clock *= ref_div;
371	req_clock += spll->reference_freq;
372	req_clock /= (2 * spll->reference_freq);
373
374	*fb_div = req_clock & 0xff;
375
376	req_clock = (req_clock & 0xffff) << 1;
377	req_clock *= spll->reference_freq;
378	req_clock /= ref_div;
379	req_clock /= *post_div;
380
381	return req_clock;
382}
383
384/* 10 khz */
385void radeon_legacy_set_engine_clock(struct radeon_device *rdev,
386				    uint32_t eng_clock)
387{
388	uint32_t tmp;
389	int fb_div, post_div;
390
391	/* XXX: wait for idle */
392
393	eng_clock = calc_eng_mem_clock(rdev, eng_clock, &fb_div, &post_div);
394
395	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
396	tmp &= ~RADEON_DONT_USE_XTALIN;
397	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
398
399	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
400	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
401	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
402
403	udelay(10);
404
405	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
406	tmp |= RADEON_SPLL_SLEEP;
407	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
408
409	udelay(2);
410
411	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
412	tmp |= RADEON_SPLL_RESET;
413	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
414
415	udelay(200);
416
417	tmp = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
418	tmp &= ~(RADEON_SPLL_FB_DIV_MASK << RADEON_SPLL_FB_DIV_SHIFT);
419	tmp |= (fb_div & RADEON_SPLL_FB_DIV_MASK) << RADEON_SPLL_FB_DIV_SHIFT;
420	WREG32_PLL(RADEON_M_SPLL_REF_FB_DIV, tmp);
421
422	/* XXX: verify on different asics */
423	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
424	tmp &= ~RADEON_SPLL_PVG_MASK;
425	if ((eng_clock * post_div) >= 90000)
426		tmp |= (0x7 << RADEON_SPLL_PVG_SHIFT);
427	else
428		tmp |= (0x4 << RADEON_SPLL_PVG_SHIFT);
429	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
430
431	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
432	tmp &= ~RADEON_SPLL_SLEEP;
433	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
434
435	udelay(2);
436
437	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
438	tmp &= ~RADEON_SPLL_RESET;
439	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
440
441	udelay(200);
442
443	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
444	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
445	switch (post_div) {
446	case 1:
447	default:
448		tmp |= 1;
449		break;
450	case 2:
451		tmp |= 2;
452		break;
453	case 4:
454		tmp |= 3;
455		break;
456	case 8:
457		tmp |= 4;
458		break;
459	}
460	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
461
462	udelay(20);
463
464	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
465	tmp |= RADEON_DONT_USE_XTALIN;
466	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
467
468	udelay(10);
469}
470
471void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable)
472{
473	uint32_t tmp;
474
475	if (enable) {
476		if (rdev->flags & RADEON_SINGLE_CRTC) {
477			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
478			if ((RREG32(RADEON_CONFIG_CNTL) &
479			     RADEON_CFG_ATI_REV_ID_MASK) >
480			    RADEON_CFG_ATI_REV_A13) {
481				tmp &=
482				    ~(RADEON_SCLK_FORCE_CP |
483				      RADEON_SCLK_FORCE_RB);
484			}
485			tmp &=
486			    ~(RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1 |
487			      RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_SE |
488			      RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE |
489			      RADEON_SCLK_FORCE_PB | RADEON_SCLK_FORCE_TAM |
490			      RADEON_SCLK_FORCE_TDM);
491			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
492		} else if (ASIC_IS_R300(rdev)) {
493			if ((rdev->family == CHIP_RS400) ||
494			    (rdev->family == CHIP_RS480)) {
495				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
496				tmp &=
497				    ~(RADEON_SCLK_FORCE_DISP2 |
498				      RADEON_SCLK_FORCE_CP |
499				      RADEON_SCLK_FORCE_HDP |
500				      RADEON_SCLK_FORCE_DISP1 |
501				      RADEON_SCLK_FORCE_TOP |
502				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
503				      | RADEON_SCLK_FORCE_IDCT |
504				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
505				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
506				      | R300_SCLK_FORCE_US |
507				      RADEON_SCLK_FORCE_TV_SCLK |
508				      R300_SCLK_FORCE_SU |
509				      RADEON_SCLK_FORCE_OV0);
510				tmp |= RADEON_DYN_STOP_LAT_MASK;
511				tmp |=
512				    RADEON_SCLK_FORCE_TOP |
513				    RADEON_SCLK_FORCE_VIP;
514				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
515
516				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
517				tmp &= ~RADEON_SCLK_MORE_FORCEON;
518				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
519				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
520
521				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
522				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
523					RADEON_PIXCLK_DAC_ALWAYS_ONb);
524				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
525
526				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
527				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
528					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
529					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
530					R300_DVOCLK_ALWAYS_ONb |
531					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
532					RADEON_PIXCLK_GV_ALWAYS_ONb |
533					R300_PIXCLK_DVO_ALWAYS_ONb |
534					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
535					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
536					R300_PIXCLK_TRANS_ALWAYS_ONb |
537					R300_PIXCLK_TVO_ALWAYS_ONb |
538					R300_P2G2CLK_ALWAYS_ONb |
539					R300_P2G2CLK_DAC_ALWAYS_ONb);
540				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
541			} else if (rdev->family >= CHIP_RV350) {
542				tmp = RREG32_PLL(R300_SCLK_CNTL2);
543				tmp &= ~(R300_SCLK_FORCE_TCL |
544					 R300_SCLK_FORCE_GA |
545					 R300_SCLK_FORCE_CBA);
546				tmp |= (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
547					R300_SCLK_GA_MAX_DYN_STOP_LAT |
548					R300_SCLK_CBA_MAX_DYN_STOP_LAT);
549				WREG32_PLL(R300_SCLK_CNTL2, tmp);
550
551				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
552				tmp &=
553				    ~(RADEON_SCLK_FORCE_DISP2 |
554				      RADEON_SCLK_FORCE_CP |
555				      RADEON_SCLK_FORCE_HDP |
556				      RADEON_SCLK_FORCE_DISP1 |
557				      RADEON_SCLK_FORCE_TOP |
558				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
559				      | RADEON_SCLK_FORCE_IDCT |
560				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
561				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
562				      | R300_SCLK_FORCE_US |
563				      RADEON_SCLK_FORCE_TV_SCLK |
564				      R300_SCLK_FORCE_SU |
565				      RADEON_SCLK_FORCE_OV0);
566				tmp |= RADEON_DYN_STOP_LAT_MASK;
567				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
568
569				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
570				tmp &= ~RADEON_SCLK_MORE_FORCEON;
571				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
572				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
573
574				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
575				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
576					RADEON_PIXCLK_DAC_ALWAYS_ONb);
577				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
578
579				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
580				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
581					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
582					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
583					R300_DVOCLK_ALWAYS_ONb |
584					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
585					RADEON_PIXCLK_GV_ALWAYS_ONb |
586					R300_PIXCLK_DVO_ALWAYS_ONb |
587					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
588					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
589					R300_PIXCLK_TRANS_ALWAYS_ONb |
590					R300_PIXCLK_TVO_ALWAYS_ONb |
591					R300_P2G2CLK_ALWAYS_ONb |
592					R300_P2G2CLK_DAC_ALWAYS_ONb);
593				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
594
595				tmp = RREG32_PLL(RADEON_MCLK_MISC);
596				tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
597					RADEON_IO_MCLK_DYN_ENABLE);
598				WREG32_PLL(RADEON_MCLK_MISC, tmp);
599
600				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
601				tmp |= (RADEON_FORCEON_MCLKA |
602					RADEON_FORCEON_MCLKB);
603
604				tmp &= ~(RADEON_FORCEON_YCLKA |
605					 RADEON_FORCEON_YCLKB |
606					 RADEON_FORCEON_MC);
607
608				/* Some releases of vbios have set DISABLE_MC_MCLKA
609				   and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
610				   bits will cause H/W hang when reading video memory with dynamic clocking
611				   enabled. */
612				if ((tmp & R300_DISABLE_MC_MCLKA) &&
613				    (tmp & R300_DISABLE_MC_MCLKB)) {
614					/* If both bits are set, then check the active channels */
615					tmp = RREG32_PLL(RADEON_MCLK_CNTL);
616					if (rdev->mc.vram_width == 64) {
617						if (RREG32(RADEON_MEM_CNTL) &
618						    R300_MEM_USE_CD_CH_ONLY)
619							tmp &=
620							    ~R300_DISABLE_MC_MCLKB;
621						else
622							tmp &=
623							    ~R300_DISABLE_MC_MCLKA;
624					} else {
625						tmp &= ~(R300_DISABLE_MC_MCLKA |
626							 R300_DISABLE_MC_MCLKB);
627					}
628				}
629
630				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
631			} else {
632				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
633				tmp &= ~(R300_SCLK_FORCE_VAP);
634				tmp |= RADEON_SCLK_FORCE_CP;
635				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
636				udelay(15000);
637
638				tmp = RREG32_PLL(R300_SCLK_CNTL2);
639				tmp &= ~(R300_SCLK_FORCE_TCL |
640					 R300_SCLK_FORCE_GA |
641					 R300_SCLK_FORCE_CBA);
642				WREG32_PLL(R300_SCLK_CNTL2, tmp);
643			}
644		} else {
645			tmp = RREG32_PLL(RADEON_CLK_PWRMGT_CNTL);
646
647			tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK |
648				 RADEON_DISP_DYN_STOP_LAT_MASK |
649				 RADEON_DYN_STOP_MODE_MASK);
650
651			tmp |= (RADEON_ENGIN_DYNCLK_MODE |
652				(0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
653			WREG32_PLL(RADEON_CLK_PWRMGT_CNTL, tmp);
654			udelay(15000);
655
656			tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
657			tmp |= RADEON_SCLK_DYN_START_CNTL;
658			WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
659			udelay(15000);
660
661			/* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
662			   to lockup randomly, leave them as set by BIOS.
663			 */
664			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
665			/*tmp &= RADEON_SCLK_SRC_SEL_MASK; */
666			tmp &= ~RADEON_SCLK_FORCEON_MASK;
667
668			/*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300 */
669			if (((rdev->family == CHIP_RV250) &&
670			     ((RREG32(RADEON_CONFIG_CNTL) &
671			       RADEON_CFG_ATI_REV_ID_MASK) <
672			      RADEON_CFG_ATI_REV_A13))
673			    || ((rdev->family == CHIP_RV100)
674				&&
675				((RREG32(RADEON_CONFIG_CNTL) &
676				  RADEON_CFG_ATI_REV_ID_MASK) <=
677				 RADEON_CFG_ATI_REV_A13))) {
678				tmp |= RADEON_SCLK_FORCE_CP;
679				tmp |= RADEON_SCLK_FORCE_VIP;
680			}
681
682			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
683
684			if ((rdev->family == CHIP_RV200) ||
685			    (rdev->family == CHIP_RV250) ||
686			    (rdev->family == CHIP_RV280)) {
687				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
688				tmp &= ~RADEON_SCLK_MORE_FORCEON;
689
690				/* RV200::A11 A12 RV250::A11 A12 */
691				if (((rdev->family == CHIP_RV200) ||
692				     (rdev->family == CHIP_RV250)) &&
693				    ((RREG32(RADEON_CONFIG_CNTL) &
694				      RADEON_CFG_ATI_REV_ID_MASK) <
695				     RADEON_CFG_ATI_REV_A13)) {
696					tmp |= RADEON_SCLK_MORE_FORCEON;
697				}
698				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
699				udelay(15000);
700			}
701
702			/* RV200::A11 A12, RV250::A11 A12 */
703			if (((rdev->family == CHIP_RV200) ||
704			     (rdev->family == CHIP_RV250)) &&
705			    ((RREG32(RADEON_CONFIG_CNTL) &
706			      RADEON_CFG_ATI_REV_ID_MASK) <
707			     RADEON_CFG_ATI_REV_A13)) {
708				tmp = RREG32_PLL(RADEON_PLL_PWRMGT_CNTL);
709				tmp |= RADEON_TCL_BYPASS_DISABLE;
710				WREG32_PLL(RADEON_PLL_PWRMGT_CNTL, tmp);
711			}
712			udelay(15000);
713
714			/*enable dynamic mode for display clocks (PIXCLK and PIX2CLK) */
715			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
716			tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
717				RADEON_PIX2CLK_DAC_ALWAYS_ONb |
718				RADEON_PIXCLK_BLEND_ALWAYS_ONb |
719				RADEON_PIXCLK_GV_ALWAYS_ONb |
720				RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
721				RADEON_PIXCLK_LVDS_ALWAYS_ONb |
722				RADEON_PIXCLK_TMDS_ALWAYS_ONb);
723
724			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
725			udelay(15000);
726
727			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
728			tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
729				RADEON_PIXCLK_DAC_ALWAYS_ONb);
730
731			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
732			udelay(15000);
733		}
734	} else {
735		/* Turn everything OFF (ForceON to everything) */
736		if (rdev->flags & RADEON_SINGLE_CRTC) {
737			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
738			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_HDP |
739				RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP
740				| RADEON_SCLK_FORCE_E2 | RADEON_SCLK_FORCE_SE |
741				RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
742				RADEON_SCLK_FORCE_RE | RADEON_SCLK_FORCE_PB |
743				RADEON_SCLK_FORCE_TAM | RADEON_SCLK_FORCE_TDM |
744				RADEON_SCLK_FORCE_RB);
745			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
746		} else if ((rdev->family == CHIP_RS400) ||
747			   (rdev->family == CHIP_RS480)) {
748			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
749			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
750				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
751				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
752				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
753				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
754				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
755				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
756				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
757			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
758
759			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
760			tmp |= RADEON_SCLK_MORE_FORCEON;
761			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
762
763			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
764			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
765				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
766				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
767			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
768
769			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
770			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
771				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
772				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
773				 R300_DVOCLK_ALWAYS_ONb |
774				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
775				 RADEON_PIXCLK_GV_ALWAYS_ONb |
776				 R300_PIXCLK_DVO_ALWAYS_ONb |
777				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
778				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
779				 R300_PIXCLK_TRANS_ALWAYS_ONb |
780				 R300_PIXCLK_TVO_ALWAYS_ONb |
781				 R300_P2G2CLK_ALWAYS_ONb |
782				 R300_P2G2CLK_DAC_ALWAYS_ONb |
783				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
784			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
785		} else if (rdev->family >= CHIP_RV350) {
786			/* for RV350/M10, no delays are required. */
787			tmp = RREG32_PLL(R300_SCLK_CNTL2);
788			tmp |= (R300_SCLK_FORCE_TCL |
789				R300_SCLK_FORCE_GA | R300_SCLK_FORCE_CBA);
790			WREG32_PLL(R300_SCLK_CNTL2, tmp);
791
792			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
793			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
794				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
795				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
796				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
797				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
798				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
799				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
800				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
801			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
802
803			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
804			tmp |= RADEON_SCLK_MORE_FORCEON;
805			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
806
807			tmp = RREG32_PLL(RADEON_MCLK_CNTL);
808			tmp |= (RADEON_FORCEON_MCLKA |
809				RADEON_FORCEON_MCLKB |
810				RADEON_FORCEON_YCLKA |
811				RADEON_FORCEON_YCLKB | RADEON_FORCEON_MC);
812			WREG32_PLL(RADEON_MCLK_CNTL, tmp);
813
814			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
815			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
816				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
817				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
818			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
819
820			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
821			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
822				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
823				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
824				 R300_DVOCLK_ALWAYS_ONb |
825				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
826				 RADEON_PIXCLK_GV_ALWAYS_ONb |
827				 R300_PIXCLK_DVO_ALWAYS_ONb |
828				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
829				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
830				 R300_PIXCLK_TRANS_ALWAYS_ONb |
831				 R300_PIXCLK_TVO_ALWAYS_ONb |
832				 R300_P2G2CLK_ALWAYS_ONb |
833				 R300_P2G2CLK_DAC_ALWAYS_ONb |
834				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
835			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
836		} else {
837			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
838			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
839			tmp |= RADEON_SCLK_FORCE_SE;
840
841			if (rdev->flags & RADEON_SINGLE_CRTC) {
842				tmp |= (RADEON_SCLK_FORCE_RB |
843					RADEON_SCLK_FORCE_TDM |
844					RADEON_SCLK_FORCE_TAM |
845					RADEON_SCLK_FORCE_PB |
846					RADEON_SCLK_FORCE_RE |
847					RADEON_SCLK_FORCE_VIP |
848					RADEON_SCLK_FORCE_IDCT |
849					RADEON_SCLK_FORCE_TOP |
850					RADEON_SCLK_FORCE_DISP1 |
851					RADEON_SCLK_FORCE_DISP2 |
852					RADEON_SCLK_FORCE_HDP);
853			} else if ((rdev->family == CHIP_R300) ||
854				   (rdev->family == CHIP_R350)) {
855				tmp |= (RADEON_SCLK_FORCE_HDP |
856					RADEON_SCLK_FORCE_DISP1 |
857					RADEON_SCLK_FORCE_DISP2 |
858					RADEON_SCLK_FORCE_TOP |
859					RADEON_SCLK_FORCE_IDCT |
860					RADEON_SCLK_FORCE_VIP);
861			}
862			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
863
864			udelay(16000);
865
866			if ((rdev->family == CHIP_R300) ||
867			    (rdev->family == CHIP_R350)) {
868				tmp = RREG32_PLL(R300_SCLK_CNTL2);
869				tmp |= (R300_SCLK_FORCE_TCL |
870					R300_SCLK_FORCE_GA |
871					R300_SCLK_FORCE_CBA);
872				WREG32_PLL(R300_SCLK_CNTL2, tmp);
873				udelay(16000);
874			}
875
876			if (rdev->flags & RADEON_IS_IGP) {
877				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
878				tmp &= ~(RADEON_FORCEON_MCLKA |
879					 RADEON_FORCEON_YCLKA);
880				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
881				udelay(16000);
882			}
883
884			if ((rdev->family == CHIP_RV200) ||
885			    (rdev->family == CHIP_RV250) ||
886			    (rdev->family == CHIP_RV280)) {
887				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
888				tmp |= RADEON_SCLK_MORE_FORCEON;
889				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
890				udelay(16000);
891			}
892
893			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
894			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
895				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
896				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
897				 RADEON_PIXCLK_GV_ALWAYS_ONb |
898				 RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
899				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
900				 RADEON_PIXCLK_TMDS_ALWAYS_ONb);
901
902			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
903			udelay(16000);
904
905			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
906			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
907				 RADEON_PIXCLK_DAC_ALWAYS_ONb);
908			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
909		}
910	}
911}
912