Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "i915_drv.h"
7#include "i915_irq.h"
8#include "i915_reg.h"
9#include "intel_backlight_regs.h"
10#include "intel_combo_phy.h"
11#include "intel_combo_phy_regs.h"
12#include "intel_crt.h"
13#include "intel_de.h"
14#include "intel_display_irq.h"
15#include "intel_display_power_well.h"
16#include "intel_display_types.h"
17#include "intel_dkl_phy.h"
18#include "intel_dkl_phy_regs.h"
19#include "intel_dmc.h"
20#include "intel_dp_aux_regs.h"
21#include "intel_dpio_phy.h"
22#include "intel_dpll.h"
23#include "intel_hotplug.h"
24#include "intel_pcode.h"
25#include "intel_pps.h"
26#include "intel_tc.h"
27#include "intel_vga.h"
28#include "skl_watermark.h"
29#include "vlv_sideband.h"
30#include "vlv_sideband_reg.h"
31
32struct i915_power_well_regs {
33 i915_reg_t bios;
34 i915_reg_t driver;
35 i915_reg_t kvmr;
36 i915_reg_t debug;
37};
38
39struct i915_power_well_ops {
40 const struct i915_power_well_regs *regs;
41 /*
42 * Synchronize the well's hw state to match the current sw state, for
43 * example enable/disable it based on the current refcount. Called
44 * during driver init and resume time, possibly after first calling
45 * the enable/disable handlers.
46 */
47 void (*sync_hw)(struct drm_i915_private *i915,
48 struct i915_power_well *power_well);
49 /*
50 * Enable the well and resources that depend on it (for example
51 * interrupts located on the well). Called after the 0->1 refcount
52 * transition.
53 */
54 void (*enable)(struct drm_i915_private *i915,
55 struct i915_power_well *power_well);
56 /*
57 * Disable the well and resources that depend on it. Called after
58 * the 1->0 refcount transition.
59 */
60 void (*disable)(struct drm_i915_private *i915,
61 struct i915_power_well *power_well);
62 /* Returns the hw enabled state. */
63 bool (*is_enabled)(struct drm_i915_private *i915,
64 struct i915_power_well *power_well);
65};
66
67static const struct i915_power_well_instance *
68i915_power_well_instance(const struct i915_power_well *power_well)
69{
70 return &power_well->desc->instances->list[power_well->instance_idx];
71}
72
73struct i915_power_well *
74lookup_power_well(struct drm_i915_private *i915,
75 enum i915_power_well_id power_well_id)
76{
77 struct i915_power_well *power_well;
78
79 for_each_power_well(i915, power_well)
80 if (i915_power_well_instance(power_well)->id == power_well_id)
81 return power_well;
82
83 /*
84 * It's not feasible to add error checking code to the callers since
85 * this condition really shouldn't happen and it doesn't even make sense
86 * to abort things like display initialization sequences. Just return
87 * the first power well and hope the WARN gets reported so we can fix
88 * our driver.
89 */
90 drm_WARN(&i915->drm, 1,
91 "Power well %d not defined for this platform\n",
92 power_well_id);
93 return &i915->display.power.domains.power_wells[0];
94}
95
96void intel_power_well_enable(struct drm_i915_private *i915,
97 struct i915_power_well *power_well)
98{
99 drm_dbg_kms(&i915->drm, "enabling %s\n", intel_power_well_name(power_well));
100 power_well->desc->ops->enable(i915, power_well);
101 power_well->hw_enabled = true;
102}
103
104void intel_power_well_disable(struct drm_i915_private *i915,
105 struct i915_power_well *power_well)
106{
107 drm_dbg_kms(&i915->drm, "disabling %s\n", intel_power_well_name(power_well));
108 power_well->hw_enabled = false;
109 power_well->desc->ops->disable(i915, power_well);
110}
111
112void intel_power_well_sync_hw(struct drm_i915_private *i915,
113 struct i915_power_well *power_well)
114{
115 power_well->desc->ops->sync_hw(i915, power_well);
116 power_well->hw_enabled =
117 power_well->desc->ops->is_enabled(i915, power_well);
118}
119
120void intel_power_well_get(struct drm_i915_private *i915,
121 struct i915_power_well *power_well)
122{
123 if (!power_well->count++)
124 intel_power_well_enable(i915, power_well);
125}
126
127void intel_power_well_put(struct drm_i915_private *i915,
128 struct i915_power_well *power_well)
129{
130 drm_WARN(&i915->drm, !power_well->count,
131 "Use count on power well %s is already zero",
132 i915_power_well_instance(power_well)->name);
133
134 if (!--power_well->count)
135 intel_power_well_disable(i915, power_well);
136}
137
138bool intel_power_well_is_enabled(struct drm_i915_private *i915,
139 struct i915_power_well *power_well)
140{
141 return power_well->desc->ops->is_enabled(i915, power_well);
142}
143
144bool intel_power_well_is_enabled_cached(struct i915_power_well *power_well)
145{
146 return power_well->hw_enabled;
147}
148
149bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
150 enum i915_power_well_id power_well_id)
151{
152 struct i915_power_well *power_well;
153
154 power_well = lookup_power_well(dev_priv, power_well_id);
155
156 return intel_power_well_is_enabled(dev_priv, power_well);
157}
158
159bool intel_power_well_is_always_on(struct i915_power_well *power_well)
160{
161 return power_well->desc->always_on;
162}
163
164const char *intel_power_well_name(struct i915_power_well *power_well)
165{
166 return i915_power_well_instance(power_well)->name;
167}
168
169struct intel_power_domain_mask *intel_power_well_domains(struct i915_power_well *power_well)
170{
171 return &power_well->domains;
172}
173
174int intel_power_well_refcount(struct i915_power_well *power_well)
175{
176 return power_well->count;
177}
178
179/*
180 * Starting with Haswell, we have a "Power Down Well" that can be turned off
181 * when not needed anymore. We have 4 registers that can request the power well
182 * to be enabled, and it will only be disabled if none of the registers is
183 * requesting it to be enabled.
184 */
185static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
186 u8 irq_pipe_mask, bool has_vga)
187{
188 if (has_vga)
189 intel_vga_reset_io_mem(dev_priv);
190
191 if (irq_pipe_mask)
192 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
193}
194
195static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
196 u8 irq_pipe_mask)
197{
198 if (irq_pipe_mask)
199 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
200}
201
202#define ICL_AUX_PW_TO_CH(pw_idx) \
203 ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
204
205#define ICL_TBT_AUX_PW_TO_CH(pw_idx) \
206 ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C)
207
208static enum aux_ch icl_aux_pw_to_ch(const struct i915_power_well *power_well)
209{
210 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
211
212 return power_well->desc->is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) :
213 ICL_AUX_PW_TO_CH(pw_idx);
214}
215
216static struct intel_digital_port *
217aux_ch_to_digital_port(struct drm_i915_private *dev_priv,
218 enum aux_ch aux_ch)
219{
220 struct intel_digital_port *dig_port = NULL;
221 struct intel_encoder *encoder;
222
223 for_each_intel_encoder(&dev_priv->drm, encoder) {
224 /* We'll check the MST primary port */
225 if (encoder->type == INTEL_OUTPUT_DP_MST)
226 continue;
227
228 dig_port = enc_to_dig_port(encoder);
229 if (!dig_port)
230 continue;
231
232 if (dig_port->aux_ch != aux_ch) {
233 dig_port = NULL;
234 continue;
235 }
236
237 break;
238 }
239
240 return dig_port;
241}
242
243static enum phy icl_aux_pw_to_phy(struct drm_i915_private *i915,
244 const struct i915_power_well *power_well)
245{
246 enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
247 struct intel_digital_port *dig_port = aux_ch_to_digital_port(i915, aux_ch);
248
249 /*
250 * FIXME should we care about the (VBT defined) dig_port->aux_ch
251 * relationship or should this be purely defined by the hardware layout?
252 * Currently if the port doesn't appear in the VBT, or if it's declared
253 * as HDMI-only and routed to a combo PHY, the encoder either won't be
254 * present at all or it will not have an aux_ch assigned.
255 */
256 return dig_port ? intel_port_to_phy(i915, dig_port->base.port) : PHY_NONE;
257}
258
259static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
260 struct i915_power_well *power_well,
261 bool timeout_expected)
262{
263 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
264 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
265 int timeout = power_well->desc->enable_timeout ? : 1;
266
267 /*
268 * For some power wells we're not supposed to watch the status bit for
269 * an ack, but rather just wait a fixed amount of time and then
270 * proceed. This is only used on DG2.
271 */
272 if (IS_DG2(dev_priv) && power_well->desc->fixed_enable_delay) {
273 usleep_range(600, 1200);
274 return;
275 }
276
277 /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
278 if (intel_de_wait_for_set(dev_priv, regs->driver,
279 HSW_PWR_WELL_CTL_STATE(pw_idx), timeout)) {
280 drm_dbg_kms(&dev_priv->drm, "%s power well enable timeout\n",
281 intel_power_well_name(power_well));
282
283 drm_WARN_ON(&dev_priv->drm, !timeout_expected);
284
285 }
286}
287
288static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
289 const struct i915_power_well_regs *regs,
290 int pw_idx)
291{
292 u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
293 u32 ret;
294
295 ret = intel_de_read(dev_priv, regs->bios) & req_mask ? 1 : 0;
296 ret |= intel_de_read(dev_priv, regs->driver) & req_mask ? 2 : 0;
297 if (regs->kvmr.reg)
298 ret |= intel_de_read(dev_priv, regs->kvmr) & req_mask ? 4 : 0;
299 ret |= intel_de_read(dev_priv, regs->debug) & req_mask ? 8 : 0;
300
301 return ret;
302}
303
304static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
305 struct i915_power_well *power_well)
306{
307 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
308 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
309 bool disabled;
310 u32 reqs;
311
312 /*
313 * Bspec doesn't require waiting for PWs to get disabled, but still do
314 * this for paranoia. The known cases where a PW will be forced on:
315 * - a KVMR request on any power well via the KVMR request register
316 * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
317 * DEBUG request registers
318 * Skip the wait in case any of the request bits are set and print a
319 * diagnostic message.
320 */
321 wait_for((disabled = !(intel_de_read(dev_priv, regs->driver) &
322 HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
323 (reqs = hsw_power_well_requesters(dev_priv, regs, pw_idx)), 1);
324 if (disabled)
325 return;
326
327 drm_dbg_kms(&dev_priv->drm,
328 "%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
329 intel_power_well_name(power_well),
330 !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
331}
332
333static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
334 enum skl_power_gate pg)
335{
336 /* Timeout 5us for PG#0, for other PGs 1us */
337 drm_WARN_ON(&dev_priv->drm,
338 intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS,
339 SKL_FUSE_PG_DIST_STATUS(pg), 1));
340}
341
342static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
343 struct i915_power_well *power_well)
344{
345 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
346 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
347
348 if (power_well->desc->has_fuses) {
349 enum skl_power_gate pg;
350
351 pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
352 SKL_PW_CTL_IDX_TO_PG(pw_idx);
353
354 /* Wa_16013190616:adlp */
355 if (IS_ALDERLAKE_P(dev_priv) && pg == SKL_PG1)
356 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, 0, DISABLE_FLR_SRC);
357
358 /*
359 * For PW1 we have to wait both for the PW0/PG0 fuse state
360 * before enabling the power well and PW1/PG1's own fuse
361 * state after the enabling. For all other power wells with
362 * fuses we only have to wait for that PW/PG's fuse state
363 * after the enabling.
364 */
365 if (pg == SKL_PG1)
366 gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
367 }
368
369 intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
370
371 hsw_wait_for_power_well_enable(dev_priv, power_well, false);
372
373 if (power_well->desc->has_fuses) {
374 enum skl_power_gate pg;
375
376 pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
377 SKL_PW_CTL_IDX_TO_PG(pw_idx);
378 gen9_wait_for_power_well_fuses(dev_priv, pg);
379 }
380
381 hsw_power_well_post_enable(dev_priv,
382 power_well->desc->irq_pipe_mask,
383 power_well->desc->has_vga);
384}
385
386static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
387 struct i915_power_well *power_well)
388{
389 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
390 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
391
392 hsw_power_well_pre_disable(dev_priv,
393 power_well->desc->irq_pipe_mask);
394
395 intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
396 hsw_wait_for_power_well_disable(dev_priv, power_well);
397}
398
399static bool intel_port_is_edp(struct drm_i915_private *i915, enum port port)
400{
401 struct intel_encoder *encoder;
402
403 for_each_intel_encoder(&i915->drm, encoder) {
404 if (encoder->type == INTEL_OUTPUT_EDP &&
405 encoder->port == port)
406 return true;
407 }
408
409 return false;
410}
411
412static void
413icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
414 struct i915_power_well *power_well)
415{
416 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
417 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
418 enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
419
420 drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
421
422 intel_de_rmw(dev_priv, regs->driver, 0, HSW_PWR_WELL_CTL_REQ(pw_idx));
423
424 /* FIXME this is a mess */
425 if (phy != PHY_NONE)
426 intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(phy),
427 0, ICL_LANE_ENABLE_AUX);
428
429 hsw_wait_for_power_well_enable(dev_priv, power_well, false);
430
431 /* Display WA #1178: icl */
432 if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
433 !intel_port_is_edp(dev_priv, (enum port)phy))
434 intel_de_rmw(dev_priv, ICL_AUX_ANAOVRD1(pw_idx),
435 0, ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS);
436}
437
438static void
439icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
440 struct i915_power_well *power_well)
441{
442 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
443 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
444 enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
445
446 drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
447
448 /* FIXME this is a mess */
449 if (phy != PHY_NONE)
450 intel_de_rmw(dev_priv, ICL_PORT_CL_DW12(phy),
451 ICL_LANE_ENABLE_AUX, 0);
452
453 intel_de_rmw(dev_priv, regs->driver, HSW_PWR_WELL_CTL_REQ(pw_idx), 0);
454
455 hsw_wait_for_power_well_disable(dev_priv, power_well);
456}
457
458#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
459
460static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
461 struct i915_power_well *power_well,
462 struct intel_digital_port *dig_port)
463{
464 if (drm_WARN_ON(&dev_priv->drm, !dig_port))
465 return;
466
467 if (DISPLAY_VER(dev_priv) == 11 && intel_tc_cold_requires_aux_pw(dig_port))
468 return;
469
470 drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port));
471}
472
473#else
474
475static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
476 struct i915_power_well *power_well,
477 struct intel_digital_port *dig_port)
478{
479}
480
481#endif
482
483#define TGL_AUX_PW_TO_TC_PORT(pw_idx) ((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1)
484
485static void icl_tc_cold_exit(struct drm_i915_private *i915)
486{
487 int ret, tries = 0;
488
489 while (1) {
490 ret = snb_pcode_write_timeout(&i915->uncore, ICL_PCODE_EXIT_TCCOLD, 0,
491 250, 1);
492 if (ret != -EAGAIN || ++tries == 3)
493 break;
494 msleep(1);
495 }
496
497 /* Spec states that TC cold exit can take up to 1ms to complete */
498 if (!ret)
499 msleep(1);
500
501 /* TODO: turn failure into a error as soon i915 CI updates ICL IFWI */
502 drm_dbg_kms(&i915->drm, "TC cold block %s\n", ret ? "failed" :
503 "succeeded");
504}
505
506static void
507icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
508 struct i915_power_well *power_well)
509{
510 enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
511 struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
512 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
513 bool is_tbt = power_well->desc->is_tc_tbt;
514 bool timeout_expected;
515
516 icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
517
518 intel_de_rmw(dev_priv, DP_AUX_CH_CTL(aux_ch),
519 DP_AUX_CH_CTL_TBT_IO, is_tbt ? DP_AUX_CH_CTL_TBT_IO : 0);
520
521 intel_de_rmw(dev_priv, regs->driver,
522 0,
523 HSW_PWR_WELL_CTL_REQ(i915_power_well_instance(power_well)->hsw.idx));
524
525 /*
526 * An AUX timeout is expected if the TBT DP tunnel is down,
527 * or need to enable AUX on a legacy TypeC port as part of the TC-cold
528 * exit sequence.
529 */
530 timeout_expected = is_tbt || intel_tc_cold_requires_aux_pw(dig_port);
531 if (DISPLAY_VER(dev_priv) == 11 && intel_tc_cold_requires_aux_pw(dig_port))
532 icl_tc_cold_exit(dev_priv);
533
534 hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected);
535
536 if (DISPLAY_VER(dev_priv) >= 12 && !is_tbt) {
537 enum tc_port tc_port;
538
539 tc_port = TGL_AUX_PW_TO_TC_PORT(i915_power_well_instance(power_well)->hsw.idx);
540
541 if (wait_for(intel_dkl_phy_read(dev_priv, DKL_CMN_UC_DW_27(tc_port)) &
542 DKL_CMN_UC_DW27_UC_HEALTH, 1))
543 drm_warn(&dev_priv->drm,
544 "Timeout waiting TC uC health\n");
545 }
546}
547
548static void
549icl_aux_power_well_enable(struct drm_i915_private *dev_priv,
550 struct i915_power_well *power_well)
551{
552 enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
553
554 if (intel_phy_is_tc(dev_priv, phy))
555 return icl_tc_phy_aux_power_well_enable(dev_priv, power_well);
556 else if (IS_ICELAKE(dev_priv))
557 return icl_combo_phy_aux_power_well_enable(dev_priv,
558 power_well);
559 else
560 return hsw_power_well_enable(dev_priv, power_well);
561}
562
563static void
564icl_aux_power_well_disable(struct drm_i915_private *dev_priv,
565 struct i915_power_well *power_well)
566{
567 enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
568
569 if (intel_phy_is_tc(dev_priv, phy))
570 return hsw_power_well_disable(dev_priv, power_well);
571 else if (IS_ICELAKE(dev_priv))
572 return icl_combo_phy_aux_power_well_disable(dev_priv,
573 power_well);
574 else
575 return hsw_power_well_disable(dev_priv, power_well);
576}
577
578/*
579 * We should only use the power well if we explicitly asked the hardware to
580 * enable it, so check if it's enabled and also check if we've requested it to
581 * be enabled.
582 */
583static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
584 struct i915_power_well *power_well)
585{
586 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
587 enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
588 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
589 u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
590 HSW_PWR_WELL_CTL_STATE(pw_idx);
591 u32 val;
592
593 val = intel_de_read(dev_priv, regs->driver);
594
595 /*
596 * On GEN9 big core due to a DMC bug the driver's request bits for PW1
597 * and the MISC_IO PW will be not restored, so check instead for the
598 * BIOS's own request bits, which are forced-on for these power wells
599 * when exiting DC5/6.
600 */
601 if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv) &&
602 (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO))
603 val |= intel_de_read(dev_priv, regs->bios);
604
605 return (val & mask) == mask;
606}
607
608static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
609{
610 drm_WARN_ONCE(&dev_priv->drm,
611 (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC9),
612 "DC9 already programmed to be enabled.\n");
613 drm_WARN_ONCE(&dev_priv->drm,
614 intel_de_read(dev_priv, DC_STATE_EN) &
615 DC_STATE_EN_UPTO_DC5,
616 "DC5 still not disabled to enable DC9.\n");
617 drm_WARN_ONCE(&dev_priv->drm,
618 intel_de_read(dev_priv, HSW_PWR_WELL_CTL2) &
619 HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
620 "Power well 2 on.\n");
621 drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
622 "Interrupts not disabled yet.\n");
623
624 /*
625 * TODO: check for the following to verify the conditions to enter DC9
626 * state are satisfied:
627 * 1] Check relevant display engine registers to verify if mode set
628 * disable sequence was followed.
629 * 2] Check if display uninitialize sequence is initialized.
630 */
631}
632
633static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
634{
635 drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
636 "Interrupts not disabled yet.\n");
637 drm_WARN_ONCE(&dev_priv->drm,
638 intel_de_read(dev_priv, DC_STATE_EN) &
639 DC_STATE_EN_UPTO_DC5,
640 "DC5 still not disabled.\n");
641
642 /*
643 * TODO: check for the following to verify DC9 state was indeed
644 * entered before programming to disable it:
645 * 1] Check relevant display engine registers to verify if mode
646 * set disable sequence was followed.
647 * 2] Check if display uninitialize sequence is initialized.
648 */
649}
650
651static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
652 u32 state)
653{
654 int rewrites = 0;
655 int rereads = 0;
656 u32 v;
657
658 intel_de_write(dev_priv, DC_STATE_EN, state);
659
660 /* It has been observed that disabling the dc6 state sometimes
661 * doesn't stick and dmc keeps returning old value. Make sure
662 * the write really sticks enough times and also force rewrite until
663 * we are confident that state is exactly what we want.
664 */
665 do {
666 v = intel_de_read(dev_priv, DC_STATE_EN);
667
668 if (v != state) {
669 intel_de_write(dev_priv, DC_STATE_EN, state);
670 rewrites++;
671 rereads = 0;
672 } else if (rereads++ > 5) {
673 break;
674 }
675
676 } while (rewrites < 100);
677
678 if (v != state)
679 drm_err(&dev_priv->drm,
680 "Writing dc state to 0x%x failed, now 0x%x\n",
681 state, v);
682
683 /* Most of the times we need one retry, avoid spam */
684 if (rewrites > 1)
685 drm_dbg_kms(&dev_priv->drm,
686 "Rewrote dc state to 0x%x %d times\n",
687 state, rewrites);
688}
689
690static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
691{
692 u32 mask;
693
694 mask = DC_STATE_EN_UPTO_DC5;
695
696 if (DISPLAY_VER(dev_priv) >= 12)
697 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
698 | DC_STATE_EN_DC9;
699 else if (DISPLAY_VER(dev_priv) == 11)
700 mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
701 else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
702 mask |= DC_STATE_EN_DC9;
703 else
704 mask |= DC_STATE_EN_UPTO_DC6;
705
706 return mask;
707}
708
709void gen9_sanitize_dc_state(struct drm_i915_private *i915)
710{
711 struct i915_power_domains *power_domains = &i915->display.power.domains;
712 u32 val;
713
714 if (!HAS_DISPLAY(i915))
715 return;
716
717 val = intel_de_read(i915, DC_STATE_EN) & gen9_dc_mask(i915);
718
719 drm_dbg_kms(&i915->drm,
720 "Resetting DC state tracking from %02x to %02x\n",
721 power_domains->dc_state, val);
722 power_domains->dc_state = val;
723}
724
725/**
726 * gen9_set_dc_state - set target display C power state
727 * @dev_priv: i915 device instance
728 * @state: target DC power state
729 * - DC_STATE_DISABLE
730 * - DC_STATE_EN_UPTO_DC5
731 * - DC_STATE_EN_UPTO_DC6
732 * - DC_STATE_EN_DC9
733 *
734 * Signal to DMC firmware/HW the target DC power state passed in @state.
735 * DMC/HW can turn off individual display clocks and power rails when entering
736 * a deeper DC power state (higher in number) and turns these back when exiting
737 * that state to a shallower power state (lower in number). The HW will decide
738 * when to actually enter a given state on an on-demand basis, for instance
739 * depending on the active state of display pipes. The state of display
740 * registers backed by affected power rails are saved/restored as needed.
741 *
742 * Based on the above enabling a deeper DC power state is asynchronous wrt.
743 * enabling it. Disabling a deeper power state is synchronous: for instance
744 * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
745 * back on and register state is restored. This is guaranteed by the MMIO write
746 * to DC_STATE_EN blocking until the state is restored.
747 */
748void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
749{
750 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
751 u32 val;
752 u32 mask;
753
754 if (!HAS_DISPLAY(dev_priv))
755 return;
756
757 if (drm_WARN_ON_ONCE(&dev_priv->drm,
758 state & ~power_domains->allowed_dc_mask))
759 state &= power_domains->allowed_dc_mask;
760
761 val = intel_de_read(dev_priv, DC_STATE_EN);
762 mask = gen9_dc_mask(dev_priv);
763 drm_dbg_kms(&dev_priv->drm, "Setting DC state from %02x to %02x\n",
764 val & mask, state);
765
766 /* Check if DMC is ignoring our DC state requests */
767 if ((val & mask) != power_domains->dc_state)
768 drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n",
769 power_domains->dc_state, val & mask);
770
771 val &= ~mask;
772 val |= state;
773
774 gen9_write_dc_state(dev_priv, val);
775
776 power_domains->dc_state = val & mask;
777}
778
779static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
780{
781 drm_dbg_kms(&dev_priv->drm, "Enabling DC3CO\n");
782 gen9_set_dc_state(dev_priv, DC_STATE_EN_DC3CO);
783}
784
785static void tgl_disable_dc3co(struct drm_i915_private *dev_priv)
786{
787 drm_dbg_kms(&dev_priv->drm, "Disabling DC3CO\n");
788 intel_de_rmw(dev_priv, DC_STATE_EN, DC_STATE_DC3CO_STATUS, 0);
789 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
790 /*
791 * Delay of 200us DC3CO Exit time B.Spec 49196
792 */
793 usleep_range(200, 210);
794}
795
796static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
797{
798 enum i915_power_well_id high_pg;
799
800 /* Power wells at this level and above must be disabled for DC5 entry */
801 if (DISPLAY_VER(dev_priv) == 12)
802 high_pg = ICL_DISP_PW_3;
803 else
804 high_pg = SKL_DISP_PW_2;
805
806 drm_WARN_ONCE(&dev_priv->drm,
807 intel_display_power_well_is_enabled(dev_priv, high_pg),
808 "Power wells above platform's DC5 limit still enabled.\n");
809
810 drm_WARN_ONCE(&dev_priv->drm,
811 (intel_de_read(dev_priv, DC_STATE_EN) &
812 DC_STATE_EN_UPTO_DC5),
813 "DC5 already programmed to be enabled.\n");
814 assert_rpm_wakelock_held(&dev_priv->runtime_pm);
815
816 assert_dmc_loaded(dev_priv);
817}
818
819void gen9_enable_dc5(struct drm_i915_private *dev_priv)
820{
821 assert_can_enable_dc5(dev_priv);
822
823 drm_dbg_kms(&dev_priv->drm, "Enabling DC5\n");
824
825 /* Wa Display #1183: skl,kbl,cfl */
826 if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
827 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
828 0, SKL_SELECT_ALTERNATE_DC_EXIT);
829
830 gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
831}
832
833static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
834{
835 drm_WARN_ONCE(&dev_priv->drm,
836 (intel_de_read(dev_priv, UTIL_PIN_CTL) &
837 (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) ==
838 (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
839 "Utility pin enabled in PWM mode\n");
840 drm_WARN_ONCE(&dev_priv->drm,
841 (intel_de_read(dev_priv, DC_STATE_EN) &
842 DC_STATE_EN_UPTO_DC6),
843 "DC6 already programmed to be enabled.\n");
844
845 assert_dmc_loaded(dev_priv);
846}
847
848void skl_enable_dc6(struct drm_i915_private *dev_priv)
849{
850 assert_can_enable_dc6(dev_priv);
851
852 drm_dbg_kms(&dev_priv->drm, "Enabling DC6\n");
853
854 /* Wa Display #1183: skl,kbl,cfl */
855 if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
856 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
857 0, SKL_SELECT_ALTERNATE_DC_EXIT);
858
859 gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
860}
861
862void bxt_enable_dc9(struct drm_i915_private *dev_priv)
863{
864 assert_can_enable_dc9(dev_priv);
865
866 drm_dbg_kms(&dev_priv->drm, "Enabling DC9\n");
867 /*
868 * Power sequencer reset is not needed on
869 * platforms with South Display Engine on PCH,
870 * because PPS registers are always on.
871 */
872 if (!HAS_PCH_SPLIT(dev_priv))
873 intel_pps_reset_all(dev_priv);
874 gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
875}
876
877void bxt_disable_dc9(struct drm_i915_private *dev_priv)
878{
879 assert_can_disable_dc9(dev_priv);
880
881 drm_dbg_kms(&dev_priv->drm, "Disabling DC9\n");
882
883 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
884
885 intel_pps_unlock_regs_wa(dev_priv);
886}
887
888static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
889 struct i915_power_well *power_well)
890{
891 const struct i915_power_well_regs *regs = power_well->desc->ops->regs;
892 int pw_idx = i915_power_well_instance(power_well)->hsw.idx;
893 u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
894 u32 bios_req = intel_de_read(dev_priv, regs->bios);
895
896 /* Take over the request bit if set by BIOS. */
897 if (bios_req & mask) {
898 u32 drv_req = intel_de_read(dev_priv, regs->driver);
899
900 if (!(drv_req & mask))
901 intel_de_write(dev_priv, regs->driver, drv_req | mask);
902 intel_de_write(dev_priv, regs->bios, bios_req & ~mask);
903 }
904}
905
906static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
907 struct i915_power_well *power_well)
908{
909 bxt_ddi_phy_init(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
910}
911
912static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
913 struct i915_power_well *power_well)
914{
915 bxt_ddi_phy_uninit(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
916}
917
918static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
919 struct i915_power_well *power_well)
920{
921 return bxt_ddi_phy_is_enabled(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
922}
923
924static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
925{
926 struct i915_power_well *power_well;
927
928 power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
929 if (intel_power_well_refcount(power_well) > 0)
930 bxt_ddi_phy_verify_state(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
931
932 power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
933 if (intel_power_well_refcount(power_well) > 0)
934 bxt_ddi_phy_verify_state(dev_priv, i915_power_well_instance(power_well)->bxt.phy);
935
936 if (IS_GEMINILAKE(dev_priv)) {
937 power_well = lookup_power_well(dev_priv,
938 GLK_DISP_PW_DPIO_CMN_C);
939 if (intel_power_well_refcount(power_well) > 0)
940 bxt_ddi_phy_verify_state(dev_priv,
941 i915_power_well_instance(power_well)->bxt.phy);
942 }
943}
944
945static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
946 struct i915_power_well *power_well)
947{
948 return ((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
949 (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
950}
951
952static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
953{
954 u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
955 u8 enabled_dbuf_slices = dev_priv->display.dbuf.enabled_slices;
956
957 drm_WARN(&dev_priv->drm,
958 hw_enabled_dbuf_slices != enabled_dbuf_slices,
959 "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n",
960 hw_enabled_dbuf_slices,
961 enabled_dbuf_slices);
962}
963
964void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
965{
966 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
967 struct intel_cdclk_config cdclk_config = {};
968
969 if (power_domains->target_dc_state == DC_STATE_EN_DC3CO) {
970 tgl_disable_dc3co(dev_priv);
971 return;
972 }
973
974 gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
975
976 if (!HAS_DISPLAY(dev_priv))
977 return;
978
979 intel_cdclk_get_cdclk(dev_priv, &cdclk_config);
980 /* Can't read out voltage_level so can't use intel_cdclk_changed() */
981 drm_WARN_ON(&dev_priv->drm,
982 intel_cdclk_needs_modeset(&dev_priv->display.cdclk.hw,
983 &cdclk_config));
984
985 gen9_assert_dbuf_enabled(dev_priv);
986
987 if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
988 bxt_verify_ddi_phy_power_wells(dev_priv);
989
990 if (DISPLAY_VER(dev_priv) >= 11)
991 /*
992 * DMC retains HW context only for port A, the other combo
993 * PHY's HW context for port B is lost after DC transitions,
994 * so we need to restore it manually.
995 */
996 intel_combo_phy_init(dev_priv);
997}
998
999static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
1000 struct i915_power_well *power_well)
1001{
1002 gen9_disable_dc_states(dev_priv);
1003}
1004
1005static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
1006 struct i915_power_well *power_well)
1007{
1008 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1009
1010 if (!intel_dmc_has_payload(dev_priv))
1011 return;
1012
1013 switch (power_domains->target_dc_state) {
1014 case DC_STATE_EN_DC3CO:
1015 tgl_enable_dc3co(dev_priv);
1016 break;
1017 case DC_STATE_EN_UPTO_DC6:
1018 skl_enable_dc6(dev_priv);
1019 break;
1020 case DC_STATE_EN_UPTO_DC5:
1021 gen9_enable_dc5(dev_priv);
1022 break;
1023 }
1024}
1025
1026static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
1027 struct i915_power_well *power_well)
1028{
1029}
1030
1031static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
1032 struct i915_power_well *power_well)
1033{
1034}
1035
1036static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
1037 struct i915_power_well *power_well)
1038{
1039 return true;
1040}
1041
1042static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
1043 struct i915_power_well *power_well)
1044{
1045 if ((intel_de_read(dev_priv, TRANSCONF(PIPE_A)) & TRANSCONF_ENABLE) == 0)
1046 i830_enable_pipe(dev_priv, PIPE_A);
1047 if ((intel_de_read(dev_priv, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE) == 0)
1048 i830_enable_pipe(dev_priv, PIPE_B);
1049}
1050
1051static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
1052 struct i915_power_well *power_well)
1053{
1054 i830_disable_pipe(dev_priv, PIPE_B);
1055 i830_disable_pipe(dev_priv, PIPE_A);
1056}
1057
1058static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
1059 struct i915_power_well *power_well)
1060{
1061 return intel_de_read(dev_priv, TRANSCONF(PIPE_A)) & TRANSCONF_ENABLE &&
1062 intel_de_read(dev_priv, TRANSCONF(PIPE_B)) & TRANSCONF_ENABLE;
1063}
1064
1065static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
1066 struct i915_power_well *power_well)
1067{
1068 if (intel_power_well_refcount(power_well) > 0)
1069 i830_pipes_power_well_enable(dev_priv, power_well);
1070 else
1071 i830_pipes_power_well_disable(dev_priv, power_well);
1072}
1073
1074static void vlv_set_power_well(struct drm_i915_private *dev_priv,
1075 struct i915_power_well *power_well, bool enable)
1076{
1077 int pw_idx = i915_power_well_instance(power_well)->vlv.idx;
1078 u32 mask;
1079 u32 state;
1080 u32 ctrl;
1081
1082 mask = PUNIT_PWRGT_MASK(pw_idx);
1083 state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
1084 PUNIT_PWRGT_PWR_GATE(pw_idx);
1085
1086 vlv_punit_get(dev_priv);
1087
1088#define COND \
1089 ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1090
1091 if (COND)
1092 goto out;
1093
1094 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1095 ctrl &= ~mask;
1096 ctrl |= state;
1097 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1098
1099 if (wait_for(COND, 100))
1100 drm_err(&dev_priv->drm,
1101 "timeout setting power well state %08x (%08x)\n",
1102 state,
1103 vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1104
1105#undef COND
1106
1107out:
1108 vlv_punit_put(dev_priv);
1109}
1110
1111static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1112 struct i915_power_well *power_well)
1113{
1114 vlv_set_power_well(dev_priv, power_well, true);
1115}
1116
1117static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1118 struct i915_power_well *power_well)
1119{
1120 vlv_set_power_well(dev_priv, power_well, false);
1121}
1122
1123static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1124 struct i915_power_well *power_well)
1125{
1126 int pw_idx = i915_power_well_instance(power_well)->vlv.idx;
1127 bool enabled = false;
1128 u32 mask;
1129 u32 state;
1130 u32 ctrl;
1131
1132 mask = PUNIT_PWRGT_MASK(pw_idx);
1133 ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
1134
1135 vlv_punit_get(dev_priv);
1136
1137 state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1138 /*
1139 * We only ever set the power-on and power-gate states, anything
1140 * else is unexpected.
1141 */
1142 drm_WARN_ON(&dev_priv->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
1143 state != PUNIT_PWRGT_PWR_GATE(pw_idx));
1144 if (state == ctrl)
1145 enabled = true;
1146
1147 /*
1148 * A transient state at this point would mean some unexpected party
1149 * is poking at the power controls too.
1150 */
1151 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1152 drm_WARN_ON(&dev_priv->drm, ctrl != state);
1153
1154 vlv_punit_put(dev_priv);
1155
1156 return enabled;
1157}
1158
1159static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1160{
1161 /*
1162 * On driver load, a pipe may be active and driving a DSI display.
1163 * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1164 * (and never recovering) in this case. intel_dsi_post_disable() will
1165 * clear it when we turn off the display.
1166 */
1167 intel_de_rmw(dev_priv, DSPCLK_GATE_D(dev_priv),
1168 ~DPOUNIT_CLOCK_GATE_DISABLE, VRHUNIT_CLOCK_GATE_DISABLE);
1169
1170 /*
1171 * Disable trickle feed and enable pnd deadline calculation
1172 */
1173 intel_de_write(dev_priv, MI_ARB_VLV,
1174 MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1175 intel_de_write(dev_priv, CBR1_VLV, 0);
1176
1177 drm_WARN_ON(&dev_priv->drm, RUNTIME_INFO(dev_priv)->rawclk_freq == 0);
1178 intel_de_write(dev_priv, RAWCLK_FREQ_VLV,
1179 DIV_ROUND_CLOSEST(RUNTIME_INFO(dev_priv)->rawclk_freq,
1180 1000));
1181}
1182
1183static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1184{
1185 struct intel_encoder *encoder;
1186 enum pipe pipe;
1187
1188 /*
1189 * Enable the CRI clock source so we can get at the
1190 * display and the reference clock for VGA
1191 * hotplug / manual detection. Supposedly DSI also
1192 * needs the ref clock up and running.
1193 *
1194 * CHV DPLL B/C have some issues if VGA mode is enabled.
1195 */
1196 for_each_pipe(dev_priv, pipe) {
1197 u32 val = intel_de_read(dev_priv, DPLL(pipe));
1198
1199 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1200 if (pipe != PIPE_A)
1201 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1202
1203 intel_de_write(dev_priv, DPLL(pipe), val);
1204 }
1205
1206 vlv_init_display_clock_gating(dev_priv);
1207
1208 spin_lock_irq(&dev_priv->irq_lock);
1209 valleyview_enable_display_irqs(dev_priv);
1210 spin_unlock_irq(&dev_priv->irq_lock);
1211
1212 /*
1213 * During driver initialization/resume we can avoid restoring the
1214 * part of the HW/SW state that will be inited anyway explicitly.
1215 */
1216 if (dev_priv->display.power.domains.initializing)
1217 return;
1218
1219 intel_hpd_init(dev_priv);
1220 intel_hpd_poll_disable(dev_priv);
1221
1222 /* Re-enable the ADPA, if we have one */
1223 for_each_intel_encoder(&dev_priv->drm, encoder) {
1224 if (encoder->type == INTEL_OUTPUT_ANALOG)
1225 intel_crt_reset(&encoder->base);
1226 }
1227
1228 intel_vga_redisable_power_on(dev_priv);
1229
1230 intel_pps_unlock_regs_wa(dev_priv);
1231}
1232
1233static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1234{
1235 spin_lock_irq(&dev_priv->irq_lock);
1236 valleyview_disable_display_irqs(dev_priv);
1237 spin_unlock_irq(&dev_priv->irq_lock);
1238
1239 /* make sure we're done processing display irqs */
1240 intel_synchronize_irq(dev_priv);
1241
1242 intel_pps_reset_all(dev_priv);
1243
1244 /* Prevent us from re-enabling polling on accident in late suspend */
1245 if (!dev_priv->drm.dev->power.is_suspended)
1246 intel_hpd_poll_enable(dev_priv);
1247}
1248
1249static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1250 struct i915_power_well *power_well)
1251{
1252 vlv_set_power_well(dev_priv, power_well, true);
1253
1254 vlv_display_power_well_init(dev_priv);
1255}
1256
1257static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1258 struct i915_power_well *power_well)
1259{
1260 vlv_display_power_well_deinit(dev_priv);
1261
1262 vlv_set_power_well(dev_priv, power_well, false);
1263}
1264
1265static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1266 struct i915_power_well *power_well)
1267{
1268 /* since ref/cri clock was enabled */
1269 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1270
1271 vlv_set_power_well(dev_priv, power_well, true);
1272
1273 /*
1274 * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1275 * 6. De-assert cmn_reset/side_reset. Same as VLV X0.
1276 * a. GUnit 0x2110 bit[0] set to 1 (def 0)
1277 * b. The other bits such as sfr settings / modesel may all
1278 * be set to 0.
1279 *
1280 * This should only be done on init and resume from S3 with
1281 * both PLLs disabled, or we risk losing DPIO and PLL
1282 * synchronization.
1283 */
1284 intel_de_rmw(dev_priv, DPIO_CTL, 0, DPIO_CMNRST);
1285}
1286
1287static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1288 struct i915_power_well *power_well)
1289{
1290 enum pipe pipe;
1291
1292 for_each_pipe(dev_priv, pipe)
1293 assert_pll_disabled(dev_priv, pipe);
1294
1295 /* Assert common reset */
1296 intel_de_rmw(dev_priv, DPIO_CTL, DPIO_CMNRST, 0);
1297
1298 vlv_set_power_well(dev_priv, power_well, false);
1299}
1300
1301#define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1302
1303static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1304{
1305 struct i915_power_well *cmn_bc =
1306 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1307 struct i915_power_well *cmn_d =
1308 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1309 u32 phy_control = dev_priv->display.power.chv_phy_control;
1310 u32 phy_status = 0;
1311 u32 phy_status_mask = 0xffffffff;
1312
1313 /*
1314 * The BIOS can leave the PHY is some weird state
1315 * where it doesn't fully power down some parts.
1316 * Disable the asserts until the PHY has been fully
1317 * reset (ie. the power well has been disabled at
1318 * least once).
1319 */
1320 if (!dev_priv->display.power.chv_phy_assert[DPIO_PHY0])
1321 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1322 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1323 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1324 PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1325 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1326 PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1327
1328 if (!dev_priv->display.power.chv_phy_assert[DPIO_PHY1])
1329 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1330 PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1331 PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1332
1333 if (intel_power_well_is_enabled(dev_priv, cmn_bc)) {
1334 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1335
1336 /* this assumes override is only used to enable lanes */
1337 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1338 phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1339
1340 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1341 phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1342
1343 /* CL1 is on whenever anything is on in either channel */
1344 if (BITS_SET(phy_control,
1345 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1346 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1347 phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1348
1349 /*
1350 * The DPLLB check accounts for the pipe B + port A usage
1351 * with CL2 powered up but all the lanes in the second channel
1352 * powered down.
1353 */
1354 if (BITS_SET(phy_control,
1355 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1356 (intel_de_read(dev_priv, DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1357 phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1358
1359 if (BITS_SET(phy_control,
1360 PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1361 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1362 if (BITS_SET(phy_control,
1363 PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1364 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1365
1366 if (BITS_SET(phy_control,
1367 PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1368 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1369 if (BITS_SET(phy_control,
1370 PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1371 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1372 }
1373
1374 if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
1375 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1376
1377 /* this assumes override is only used to enable lanes */
1378 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1379 phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1380
1381 if (BITS_SET(phy_control,
1382 PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1383 phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1384
1385 if (BITS_SET(phy_control,
1386 PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1387 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1388 if (BITS_SET(phy_control,
1389 PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1390 phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1391 }
1392
1393 phy_status &= phy_status_mask;
1394
1395 /*
1396 * The PHY may be busy with some initial calibration and whatnot,
1397 * so the power state can take a while to actually change.
1398 */
1399 if (intel_de_wait_for_register(dev_priv, DISPLAY_PHY_STATUS,
1400 phy_status_mask, phy_status, 10))
1401 drm_err(&dev_priv->drm,
1402 "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1403 intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask,
1404 phy_status, dev_priv->display.power.chv_phy_control);
1405}
1406
1407#undef BITS_SET
1408
1409static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1410 struct i915_power_well *power_well)
1411{
1412 enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
1413 enum dpio_phy phy;
1414 u32 tmp;
1415
1416 drm_WARN_ON_ONCE(&dev_priv->drm,
1417 id != VLV_DISP_PW_DPIO_CMN_BC &&
1418 id != CHV_DISP_PW_DPIO_CMN_D);
1419
1420 if (id == VLV_DISP_PW_DPIO_CMN_BC)
1421 phy = DPIO_PHY0;
1422 else
1423 phy = DPIO_PHY1;
1424
1425 /* since ref/cri clock was enabled */
1426 udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1427 vlv_set_power_well(dev_priv, power_well, true);
1428
1429 /* Poll for phypwrgood signal */
1430 if (intel_de_wait_for_set(dev_priv, DISPLAY_PHY_STATUS,
1431 PHY_POWERGOOD(phy), 1))
1432 drm_err(&dev_priv->drm, "Display PHY %d is not power up\n",
1433 phy);
1434
1435 vlv_dpio_get(dev_priv);
1436
1437 /* Enable dynamic power down */
1438 tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW28);
1439 tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1440 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1441 vlv_dpio_write(dev_priv, phy, CHV_CMN_DW28, tmp);
1442
1443 if (id == VLV_DISP_PW_DPIO_CMN_BC) {
1444 tmp = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW6_CH1);
1445 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1446 vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW6_CH1, tmp);
1447 } else {
1448 /*
1449 * Force the non-existing CL2 off. BXT does this
1450 * too, so maybe it saves some power even though
1451 * CL2 doesn't exist?
1452 */
1453 tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW30);
1454 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1455 vlv_dpio_write(dev_priv, phy, CHV_CMN_DW30, tmp);
1456 }
1457
1458 vlv_dpio_put(dev_priv);
1459
1460 dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1461 intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1462 dev_priv->display.power.chv_phy_control);
1463
1464 drm_dbg_kms(&dev_priv->drm,
1465 "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1466 phy, dev_priv->display.power.chv_phy_control);
1467
1468 assert_chv_phy_status(dev_priv);
1469}
1470
1471static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1472 struct i915_power_well *power_well)
1473{
1474 enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
1475 enum dpio_phy phy;
1476
1477 drm_WARN_ON_ONCE(&dev_priv->drm,
1478 id != VLV_DISP_PW_DPIO_CMN_BC &&
1479 id != CHV_DISP_PW_DPIO_CMN_D);
1480
1481 if (id == VLV_DISP_PW_DPIO_CMN_BC) {
1482 phy = DPIO_PHY0;
1483 assert_pll_disabled(dev_priv, PIPE_A);
1484 assert_pll_disabled(dev_priv, PIPE_B);
1485 } else {
1486 phy = DPIO_PHY1;
1487 assert_pll_disabled(dev_priv, PIPE_C);
1488 }
1489
1490 dev_priv->display.power.chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1491 intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1492 dev_priv->display.power.chv_phy_control);
1493
1494 vlv_set_power_well(dev_priv, power_well, false);
1495
1496 drm_dbg_kms(&dev_priv->drm,
1497 "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1498 phy, dev_priv->display.power.chv_phy_control);
1499
1500 /* PHY is fully reset now, so we can enable the PHY state asserts */
1501 dev_priv->display.power.chv_phy_assert[phy] = true;
1502
1503 assert_chv_phy_status(dev_priv);
1504}
1505
1506static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1507 enum dpio_channel ch, bool override, unsigned int mask)
1508{
1509 u32 reg, val, expected, actual;
1510
1511 /*
1512 * The BIOS can leave the PHY is some weird state
1513 * where it doesn't fully power down some parts.
1514 * Disable the asserts until the PHY has been fully
1515 * reset (ie. the power well has been disabled at
1516 * least once).
1517 */
1518 if (!dev_priv->display.power.chv_phy_assert[phy])
1519 return;
1520
1521 if (ch == DPIO_CH0)
1522 reg = _CHV_CMN_DW0_CH0;
1523 else
1524 reg = _CHV_CMN_DW6_CH1;
1525
1526 vlv_dpio_get(dev_priv);
1527 val = vlv_dpio_read(dev_priv, phy, reg);
1528 vlv_dpio_put(dev_priv);
1529
1530 /*
1531 * This assumes !override is only used when the port is disabled.
1532 * All lanes should power down even without the override when
1533 * the port is disabled.
1534 */
1535 if (!override || mask == 0xf) {
1536 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1537 /*
1538 * If CH1 common lane is not active anymore
1539 * (eg. for pipe B DPLL) the entire channel will
1540 * shut down, which causes the common lane registers
1541 * to read as 0. That means we can't actually check
1542 * the lane power down status bits, but as the entire
1543 * register reads as 0 it's a good indication that the
1544 * channel is indeed entirely powered down.
1545 */
1546 if (ch == DPIO_CH1 && val == 0)
1547 expected = 0;
1548 } else if (mask != 0x0) {
1549 expected = DPIO_ANYDL_POWERDOWN;
1550 } else {
1551 expected = 0;
1552 }
1553
1554 if (ch == DPIO_CH0)
1555 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1556 else
1557 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1558 actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1559
1560 drm_WARN(&dev_priv->drm, actual != expected,
1561 "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1562 !!(actual & DPIO_ALLDL_POWERDOWN),
1563 !!(actual & DPIO_ANYDL_POWERDOWN),
1564 !!(expected & DPIO_ALLDL_POWERDOWN),
1565 !!(expected & DPIO_ANYDL_POWERDOWN),
1566 reg, val);
1567}
1568
1569bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1570 enum dpio_channel ch, bool override)
1571{
1572 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1573 bool was_override;
1574
1575 mutex_lock(&power_domains->lock);
1576
1577 was_override = dev_priv->display.power.chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1578
1579 if (override == was_override)
1580 goto out;
1581
1582 if (override)
1583 dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1584 else
1585 dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1586
1587 intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1588 dev_priv->display.power.chv_phy_control);
1589
1590 drm_dbg_kms(&dev_priv->drm,
1591 "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1592 phy, ch, dev_priv->display.power.chv_phy_control);
1593
1594 assert_chv_phy_status(dev_priv);
1595
1596out:
1597 mutex_unlock(&power_domains->lock);
1598
1599 return was_override;
1600}
1601
1602void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1603 bool override, unsigned int mask)
1604{
1605 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1606 struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1607 enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
1608 enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
1609
1610 mutex_lock(&power_domains->lock);
1611
1612 dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1613 dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1614
1615 if (override)
1616 dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1617 else
1618 dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1619
1620 intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1621 dev_priv->display.power.chv_phy_control);
1622
1623 drm_dbg_kms(&dev_priv->drm,
1624 "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1625 phy, ch, mask, dev_priv->display.power.chv_phy_control);
1626
1627 assert_chv_phy_status(dev_priv);
1628
1629 assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1630
1631 mutex_unlock(&power_domains->lock);
1632}
1633
1634static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1635 struct i915_power_well *power_well)
1636{
1637 enum pipe pipe = PIPE_A;
1638 bool enabled;
1639 u32 state, ctrl;
1640
1641 vlv_punit_get(dev_priv);
1642
1643 state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
1644 /*
1645 * We only ever set the power-on and power-gate states, anything
1646 * else is unexpected.
1647 */
1648 drm_WARN_ON(&dev_priv->drm, state != DP_SSS_PWR_ON(pipe) &&
1649 state != DP_SSS_PWR_GATE(pipe));
1650 enabled = state == DP_SSS_PWR_ON(pipe);
1651
1652 /*
1653 * A transient state at this point would mean some unexpected party
1654 * is poking at the power controls too.
1655 */
1656 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
1657 drm_WARN_ON(&dev_priv->drm, ctrl << 16 != state);
1658
1659 vlv_punit_put(dev_priv);
1660
1661 return enabled;
1662}
1663
1664static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1665 struct i915_power_well *power_well,
1666 bool enable)
1667{
1668 enum pipe pipe = PIPE_A;
1669 u32 state;
1670 u32 ctrl;
1671
1672 state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1673
1674 vlv_punit_get(dev_priv);
1675
1676#define COND \
1677 ((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
1678
1679 if (COND)
1680 goto out;
1681
1682 ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
1683 ctrl &= ~DP_SSC_MASK(pipe);
1684 ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1685 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
1686
1687 if (wait_for(COND, 100))
1688 drm_err(&dev_priv->drm,
1689 "timeout setting power well state %08x (%08x)\n",
1690 state,
1691 vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
1692
1693#undef COND
1694
1695out:
1696 vlv_punit_put(dev_priv);
1697}
1698
1699static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
1700 struct i915_power_well *power_well)
1701{
1702 intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1703 dev_priv->display.power.chv_phy_control);
1704}
1705
1706static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1707 struct i915_power_well *power_well)
1708{
1709 chv_set_pipe_power_well(dev_priv, power_well, true);
1710
1711 vlv_display_power_well_init(dev_priv);
1712}
1713
1714static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1715 struct i915_power_well *power_well)
1716{
1717 vlv_display_power_well_deinit(dev_priv);
1718
1719 chv_set_pipe_power_well(dev_priv, power_well, false);
1720}
1721
1722static void
1723tgl_tc_cold_request(struct drm_i915_private *i915, bool block)
1724{
1725 u8 tries = 0;
1726 int ret;
1727
1728 while (1) {
1729 u32 low_val;
1730 u32 high_val = 0;
1731
1732 if (block)
1733 low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ;
1734 else
1735 low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ;
1736
1737 /*
1738 * Spec states that we should timeout the request after 200us
1739 * but the function below will timeout after 500us
1740 */
1741 ret = snb_pcode_read(&i915->uncore, TGL_PCODE_TCCOLD, &low_val, &high_val);
1742 if (ret == 0) {
1743 if (block &&
1744 (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED))
1745 ret = -EIO;
1746 else
1747 break;
1748 }
1749
1750 if (++tries == 3)
1751 break;
1752
1753 msleep(1);
1754 }
1755
1756 if (ret)
1757 drm_err(&i915->drm, "TC cold %sblock failed\n",
1758 block ? "" : "un");
1759 else
1760 drm_dbg_kms(&i915->drm, "TC cold %sblock succeeded\n",
1761 block ? "" : "un");
1762}
1763
1764static void
1765tgl_tc_cold_off_power_well_enable(struct drm_i915_private *i915,
1766 struct i915_power_well *power_well)
1767{
1768 tgl_tc_cold_request(i915, true);
1769}
1770
1771static void
1772tgl_tc_cold_off_power_well_disable(struct drm_i915_private *i915,
1773 struct i915_power_well *power_well)
1774{
1775 tgl_tc_cold_request(i915, false);
1776}
1777
1778static void
1779tgl_tc_cold_off_power_well_sync_hw(struct drm_i915_private *i915,
1780 struct i915_power_well *power_well)
1781{
1782 if (intel_power_well_refcount(power_well) > 0)
1783 tgl_tc_cold_off_power_well_enable(i915, power_well);
1784 else
1785 tgl_tc_cold_off_power_well_disable(i915, power_well);
1786}
1787
1788static bool
1789tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv,
1790 struct i915_power_well *power_well)
1791{
1792 /*
1793 * Not the correctly implementation but there is no way to just read it
1794 * from PCODE, so returning count to avoid state mismatch errors
1795 */
1796 return intel_power_well_refcount(power_well);
1797}
1798
1799static void xelpdp_aux_power_well_enable(struct drm_i915_private *dev_priv,
1800 struct i915_power_well *power_well)
1801{
1802 enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1803 enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
1804
1805 if (intel_phy_is_tc(dev_priv, phy))
1806 icl_tc_port_assert_ref_held(dev_priv, power_well,
1807 aux_ch_to_digital_port(dev_priv, aux_ch));
1808
1809 intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch),
1810 XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
1811 XELPDP_DP_AUX_CH_CTL_POWER_REQUEST);
1812
1813 /*
1814 * The power status flag cannot be used to determine whether aux
1815 * power wells have finished powering up. Instead we're
1816 * expected to just wait a fixed 600us after raising the request
1817 * bit.
1818 */
1819 usleep_range(600, 1200);
1820}
1821
1822static void xelpdp_aux_power_well_disable(struct drm_i915_private *dev_priv,
1823 struct i915_power_well *power_well)
1824{
1825 enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1826
1827 intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch),
1828 XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
1829 0);
1830 usleep_range(10, 30);
1831}
1832
1833static bool xelpdp_aux_power_well_enabled(struct drm_i915_private *dev_priv,
1834 struct i915_power_well *power_well)
1835{
1836 enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
1837
1838 return intel_de_read(dev_priv, XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch)) &
1839 XELPDP_DP_AUX_CH_CTL_POWER_STATUS;
1840}
1841
1842static void xe2lpd_pica_power_well_enable(struct drm_i915_private *dev_priv,
1843 struct i915_power_well *power_well)
1844{
1845 intel_de_write(dev_priv, XE2LPD_PICA_PW_CTL,
1846 XE2LPD_PICA_CTL_POWER_REQUEST);
1847
1848 if (intel_de_wait_for_set(dev_priv, XE2LPD_PICA_PW_CTL,
1849 XE2LPD_PICA_CTL_POWER_STATUS, 1)) {
1850 drm_dbg_kms(&dev_priv->drm, "pica power well enable timeout\n");
1851
1852 drm_WARN(&dev_priv->drm, 1, "Power well PICA timeout when enabled");
1853 }
1854}
1855
1856static void xe2lpd_pica_power_well_disable(struct drm_i915_private *dev_priv,
1857 struct i915_power_well *power_well)
1858{
1859 intel_de_write(dev_priv, XE2LPD_PICA_PW_CTL, 0);
1860
1861 if (intel_de_wait_for_clear(dev_priv, XE2LPD_PICA_PW_CTL,
1862 XE2LPD_PICA_CTL_POWER_STATUS, 1)) {
1863 drm_dbg_kms(&dev_priv->drm, "pica power well disable timeout\n");
1864
1865 drm_WARN(&dev_priv->drm, 1, "Power well PICA timeout when disabled");
1866 }
1867}
1868
1869static bool xe2lpd_pica_power_well_enabled(struct drm_i915_private *dev_priv,
1870 struct i915_power_well *power_well)
1871{
1872 return intel_de_read(dev_priv, XE2LPD_PICA_PW_CTL) &
1873 XE2LPD_PICA_CTL_POWER_STATUS;
1874}
1875
1876const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1877 .sync_hw = i9xx_power_well_sync_hw_noop,
1878 .enable = i9xx_always_on_power_well_noop,
1879 .disable = i9xx_always_on_power_well_noop,
1880 .is_enabled = i9xx_always_on_power_well_enabled,
1881};
1882
1883const struct i915_power_well_ops chv_pipe_power_well_ops = {
1884 .sync_hw = chv_pipe_power_well_sync_hw,
1885 .enable = chv_pipe_power_well_enable,
1886 .disable = chv_pipe_power_well_disable,
1887 .is_enabled = chv_pipe_power_well_enabled,
1888};
1889
1890const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1891 .sync_hw = i9xx_power_well_sync_hw_noop,
1892 .enable = chv_dpio_cmn_power_well_enable,
1893 .disable = chv_dpio_cmn_power_well_disable,
1894 .is_enabled = vlv_power_well_enabled,
1895};
1896
1897const struct i915_power_well_ops i830_pipes_power_well_ops = {
1898 .sync_hw = i830_pipes_power_well_sync_hw,
1899 .enable = i830_pipes_power_well_enable,
1900 .disable = i830_pipes_power_well_disable,
1901 .is_enabled = i830_pipes_power_well_enabled,
1902};
1903
1904static const struct i915_power_well_regs hsw_power_well_regs = {
1905 .bios = HSW_PWR_WELL_CTL1,
1906 .driver = HSW_PWR_WELL_CTL2,
1907 .kvmr = HSW_PWR_WELL_CTL3,
1908 .debug = HSW_PWR_WELL_CTL4,
1909};
1910
1911const struct i915_power_well_ops hsw_power_well_ops = {
1912 .regs = &hsw_power_well_regs,
1913 .sync_hw = hsw_power_well_sync_hw,
1914 .enable = hsw_power_well_enable,
1915 .disable = hsw_power_well_disable,
1916 .is_enabled = hsw_power_well_enabled,
1917};
1918
1919const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1920 .sync_hw = i9xx_power_well_sync_hw_noop,
1921 .enable = gen9_dc_off_power_well_enable,
1922 .disable = gen9_dc_off_power_well_disable,
1923 .is_enabled = gen9_dc_off_power_well_enabled,
1924};
1925
1926const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1927 .sync_hw = i9xx_power_well_sync_hw_noop,
1928 .enable = bxt_dpio_cmn_power_well_enable,
1929 .disable = bxt_dpio_cmn_power_well_disable,
1930 .is_enabled = bxt_dpio_cmn_power_well_enabled,
1931};
1932
1933const struct i915_power_well_ops vlv_display_power_well_ops = {
1934 .sync_hw = i9xx_power_well_sync_hw_noop,
1935 .enable = vlv_display_power_well_enable,
1936 .disable = vlv_display_power_well_disable,
1937 .is_enabled = vlv_power_well_enabled,
1938};
1939
1940const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1941 .sync_hw = i9xx_power_well_sync_hw_noop,
1942 .enable = vlv_dpio_cmn_power_well_enable,
1943 .disable = vlv_dpio_cmn_power_well_disable,
1944 .is_enabled = vlv_power_well_enabled,
1945};
1946
1947const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1948 .sync_hw = i9xx_power_well_sync_hw_noop,
1949 .enable = vlv_power_well_enable,
1950 .disable = vlv_power_well_disable,
1951 .is_enabled = vlv_power_well_enabled,
1952};
1953
1954static const struct i915_power_well_regs icl_aux_power_well_regs = {
1955 .bios = ICL_PWR_WELL_CTL_AUX1,
1956 .driver = ICL_PWR_WELL_CTL_AUX2,
1957 .debug = ICL_PWR_WELL_CTL_AUX4,
1958};
1959
1960const struct i915_power_well_ops icl_aux_power_well_ops = {
1961 .regs = &icl_aux_power_well_regs,
1962 .sync_hw = hsw_power_well_sync_hw,
1963 .enable = icl_aux_power_well_enable,
1964 .disable = icl_aux_power_well_disable,
1965 .is_enabled = hsw_power_well_enabled,
1966};
1967
1968static const struct i915_power_well_regs icl_ddi_power_well_regs = {
1969 .bios = ICL_PWR_WELL_CTL_DDI1,
1970 .driver = ICL_PWR_WELL_CTL_DDI2,
1971 .debug = ICL_PWR_WELL_CTL_DDI4,
1972};
1973
1974const struct i915_power_well_ops icl_ddi_power_well_ops = {
1975 .regs = &icl_ddi_power_well_regs,
1976 .sync_hw = hsw_power_well_sync_hw,
1977 .enable = hsw_power_well_enable,
1978 .disable = hsw_power_well_disable,
1979 .is_enabled = hsw_power_well_enabled,
1980};
1981
1982const struct i915_power_well_ops tgl_tc_cold_off_ops = {
1983 .sync_hw = tgl_tc_cold_off_power_well_sync_hw,
1984 .enable = tgl_tc_cold_off_power_well_enable,
1985 .disable = tgl_tc_cold_off_power_well_disable,
1986 .is_enabled = tgl_tc_cold_off_power_well_is_enabled,
1987};
1988
1989const struct i915_power_well_ops xelpdp_aux_power_well_ops = {
1990 .sync_hw = i9xx_power_well_sync_hw_noop,
1991 .enable = xelpdp_aux_power_well_enable,
1992 .disable = xelpdp_aux_power_well_disable,
1993 .is_enabled = xelpdp_aux_power_well_enabled,
1994};
1995
1996const struct i915_power_well_ops xe2lpd_pica_power_well_ops = {
1997 .sync_hw = i9xx_power_well_sync_hw_noop,
1998 .enable = xe2lpd_pica_power_well_enable,
1999 .disable = xe2lpd_pica_power_well_disable,
2000 .is_enabled = xe2lpd_pica_power_well_enabled,
2001};