Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5
6#include <linux/kernel.h>
7
8#include <drm/drm_print.h>
9
10#include "i915_drv.h"
11#include "i915_reg.h"
12#include "i915_trace.h"
13#include "i915_utils.h"
14#include "intel_pm.h"
15#include "vlv_suspend.h"
16
17struct vlv_s0ix_state {
18 /* GAM */
19 u32 wr_watermark;
20 u32 gfx_prio_ctrl;
21 u32 arb_mode;
22 u32 gfx_pend_tlb0;
23 u32 gfx_pend_tlb1;
24 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
25 u32 media_max_req_count;
26 u32 gfx_max_req_count;
27 u32 render_hwsp;
28 u32 ecochk;
29 u32 bsd_hwsp;
30 u32 blt_hwsp;
31 u32 tlb_rd_addr;
32
33 /* MBC */
34 u32 g3dctl;
35 u32 gsckgctl;
36 u32 mbctl;
37
38 /* GCP */
39 u32 ucgctl1;
40 u32 ucgctl3;
41 u32 rcgctl1;
42 u32 rcgctl2;
43 u32 rstctl;
44 u32 misccpctl;
45
46 /* GPM */
47 u32 gfxpause;
48 u32 rpdeuhwtc;
49 u32 rpdeuc;
50 u32 ecobus;
51 u32 pwrdwnupctl;
52 u32 rp_down_timeout;
53 u32 rp_deucsw;
54 u32 rcubmabdtmr;
55 u32 rcedata;
56 u32 spare2gh;
57
58 /* Display 1 CZ domain */
59 u32 gt_imr;
60 u32 gt_ier;
61 u32 pm_imr;
62 u32 pm_ier;
63 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
64
65 /* GT SA CZ domain */
66 u32 tilectl;
67 u32 gt_fifoctl;
68 u32 gtlc_wake_ctrl;
69 u32 gtlc_survive;
70 u32 pmwgicz;
71
72 /* Display 2 CZ domain */
73 u32 gu_ctl0;
74 u32 gu_ctl1;
75 u32 pcbr;
76 u32 clock_gate_dis2;
77};
78
79/*
80 * Save all Gunit registers that may be lost after a D3 and a subsequent
81 * S0i[R123] transition. The list of registers needing a save/restore is
82 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
83 * registers in the following way:
84 * - Driver: saved/restored by the driver
85 * - Punit : saved/restored by the Punit firmware
86 * - No, w/o marking: no need to save/restore, since the register is R/O or
87 * used internally by the HW in a way that doesn't depend
88 * keeping the content across a suspend/resume.
89 * - Debug : used for debugging
90 *
91 * We save/restore all registers marked with 'Driver', with the following
92 * exceptions:
93 * - Registers out of use, including also registers marked with 'Debug'.
94 * These have no effect on the driver's operation, so we don't save/restore
95 * them to reduce the overhead.
96 * - Registers that are fully setup by an initialization function called from
97 * the resume path. For example many clock gating and RPS/RC6 registers.
98 * - Registers that provide the right functionality with their reset defaults.
99 *
100 * TODO: Except for registers that based on the above 3 criteria can be safely
101 * ignored, we save/restore all others, practically treating the HW context as
102 * a black-box for the driver. Further investigation is needed to reduce the
103 * saved/restored registers even further, by following the same 3 criteria.
104 */
105static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915)
106{
107 struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
108 struct intel_uncore *uncore = &i915->uncore;
109 int i;
110
111 if (!s)
112 return;
113
114 /* GAM 0x4000-0x4770 */
115 s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK);
116 s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL);
117 s->arb_mode = intel_uncore_read(uncore, ARB_MODE);
118 s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0);
119 s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1);
120
121 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
122 s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i));
123
124 s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT);
125 s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT);
126
127 s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7);
128 s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
129 s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7);
130 s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7);
131
132 s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR);
133
134 /* MBC 0x9024-0x91D0, 0x8500 */
135 s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL);
136 s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL);
137 s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL);
138
139 /* GCP 0x9400-0x9424, 0x8100-0x810C */
140 s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1);
141 s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3);
142 s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1);
143 s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2);
144 s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL);
145 s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL);
146
147 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
148 s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE);
149 s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC);
150 s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC);
151 s->ecobus = intel_uncore_read(uncore, ECOBUS);
152 s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL);
153 s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT);
154 s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW);
155 s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR);
156 s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA);
157 s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H);
158
159 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
160 s->gt_imr = intel_uncore_read(uncore, GTIMR);
161 s->gt_ier = intel_uncore_read(uncore, GTIER);
162 s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
163 s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
164
165 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
166 s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i));
167
168 /* GT SA CZ domain, 0x100000-0x138124 */
169 s->tilectl = intel_uncore_read(uncore, TILECTL);
170 s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL);
171 s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
172 s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
173 s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ);
174
175 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
176 s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0);
177 s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1);
178 s->pcbr = intel_uncore_read(uncore, VLV_PCBR);
179 s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2);
180
181 /*
182 * Not saving any of:
183 * DFT, 0x9800-0x9EC0
184 * SARB, 0xB000-0xB1FC
185 * GAC, 0x5208-0x524C, 0x14000-0x14C000
186 * PCI CFG
187 */
188}
189
190static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915)
191{
192 struct vlv_s0ix_state *s = i915->vlv_s0ix_state;
193 struct intel_uncore *uncore = &i915->uncore;
194 u32 val;
195 int i;
196
197 if (!s)
198 return;
199
200 /* GAM 0x4000-0x4770 */
201 intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark);
202 intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);
203 intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16));
204 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);
205 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);
206
207 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
208 intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]);
209
210 intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
211 intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
212
213 intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp);
214 intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk);
215 intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp);
216 intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp);
217
218 intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr);
219
220 /* MBC 0x9024-0x91D0, 0x8500 */
221 intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl);
222 intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl);
223 intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl);
224
225 /* GCP 0x9400-0x9424, 0x8100-0x810C */
226 intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1);
227 intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3);
228 intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1);
229 intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2);
230 intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl);
231 intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl);
232
233 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
234 intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause);
235 intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc);
236 intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc);
237 intel_uncore_write(uncore, ECOBUS, s->ecobus);
238 intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl);
239 intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout);
240 intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw);
241 intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr);
242 intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata);
243 intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh);
244
245 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
246 intel_uncore_write(uncore, GTIMR, s->gt_imr);
247 intel_uncore_write(uncore, GTIER, s->gt_ier);
248 intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr);
249 intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier);
250
251 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
252 intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
253
254 /* GT SA CZ domain, 0x100000-0x138124 */
255 intel_uncore_write(uncore, TILECTL, s->tilectl);
256 intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl);
257 /*
258 * Preserve the GT allow wake and GFX force clock bit, they are not
259 * be restored, as they are used to control the s0ix suspend/resume
260 * sequence by the caller.
261 */
262 val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
263 val &= VLV_GTLC_ALLOWWAKEREQ;
264 val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
265 intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val);
266
267 val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
268 val &= VLV_GFX_CLK_FORCE_ON_BIT;
269 val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
270 intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val);
271
272 intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz);
273
274 /* Gunit-Display CZ domain, 0x182028-0x1821CF */
275 intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0);
276 intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1);
277 intel_uncore_write(uncore, VLV_PCBR, s->pcbr);
278 intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);
279}
280
281static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
282 u32 mask, u32 val)
283{
284 i915_reg_t reg = VLV_GTLC_PW_STATUS;
285 u32 reg_value;
286 int ret;
287
288 /* The HW does not like us polling for PW_STATUS frequently, so
289 * use the sleeping loop rather than risk the busy spin within
290 * intel_wait_for_register().
291 *
292 * Transitioning between RC6 states should be at most 2ms (see
293 * valleyview_enable_rps) so use a 3ms timeout.
294 */
295 ret = wait_for(((reg_value =
296 intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
297 == val, 3);
298
299 /* just trace the final value */
300 trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
301
302 return ret;
303}
304
305static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on)
306{
307 struct intel_uncore *uncore = &i915->uncore;
308 u32 val;
309 int err;
310
311 val = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);
312 val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
313 if (force_on)
314 val |= VLV_GFX_CLK_FORCE_ON_BIT;
315 intel_uncore_write(uncore, VLV_GTLC_SURVIVABILITY_REG, val);
316
317 if (!force_on)
318 return 0;
319
320 err = intel_wait_for_register(uncore,
321 VLV_GTLC_SURVIVABILITY_REG,
322 VLV_GFX_CLK_STATUS_BIT,
323 VLV_GFX_CLK_STATUS_BIT,
324 20);
325 if (err)
326 drm_err(&i915->drm,
327 "timeout waiting for GFX clock force-on (%08x)\n",
328 intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG));
329
330 return err;
331}
332
333static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow)
334{
335 struct intel_uncore *uncore = &i915->uncore;
336 u32 mask;
337 u32 val;
338 int err;
339
340 val = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);
341 val &= ~VLV_GTLC_ALLOWWAKEREQ;
342 if (allow)
343 val |= VLV_GTLC_ALLOWWAKEREQ;
344 intel_uncore_write(uncore, VLV_GTLC_WAKE_CTRL, val);
345 intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL);
346
347 mask = VLV_GTLC_ALLOWWAKEACK;
348 val = allow ? mask : 0;
349
350 err = vlv_wait_for_pw_status(i915, mask, val);
351 if (err)
352 drm_err(&i915->drm, "timeout disabling GT waking\n");
353
354 return err;
355}
356
357static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
358 bool wait_for_on)
359{
360 u32 mask;
361 u32 val;
362
363 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
364 val = wait_for_on ? mask : 0;
365
366 /*
367 * RC6 transitioning can be delayed up to 2 msec (see
368 * valleyview_enable_rps), use 3 msec for safety.
369 *
370 * This can fail to turn off the rc6 if the GPU is stuck after a failed
371 * reset and we are trying to force the machine to sleep.
372 */
373 if (vlv_wait_for_pw_status(dev_priv, mask, val))
374 drm_dbg(&dev_priv->drm,
375 "timeout waiting for GT wells to go %s\n",
376 onoff(wait_for_on));
377}
378
379static void vlv_check_no_gt_access(struct drm_i915_private *i915)
380{
381 struct intel_uncore *uncore = &i915->uncore;
382
383 if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
384 return;
385
386 drm_dbg(&i915->drm, "GT register access while GT waking disabled\n");
387 intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
388}
389
390int vlv_suspend_complete(struct drm_i915_private *dev_priv)
391{
392 u32 mask;
393 int err;
394
395 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
396 return 0;
397
398 /*
399 * Bspec defines the following GT well on flags as debug only, so
400 * don't treat them as hard failures.
401 */
402 vlv_wait_for_gt_wells(dev_priv, false);
403
404 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
405 drm_WARN_ON(&dev_priv->drm,
406 (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask);
407
408 vlv_check_no_gt_access(dev_priv);
409
410 err = vlv_force_gfx_clock(dev_priv, true);
411 if (err)
412 goto err1;
413
414 err = vlv_allow_gt_wake(dev_priv, false);
415 if (err)
416 goto err2;
417
418 vlv_save_gunit_s0ix_state(dev_priv);
419
420 err = vlv_force_gfx_clock(dev_priv, false);
421 if (err)
422 goto err2;
423
424 return 0;
425
426err2:
427 /* For safety always re-enable waking and disable gfx clock forcing */
428 vlv_allow_gt_wake(dev_priv, true);
429err1:
430 vlv_force_gfx_clock(dev_priv, false);
431
432 return err;
433}
434
435int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
436{
437 int err;
438 int ret;
439
440 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
441 return 0;
442
443 /*
444 * If any of the steps fail just try to continue, that's the best we
445 * can do at this point. Return the first error code (which will also
446 * leave RPM permanently disabled).
447 */
448 ret = vlv_force_gfx_clock(dev_priv, true);
449
450 vlv_restore_gunit_s0ix_state(dev_priv);
451
452 err = vlv_allow_gt_wake(dev_priv, true);
453 if (!ret)
454 ret = err;
455
456 err = vlv_force_gfx_clock(dev_priv, false);
457 if (!ret)
458 ret = err;
459
460 vlv_check_no_gt_access(dev_priv);
461
462 if (rpm_resume)
463 intel_init_clock_gating(dev_priv);
464
465 return ret;
466}
467
468int vlv_suspend_init(struct drm_i915_private *i915)
469{
470 if (!IS_VALLEYVIEW(i915))
471 return 0;
472
473 /* we write all the values in the struct, so no need to zero it out */
474 i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
475 GFP_KERNEL);
476 if (!i915->vlv_s0ix_state)
477 return -ENOMEM;
478
479 return 0;
480}
481
482void vlv_suspend_cleanup(struct drm_i915_private *i915)
483{
484 if (!i915->vlv_s0ix_state)
485 return;
486
487 kfree(i915->vlv_s0ix_state);
488 i915->vlv_s0ix_state = NULL;
489}