1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include <linux/string_helpers.h>
  7#include <linux/suspend.h>
  8
  9#include "i915_drv.h"
 10#include "i915_irq.h"
 11#include "i915_params.h"
 12#include "intel_context.h"
 13#include "intel_engine_pm.h"
 14#include "intel_gt.h"
 15#include "intel_gt_clock_utils.h"
 16#include "intel_gt_pm.h"
 17#include "intel_gt_requests.h"
 18#include "intel_llc.h"
 19#include "intel_pm.h"
 20#include "intel_rc6.h"
 21#include "intel_rps.h"
 22#include "intel_wakeref.h"
 23#include "intel_pcode.h"
 24#include "pxp/intel_pxp_pm.h"
 25
 26#define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
 27
 28static void mtl_media_busy(struct intel_gt *gt)
 29{
 30	/* Wa_14017073508: mtl */
 31	if (IS_MTL_GRAPHICS_STEP(gt->i915, P, STEP_A0, STEP_B0) &&
 32	    gt->type == GT_MEDIA)
 33		snb_pcode_write_p(gt->uncore, PCODE_MBOX_GT_STATE,
 34				  PCODE_MBOX_GT_STATE_MEDIA_BUSY,
 35				  PCODE_MBOX_GT_STATE_DOMAIN_MEDIA, 0);
 36}
 37
 38static void mtl_media_idle(struct intel_gt *gt)
 39{
 40	/* Wa_14017073508: mtl */
 41	if (IS_MTL_GRAPHICS_STEP(gt->i915, P, STEP_A0, STEP_B0) &&
 42	    gt->type == GT_MEDIA)
 43		snb_pcode_write_p(gt->uncore, PCODE_MBOX_GT_STATE,
 44				  PCODE_MBOX_GT_STATE_MEDIA_NOT_BUSY,
 45				  PCODE_MBOX_GT_STATE_DOMAIN_MEDIA, 0);
 46}
 47
 48static void user_forcewake(struct intel_gt *gt, bool suspend)
 49{
 50	int count = atomic_read(&gt->user_wakeref);
 51
 52	/* Inside suspend/resume so single threaded, no races to worry about. */
 53	if (likely(!count))
 54		return;
 55
 56	intel_gt_pm_get(gt);
 57	if (suspend) {
 58		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
 59		atomic_sub(count, &gt->wakeref.count);
 60	} else {
 61		atomic_add(count, &gt->wakeref.count);
 62	}
 63	intel_gt_pm_put(gt);
 64}
 65
 66static void runtime_begin(struct intel_gt *gt)
 67{
 68	local_irq_disable();
 69	write_seqcount_begin(&gt->stats.lock);
 70	gt->stats.start = ktime_get();
 71	gt->stats.active = true;
 72	write_seqcount_end(&gt->stats.lock);
 73	local_irq_enable();
 74}
 75
 76static void runtime_end(struct intel_gt *gt)
 77{
 78	local_irq_disable();
 79	write_seqcount_begin(&gt->stats.lock);
 80	gt->stats.active = false;
 81	gt->stats.total =
 82		ktime_add(gt->stats.total,
 83			  ktime_sub(ktime_get(), gt->stats.start));
 84	write_seqcount_end(&gt->stats.lock);
 85	local_irq_enable();
 86}
 87
 88static int __gt_unpark(struct intel_wakeref *wf)
 89{
 90	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
 91	struct drm_i915_private *i915 = gt->i915;
 92
 93	GT_TRACE(gt, "\n");
 94
 95	/* Wa_14017073508: mtl */
 96	mtl_media_busy(gt);
 97
 98	/*
 99	 * It seems that the DMC likes to transition between the DC states a lot
100	 * when there are no connected displays (no active power domains) during
101	 * command submission.
102	 *
103	 * This activity has negative impact on the performance of the chip with
104	 * huge latencies observed in the interrupt handler and elsewhere.
105	 *
106	 * Work around it by grabbing a GT IRQ power domain whilst there is any
107	 * GT activity, preventing any DC state transitions.
108	 */
109	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
110	GEM_BUG_ON(!gt->awake);
111
112	intel_rc6_unpark(&gt->rc6);
113	intel_rps_unpark(&gt->rps);
114	i915_pmu_gt_unparked(i915);
115	intel_guc_busyness_unpark(gt);
116
117	intel_gt_unpark_requests(gt);
118	runtime_begin(gt);
119
120	return 0;
121}
122
123static int __gt_park(struct intel_wakeref *wf)
124{
125	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
126	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
127	struct drm_i915_private *i915 = gt->i915;
128
129	GT_TRACE(gt, "\n");
130
131	runtime_end(gt);
132	intel_gt_park_requests(gt);
133
134	intel_guc_busyness_park(gt);
135	i915_vma_parked(gt);
136	i915_pmu_gt_parked(i915);
137	intel_rps_park(&gt->rps);
138	intel_rc6_park(&gt->rc6);
139
140	/* Everything switched off, flush any residual interrupt just in case */
141	intel_synchronize_irq(i915);
142
143	/* Defer dropping the display power well for 100ms, it's slow! */
144	GEM_BUG_ON(!wakeref);
145	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
146
147	/* Wa_14017073508: mtl */
148	mtl_media_idle(gt);
149
150	return 0;
151}
152
153static const struct intel_wakeref_ops wf_ops = {
154	.get = __gt_unpark,
155	.put = __gt_park,
156};
157
158void intel_gt_pm_init_early(struct intel_gt *gt)
159{
160	/*
161	 * We access the runtime_pm structure via gt->i915 here rather than
162	 * gt->uncore as we do elsewhere in the file because gt->uncore is not
163	 * yet initialized for all tiles at this point in the driver startup.
164	 * runtime_pm is per-device rather than per-tile, so this is still the
165	 * correct structure.
166	 */
167	intel_wakeref_init(&gt->wakeref, &gt->i915->runtime_pm, &wf_ops);
168	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
169}
170
171void intel_gt_pm_init(struct intel_gt *gt)
172{
173	/*
174	 * Enabling power-management should be "self-healing". If we cannot
175	 * enable a feature, simply leave it disabled with a notice to the
176	 * user.
177	 */
178	intel_rc6_init(&gt->rc6);
179	intel_rps_init(&gt->rps);
180}
181
182static bool reset_engines(struct intel_gt *gt)
183{
184	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
185		return false;
186
187	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
188}
189
190static void gt_sanitize(struct intel_gt *gt, bool force)
191{
192	struct intel_engine_cs *engine;
193	enum intel_engine_id id;
194	intel_wakeref_t wakeref;
195
196	GT_TRACE(gt, "force:%s", str_yes_no(force));
197
198	/* Use a raw wakeref to avoid calling intel_display_power_get early */
199	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
200	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
201
202	intel_gt_check_clock_frequency(gt);
203
204	/*
205	 * As we have just resumed the machine and woken the device up from
206	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
207	 * back to defaults, recovering from whatever wedged state we left it
208	 * in and so worth trying to use the device once more.
209	 */
210	if (intel_gt_is_wedged(gt))
211		intel_gt_unset_wedged(gt);
212
213	/* For GuC mode, ensure submission is disabled before stopping ring */
214	intel_uc_reset_prepare(&gt->uc);
215
216	for_each_engine(engine, gt, id) {
217		if (engine->reset.prepare)
218			engine->reset.prepare(engine);
219
220		if (engine->sanitize)
221			engine->sanitize(engine);
222	}
223
224	if (reset_engines(gt) || force) {
225		for_each_engine(engine, gt, id)
226			__intel_engine_reset(engine, false);
227	}
228
229	intel_uc_reset(&gt->uc, false);
230
231	for_each_engine(engine, gt, id)
232		if (engine->reset.finish)
233			engine->reset.finish(engine);
234
235	intel_rps_sanitize(&gt->rps);
236
237	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
238	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
239}
240
241void intel_gt_pm_fini(struct intel_gt *gt)
242{
243	intel_rc6_fini(&gt->rc6);
244}
245
246int intel_gt_resume(struct intel_gt *gt)
247{
248	struct intel_engine_cs *engine;
249	enum intel_engine_id id;
250	int err;
251
252	err = intel_gt_has_unrecoverable_error(gt);
253	if (err)
254		return err;
255
256	GT_TRACE(gt, "\n");
257
258	/*
259	 * After resume, we may need to poke into the pinned kernel
260	 * contexts to paper over any damage caused by the sudden suspend.
261	 * Only the kernel contexts should remain pinned over suspend,
262	 * allowing us to fixup the user contexts on their first pin.
263	 */
264	gt_sanitize(gt, true);
265
266	intel_gt_pm_get(gt);
267
268	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
269	intel_rc6_sanitize(&gt->rc6);
270	if (intel_gt_is_wedged(gt)) {
271		err = -EIO;
272		goto out_fw;
273	}
274
275	/* Only when the HW is re-initialised, can we replay the requests */
276	err = intel_gt_init_hw(gt);
277	if (err) {
278		i915_probe_error(gt->i915,
279				 "Failed to initialize GPU, declaring it wedged!\n");
280		goto err_wedged;
281	}
282
283	intel_uc_reset_finish(&gt->uc);
284
285	intel_rps_enable(&gt->rps);
286	intel_llc_enable(&gt->llc);
287
288	for_each_engine(engine, gt, id) {
289		intel_engine_pm_get(engine);
290
291		engine->serial++; /* kernel context lost */
292		err = intel_engine_resume(engine);
293
294		intel_engine_pm_put(engine);
295		if (err) {
296			drm_err(&gt->i915->drm,
297				"Failed to restart %s (%d)\n",
298				engine->name, err);
299			goto err_wedged;
300		}
301	}
302
303	intel_rc6_enable(&gt->rc6);
304
305	intel_uc_resume(&gt->uc);
306
307	intel_pxp_resume(&gt->pxp);
308
309	user_forcewake(gt, false);
310
311out_fw:
312	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
313	intel_gt_pm_put(gt);
314	return err;
315
316err_wedged:
317	intel_gt_set_wedged(gt);
318	goto out_fw;
319}
320
321static void wait_for_suspend(struct intel_gt *gt)
322{
323	if (!intel_gt_pm_is_awake(gt))
324		return;
325
326	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
327		/*
328		 * Forcibly cancel outstanding work and leave
329		 * the gpu quiet.
330		 */
331		intel_gt_set_wedged(gt);
332		intel_gt_retire_requests(gt);
333	}
334
335	intel_gt_pm_wait_for_idle(gt);
336}
337
338void intel_gt_suspend_prepare(struct intel_gt *gt)
339{
340	user_forcewake(gt, true);
341	wait_for_suspend(gt);
342
343	intel_pxp_suspend_prepare(&gt->pxp);
344}
345
346static suspend_state_t pm_suspend_target(void)
347{
348#if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
349	return pm_suspend_target_state;
350#else
351	return PM_SUSPEND_TO_IDLE;
352#endif
353}
354
355void intel_gt_suspend_late(struct intel_gt *gt)
356{
357	intel_wakeref_t wakeref;
358
359	/* We expect to be idle already; but also want to be independent */
360	wait_for_suspend(gt);
361
362	if (is_mock_gt(gt))
363		return;
364
365	GEM_BUG_ON(gt->awake);
366
367	intel_uc_suspend(&gt->uc);
368	intel_pxp_suspend(&gt->pxp);
369
370	/*
371	 * On disabling the device, we want to turn off HW access to memory
372	 * that we no longer own.
373	 *
374	 * However, not all suspend-states disable the device. S0 (s2idle)
375	 * is effectively runtime-suspend, the device is left powered on
376	 * but needs to be put into a low power state. We need to keep
377	 * powermanagement enabled, but we also retain system state and so
378	 * it remains safe to keep on using our allocated memory.
379	 */
380	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
381		return;
382
383	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
384		intel_rps_disable(&gt->rps);
385		intel_rc6_disable(&gt->rc6);
386		intel_llc_disable(&gt->llc);
387	}
388
389	gt_sanitize(gt, false);
390
391	GT_TRACE(gt, "\n");
392}
393
394void intel_gt_runtime_suspend(struct intel_gt *gt)
395{
396	intel_pxp_runtime_suspend(&gt->pxp);
397	intel_uc_runtime_suspend(&gt->uc);
398
399	GT_TRACE(gt, "\n");
400}
401
402int intel_gt_runtime_resume(struct intel_gt *gt)
403{
404	int ret;
405
406	GT_TRACE(gt, "\n");
407	intel_gt_init_swizzling(gt);
408	intel_ggtt_restore_fences(gt->ggtt);
409
410	ret = intel_uc_runtime_resume(&gt->uc);
411	if (ret)
412		return ret;
413
414	intel_pxp_runtime_resume(&gt->pxp);
415
416	return 0;
417}
418
419static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
420{
421	ktime_t total = gt->stats.total;
422
423	if (gt->stats.active)
424		total = ktime_add(total,
425				  ktime_sub(ktime_get(), gt->stats.start));
426
427	return total;
428}
429
430ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
431{
432	unsigned int seq;
433	ktime_t total;
434
435	do {
436		seq = read_seqcount_begin(&gt->stats.lock);
437		total = __intel_gt_get_awake_time(gt);
438	} while (read_seqcount_retry(&gt->stats.lock, seq));
439
440	return total;
441}
442
443#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
444#include "selftest_gt_pm.c"
445#endif