1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2019 Intel Corporation
  5 */
  6
  7#include "gem/i915_gem_pm.h"
  8#include "gem/i915_gem_ttm_pm.h"
  9#include "gt/intel_gt.h"
 10#include "gt/intel_gt_pm.h"
 11#include "gt/intel_gt_requests.h"
 12
 13#include "i915_driver.h"
 14#include "i915_drv.h"
 15
 16#if defined(CONFIG_X86)
 17#include <asm/smp.h>
 18#else
 19#define wbinvd_on_all_cpus() \
 20	pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
 21#endif
 22
 23void i915_gem_suspend(struct drm_i915_private *i915)
 24{
 25	struct intel_gt *gt;
 26	unsigned int i;
 27
 28	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
 29
 30	intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0);
 31	/*
 32	 * On rare occasions, we've observed the fence completion triggers
 33	 * free_engines asynchronously via rcu_call. Ensure those are done.
 34	 * This path is only called on suspend, so it's an acceptable cost.
 35	 */
 36	rcu_barrier();
 37
 38	flush_workqueue(i915->wq);
 39
 40	/*
 41	 * We have to flush all the executing contexts to main memory so
 42	 * that they can saved in the hibernation image. To ensure the last
 43	 * context image is coherent, we have to switch away from it. That
 44	 * leaves the i915->kernel_context still active when
 45	 * we actually suspend, and its image in memory may not match the GPU
 46	 * state. Fortunately, the kernel_context is disposable and we do
 47	 * not rely on its state.
 48	 */
 49	for_each_gt(gt, i915, i)
 50		intel_gt_suspend_prepare(gt);
 51
 52	i915_gem_drain_freed_objects(i915);
 53}
 54
 55static int lmem_restore(struct drm_i915_private *i915, u32 flags)
 56{
 57	struct intel_memory_region *mr;
 58	int ret = 0, id;
 59
 60	for_each_memory_region(mr, i915, id) {
 61		if (mr->type == INTEL_MEMORY_LOCAL) {
 62			ret = i915_ttm_restore_region(mr, flags);
 63			if (ret)
 64				break;
 65		}
 66	}
 67
 68	return ret;
 69}
 70
 71static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
 72{
 73	struct intel_memory_region *mr;
 74	int ret = 0, id;
 75
 76	for_each_memory_region(mr, i915, id) {
 77		if (mr->type == INTEL_MEMORY_LOCAL) {
 78			ret = i915_ttm_backup_region(mr, flags);
 79			if (ret)
 80				break;
 81		}
 82	}
 83
 84	return ret;
 85}
 86
 87static void lmem_recover(struct drm_i915_private *i915)
 88{
 89	struct intel_memory_region *mr;
 90	int id;
 91
 92	for_each_memory_region(mr, i915, id)
 93		if (mr->type == INTEL_MEMORY_LOCAL)
 94			i915_ttm_recover_region(mr);
 95}
 96
 97int i915_gem_backup_suspend(struct drm_i915_private *i915)
 98{
 99	int ret;
100
101	/* Opportunistically try to evict unpinned objects */
102	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
103	if (ret)
104		goto out_recover;
105
106	i915_gem_suspend(i915);
107
108	/*
109	 * More objects may have become unpinned as requests were
110	 * retired. Now try to evict again. The gt may be wedged here
111	 * in which case we automatically fall back to memcpy.
112	 * We allow also backing up pinned objects that have not been
113	 * marked for early recover, and that may contain, for example,
114	 * page-tables for the migrate context.
115	 */
116	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
117			   I915_TTM_BACKUP_PINNED);
118	if (ret)
119		goto out_recover;
120
121	/*
122	 * Remaining objects are backed up using memcpy once we've stopped
123	 * using the migrate context.
124	 */
125	ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
126	if (ret)
127		goto out_recover;
128
129	return 0;
130
131out_recover:
132	lmem_recover(i915);
133
134	return ret;
135}
136
137void i915_gem_suspend_late(struct drm_i915_private *i915)
138{
139	struct drm_i915_gem_object *obj;
140	struct list_head *phases[] = {
141		&i915->mm.shrink_list,
142		&i915->mm.purge_list,
143		NULL
144	}, **phase;
145	struct intel_gt *gt;
146	unsigned long flags;
147	unsigned int i;
148	bool flush = false;
149
150	/*
151	 * Neither the BIOS, ourselves or any other kernel
152	 * expects the system to be in execlists mode on startup,
153	 * so we need to reset the GPU back to legacy mode. And the only
154	 * known way to disable logical contexts is through a GPU reset.
155	 *
156	 * So in order to leave the system in a known default configuration,
157	 * always reset the GPU upon unload and suspend. Afterwards we then
158	 * clean up the GEM state tracking, flushing off the requests and
159	 * leaving the system in a known idle state.
160	 *
161	 * Note that is of the upmost importance that the GPU is idle and
162	 * all stray writes are flushed *before* we dismantle the backing
163	 * storage for the pinned objects.
164	 *
165	 * However, since we are uncertain that resetting the GPU on older
166	 * machines is a good idea, we don't - just in case it leaves the
167	 * machine in an unusable condition.
168	 */
169
170	/* Like i915_gem_suspend, flush tasks staged from fence triggers */
171	rcu_barrier();
172
173	for_each_gt(gt, i915, i)
174		intel_gt_suspend_late(gt);
175
176	spin_lock_irqsave(&i915->mm.obj_lock, flags);
177	for (phase = phases; *phase; phase++) {
178		list_for_each_entry(obj, *phase, mm.link) {
179			if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
180				flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
181			__start_cpu_write(obj); /* presume auto-hibernate */
182		}
183	}
184	spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
185	if (flush)
186		wbinvd_on_all_cpus();
187}
188
189int i915_gem_freeze(struct drm_i915_private *i915)
190{
191	/* Discard all purgeable objects, let userspace recover those as
192	 * required after resuming.
193	 */
194	i915_gem_shrink_all(i915);
195
196	return 0;
197}
198
199int i915_gem_freeze_late(struct drm_i915_private *i915)
200{
201	struct drm_i915_gem_object *obj;
202	intel_wakeref_t wakeref;
203
204	/*
205	 * Called just before we write the hibernation image.
206	 *
207	 * We need to update the domain tracking to reflect that the CPU
208	 * will be accessing all the pages to create and restore from the
209	 * hibernation, and so upon restoration those pages will be in the
210	 * CPU domain.
211	 *
212	 * To make sure the hibernation image contains the latest state,
213	 * we update that state just before writing out the image.
214	 *
215	 * To try and reduce the hibernation image, we manually shrink
216	 * the objects as well, see i915_gem_freeze()
217	 */
218
219	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
220		i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
221	i915_gem_drain_freed_objects(i915);
222
223	wbinvd_on_all_cpus();
224	list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
225		__start_cpu_write(obj);
226
227	return 0;
228}
229
230void i915_gem_resume(struct drm_i915_private *i915)
231{
232	struct intel_gt *gt;
233	int ret, i, j;
234
235	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
236
237	ret = lmem_restore(i915, 0);
238	GEM_WARN_ON(ret);
239
240	/*
241	 * As we didn't flush the kernel context before suspend, we cannot
242	 * guarantee that the context image is complete. So let's just reset
243	 * it and start again.
244	 */
245	for_each_gt(gt, i915, i)
246		if (intel_gt_resume(gt))
247			goto err_wedged;
248
249	ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
250	GEM_WARN_ON(ret);
251
252	return;
253
254err_wedged:
255	for_each_gt(gt, i915, j) {
256		if (!intel_gt_is_wedged(gt)) {
257			dev_err(i915->drm.dev,
258				"Failed to re-initialize GPU[%u], declaring it wedged!\n",
259				j);
260			intel_gt_set_wedged(gt);
261		}
262
263		if (j == i)
264			break;
265	}
266}

  1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2019 Intel Corporation
  5 */
  6
  7#include "gem/i915_gem_pm.h"
  8#include "gem/i915_gem_ttm_pm.h"
  9#include "gt/intel_gt.h"
 10#include "gt/intel_gt_pm.h"
 11#include "gt/intel_gt_requests.h"
 12
 13#include "i915_driver.h"
 14#include "i915_drv.h"
 15
 16#if defined(CONFIG_X86)
 17#include <asm/smp.h>
 18#else
 19#define wbinvd_on_all_cpus() \
 20	pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
 21#endif
 22
 23void i915_gem_suspend(struct drm_i915_private *i915)
 24{
 25	struct intel_gt *gt;
 26	unsigned int i;
 27
 28	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
 29
 30	intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0);
 
 
 
 
 
 
 
 31	flush_workqueue(i915->wq);
 32
 33	/*
 34	 * We have to flush all the executing contexts to main memory so
 35	 * that they can saved in the hibernation image. To ensure the last
 36	 * context image is coherent, we have to switch away from it. That
 37	 * leaves the i915->kernel_context still active when
 38	 * we actually suspend, and its image in memory may not match the GPU
 39	 * state. Fortunately, the kernel_context is disposable and we do
 40	 * not rely on its state.
 41	 */
 42	for_each_gt(gt, i915, i)
 43		intel_gt_suspend_prepare(gt);
 44
 45	i915_gem_drain_freed_objects(i915);
 46}
 47
 48static int lmem_restore(struct drm_i915_private *i915, u32 flags)
 49{
 50	struct intel_memory_region *mr;
 51	int ret = 0, id;
 52
 53	for_each_memory_region(mr, i915, id) {
 54		if (mr->type == INTEL_MEMORY_LOCAL) {
 55			ret = i915_ttm_restore_region(mr, flags);
 56			if (ret)
 57				break;
 58		}
 59	}
 60
 61	return ret;
 62}
 63
 64static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
 65{
 66	struct intel_memory_region *mr;
 67	int ret = 0, id;
 68
 69	for_each_memory_region(mr, i915, id) {
 70		if (mr->type == INTEL_MEMORY_LOCAL) {
 71			ret = i915_ttm_backup_region(mr, flags);
 72			if (ret)
 73				break;
 74		}
 75	}
 76
 77	return ret;
 78}
 79
 80static void lmem_recover(struct drm_i915_private *i915)
 81{
 82	struct intel_memory_region *mr;
 83	int id;
 84
 85	for_each_memory_region(mr, i915, id)
 86		if (mr->type == INTEL_MEMORY_LOCAL)
 87			i915_ttm_recover_region(mr);
 88}
 89
 90int i915_gem_backup_suspend(struct drm_i915_private *i915)
 91{
 92	int ret;
 93
 94	/* Opportunistically try to evict unpinned objects */
 95	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
 96	if (ret)
 97		goto out_recover;
 98
 99	i915_gem_suspend(i915);
100
101	/*
102	 * More objects may have become unpinned as requests were
103	 * retired. Now try to evict again. The gt may be wedged here
104	 * in which case we automatically fall back to memcpy.
105	 * We allow also backing up pinned objects that have not been
106	 * marked for early recover, and that may contain, for example,
107	 * page-tables for the migrate context.
108	 */
109	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
110			   I915_TTM_BACKUP_PINNED);
111	if (ret)
112		goto out_recover;
113
114	/*
115	 * Remaining objects are backed up using memcpy once we've stopped
116	 * using the migrate context.
117	 */
118	ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
119	if (ret)
120		goto out_recover;
121
122	return 0;
123
124out_recover:
125	lmem_recover(i915);
126
127	return ret;
128}
129
130void i915_gem_suspend_late(struct drm_i915_private *i915)
131{
132	struct drm_i915_gem_object *obj;
133	struct list_head *phases[] = {
134		&i915->mm.shrink_list,
135		&i915->mm.purge_list,
136		NULL
137	}, **phase;
138	struct intel_gt *gt;
139	unsigned long flags;
140	unsigned int i;
141	bool flush = false;
142
143	/*
144	 * Neither the BIOS, ourselves or any other kernel
145	 * expects the system to be in execlists mode on startup,
146	 * so we need to reset the GPU back to legacy mode. And the only
147	 * known way to disable logical contexts is through a GPU reset.
148	 *
149	 * So in order to leave the system in a known default configuration,
150	 * always reset the GPU upon unload and suspend. Afterwards we then
151	 * clean up the GEM state tracking, flushing off the requests and
152	 * leaving the system in a known idle state.
153	 *
154	 * Note that is of the upmost importance that the GPU is idle and
155	 * all stray writes are flushed *before* we dismantle the backing
156	 * storage for the pinned objects.
157	 *
158	 * However, since we are uncertain that resetting the GPU on older
159	 * machines is a good idea, we don't - just in case it leaves the
160	 * machine in an unusable condition.
161	 */
 
 
 
162
163	for_each_gt(gt, i915, i)
164		intel_gt_suspend_late(gt);
165
166	spin_lock_irqsave(&i915->mm.obj_lock, flags);
167	for (phase = phases; *phase; phase++) {
168		list_for_each_entry(obj, *phase, mm.link) {
169			if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
170				flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
171			__start_cpu_write(obj); /* presume auto-hibernate */
172		}
173	}
174	spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
175	if (flush)
176		wbinvd_on_all_cpus();
177}
178
179int i915_gem_freeze(struct drm_i915_private *i915)
180{
181	/* Discard all purgeable objects, let userspace recover those as
182	 * required after resuming.
183	 */
184	i915_gem_shrink_all(i915);
185
186	return 0;
187}
188
189int i915_gem_freeze_late(struct drm_i915_private *i915)
190{
191	struct drm_i915_gem_object *obj;
192	intel_wakeref_t wakeref;
193
194	/*
195	 * Called just before we write the hibernation image.
196	 *
197	 * We need to update the domain tracking to reflect that the CPU
198	 * will be accessing all the pages to create and restore from the
199	 * hibernation, and so upon restoration those pages will be in the
200	 * CPU domain.
201	 *
202	 * To make sure the hibernation image contains the latest state,
203	 * we update that state just before writing out the image.
204	 *
205	 * To try and reduce the hibernation image, we manually shrink
206	 * the objects as well, see i915_gem_freeze()
207	 */
208
209	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
210		i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
211	i915_gem_drain_freed_objects(i915);
212
213	wbinvd_on_all_cpus();
214	list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
215		__start_cpu_write(obj);
216
217	return 0;
218}
219
220void i915_gem_resume(struct drm_i915_private *i915)
221{
222	struct intel_gt *gt;
223	int ret, i, j;
224
225	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
226
227	ret = lmem_restore(i915, 0);
228	GEM_WARN_ON(ret);
229
230	/*
231	 * As we didn't flush the kernel context before suspend, we cannot
232	 * guarantee that the context image is complete. So let's just reset
233	 * it and start again.
234	 */
235	for_each_gt(gt, i915, i)
236		if (intel_gt_resume(gt))
237			goto err_wedged;
238
239	ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
240	GEM_WARN_ON(ret);
241
242	return;
243
244err_wedged:
245	for_each_gt(gt, i915, j) {
246		if (!intel_gt_is_wedged(gt)) {
247			dev_err(i915->drm.dev,
248				"Failed to re-initialize GPU[%u], declaring it wedged!\n",
249				j);
250			intel_gt_set_wedged(gt);
251		}
252
253		if (j == i)
254			break;
255	}
256}