Loading...
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/oom.h>
26#include <linux/shmem_fs.h>
27#include <linux/slab.h>
28#include <linux/swap.h>
29#include <linux/pci.h>
30#include <linux/dma-buf.h>
31#include <drm/drmP.h>
32#include <drm/i915_drm.h>
33
34#include "i915_drv.h"
35#include "i915_trace.h"
36
37static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
38{
39 if (!mutex_is_locked(mutex))
40 return false;
41
42#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
43 return mutex->owner == task;
44#else
45 /* Since UP may be pre-empted, we cannot assume that we own the lock */
46 return false;
47#endif
48}
49
50static int num_vma_bound(struct drm_i915_gem_object *obj)
51{
52 struct i915_vma *vma;
53 int count = 0;
54
55 list_for_each_entry(vma, &obj->vma_list, obj_link) {
56 if (drm_mm_node_allocated(&vma->node))
57 count++;
58 if (vma->pin_count)
59 count++;
60 }
61
62 return count;
63}
64
65static bool swap_available(void)
66{
67 return get_nr_swap_pages() > 0;
68}
69
70static bool can_release_pages(struct drm_i915_gem_object *obj)
71{
72 /* Only report true if by unbinding the object and putting its pages
73 * we can actually make forward progress towards freeing physical
74 * pages.
75 *
76 * If the pages are pinned for any other reason than being bound
77 * to the GPU, simply unbinding from the GPU is not going to succeed
78 * in releasing our pin count on the pages themselves.
79 */
80 if (obj->pages_pin_count != num_vma_bound(obj))
81 return false;
82
83 /* We can only return physical pages to the system if we can either
84 * discard the contents (because the user has marked them as being
85 * purgeable) or if we can move their contents out to swap.
86 */
87 return swap_available() || obj->madv == I915_MADV_DONTNEED;
88}
89
90/**
91 * i915_gem_shrink - Shrink buffer object caches
92 * @dev_priv: i915 device
93 * @target: amount of memory to make available, in pages
94 * @flags: control flags for selecting cache types
95 *
96 * This function is the main interface to the shrinker. It will try to release
97 * up to @target pages of main memory backing storage from buffer objects.
98 * Selection of the specific caches can be done with @flags. This is e.g. useful
99 * when purgeable objects should be removed from caches preferentially.
100 *
101 * Note that it's not guaranteed that released amount is actually available as
102 * free system memory - the pages might still be in-used to due to other reasons
103 * (like cpu mmaps) or the mm core has reused them before we could grab them.
104 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
105 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
106 *
107 * Also note that any kind of pinning (both per-vma address space pins and
108 * backing storage pins at the buffer object level) result in the shrinker code
109 * having to skip the object.
110 *
111 * Returns:
112 * The number of pages of backing storage actually released.
113 */
114unsigned long
115i915_gem_shrink(struct drm_i915_private *dev_priv,
116 unsigned long target, unsigned flags)
117{
118 const struct {
119 struct list_head *list;
120 unsigned int bit;
121 } phases[] = {
122 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
123 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
124 { NULL, 0 },
125 }, *phase;
126 unsigned long count = 0;
127
128 trace_i915_gem_shrink(dev_priv, target, flags);
129 i915_gem_retire_requests(dev_priv->dev);
130
131 /*
132 * As we may completely rewrite the (un)bound list whilst unbinding
133 * (due to retiring requests) we have to strictly process only
134 * one element of the list at the time, and recheck the list
135 * on every iteration.
136 *
137 * In particular, we must hold a reference whilst removing the
138 * object as we may end up waiting for and/or retiring the objects.
139 * This might release the final reference (held by the active list)
140 * and result in the object being freed from under us. This is
141 * similar to the precautions the eviction code must take whilst
142 * removing objects.
143 *
144 * Also note that although these lists do not hold a reference to
145 * the object we can safely grab one here: The final object
146 * unreferencing and the bound_list are both protected by the
147 * dev->struct_mutex and so we won't ever be able to observe an
148 * object on the bound_list with a reference count equals 0.
149 */
150 for (phase = phases; phase->list; phase++) {
151 struct list_head still_in_list;
152
153 if ((flags & phase->bit) == 0)
154 continue;
155
156 INIT_LIST_HEAD(&still_in_list);
157 while (count < target && !list_empty(phase->list)) {
158 struct drm_i915_gem_object *obj;
159 struct i915_vma *vma, *v;
160
161 obj = list_first_entry(phase->list,
162 typeof(*obj), global_list);
163 list_move_tail(&obj->global_list, &still_in_list);
164
165 if (flags & I915_SHRINK_PURGEABLE &&
166 obj->madv != I915_MADV_DONTNEED)
167 continue;
168
169 if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active)
170 continue;
171
172 if (!can_release_pages(obj))
173 continue;
174
175 drm_gem_object_reference(&obj->base);
176
177 /* For the unbound phase, this should be a no-op! */
178 list_for_each_entry_safe(vma, v,
179 &obj->vma_list, obj_link)
180 if (i915_vma_unbind(vma))
181 break;
182
183 if (i915_gem_object_put_pages(obj) == 0)
184 count += obj->base.size >> PAGE_SHIFT;
185
186 drm_gem_object_unreference(&obj->base);
187 }
188 list_splice(&still_in_list, phase->list);
189 }
190
191 i915_gem_retire_requests(dev_priv->dev);
192
193 return count;
194}
195
196/**
197 * i915_gem_shrink_all - Shrink buffer object caches completely
198 * @dev_priv: i915 device
199 *
200 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
201 * caches completely. It also first waits for and retires all outstanding
202 * requests to also be able to release backing storage for active objects.
203 *
204 * This should only be used in code to intentionally quiescent the gpu or as a
205 * last-ditch effort when memory seems to have run out.
206 *
207 * Returns:
208 * The number of pages of backing storage actually released.
209 */
210unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
211{
212 return i915_gem_shrink(dev_priv, -1UL,
213 I915_SHRINK_BOUND |
214 I915_SHRINK_UNBOUND |
215 I915_SHRINK_ACTIVE);
216}
217
218static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
219{
220 if (!mutex_trylock(&dev->struct_mutex)) {
221 if (!mutex_is_locked_by(&dev->struct_mutex, current))
222 return false;
223
224 if (to_i915(dev)->mm.shrinker_no_lock_stealing)
225 return false;
226
227 *unlock = false;
228 } else
229 *unlock = true;
230
231 return true;
232}
233
234static unsigned long
235i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
236{
237 struct drm_i915_private *dev_priv =
238 container_of(shrinker, struct drm_i915_private, mm.shrinker);
239 struct drm_device *dev = dev_priv->dev;
240 struct drm_i915_gem_object *obj;
241 unsigned long count;
242 bool unlock;
243
244 if (!i915_gem_shrinker_lock(dev, &unlock))
245 return 0;
246
247 count = 0;
248 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
249 if (obj->pages_pin_count == 0)
250 count += obj->base.size >> PAGE_SHIFT;
251
252 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
253 if (!obj->active && can_release_pages(obj))
254 count += obj->base.size >> PAGE_SHIFT;
255 }
256
257 if (unlock)
258 mutex_unlock(&dev->struct_mutex);
259
260 return count;
261}
262
263static unsigned long
264i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
265{
266 struct drm_i915_private *dev_priv =
267 container_of(shrinker, struct drm_i915_private, mm.shrinker);
268 struct drm_device *dev = dev_priv->dev;
269 unsigned long freed;
270 bool unlock;
271
272 if (!i915_gem_shrinker_lock(dev, &unlock))
273 return SHRINK_STOP;
274
275 freed = i915_gem_shrink(dev_priv,
276 sc->nr_to_scan,
277 I915_SHRINK_BOUND |
278 I915_SHRINK_UNBOUND |
279 I915_SHRINK_PURGEABLE);
280 if (freed < sc->nr_to_scan)
281 freed += i915_gem_shrink(dev_priv,
282 sc->nr_to_scan - freed,
283 I915_SHRINK_BOUND |
284 I915_SHRINK_UNBOUND);
285 if (unlock)
286 mutex_unlock(&dev->struct_mutex);
287
288 return freed;
289}
290
291static int
292i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
293{
294 struct drm_i915_private *dev_priv =
295 container_of(nb, struct drm_i915_private, mm.oom_notifier);
296 struct drm_device *dev = dev_priv->dev;
297 struct drm_i915_gem_object *obj;
298 unsigned long timeout = msecs_to_jiffies(5000) + 1;
299 unsigned long pinned, bound, unbound, freed_pages;
300 bool was_interruptible;
301 bool unlock;
302
303 while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) {
304 schedule_timeout_killable(1);
305 if (fatal_signal_pending(current))
306 return NOTIFY_DONE;
307 }
308 if (timeout == 0) {
309 pr_err("Unable to purge GPU memory due lock contention.\n");
310 return NOTIFY_DONE;
311 }
312
313 was_interruptible = dev_priv->mm.interruptible;
314 dev_priv->mm.interruptible = false;
315
316 freed_pages = i915_gem_shrink_all(dev_priv);
317
318 dev_priv->mm.interruptible = was_interruptible;
319
320 /* Because we may be allocating inside our own driver, we cannot
321 * assert that there are no objects with pinned pages that are not
322 * being pointed to by hardware.
323 */
324 unbound = bound = pinned = 0;
325 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
326 if (!obj->base.filp) /* not backed by a freeable object */
327 continue;
328
329 if (obj->pages_pin_count)
330 pinned += obj->base.size;
331 else
332 unbound += obj->base.size;
333 }
334 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
335 if (!obj->base.filp)
336 continue;
337
338 if (obj->pages_pin_count)
339 pinned += obj->base.size;
340 else
341 bound += obj->base.size;
342 }
343
344 if (unlock)
345 mutex_unlock(&dev->struct_mutex);
346
347 if (freed_pages || unbound || bound)
348 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
349 freed_pages << PAGE_SHIFT, pinned);
350 if (unbound || bound)
351 pr_err("%lu and %lu bytes still available in the "
352 "bound and unbound GPU page lists.\n",
353 bound, unbound);
354
355 *(unsigned long *)ptr += freed_pages;
356 return NOTIFY_DONE;
357}
358
359/**
360 * i915_gem_shrinker_init - Initialize i915 shrinker
361 * @dev_priv: i915 device
362 *
363 * This function registers and sets up the i915 shrinker and OOM handler.
364 */
365void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
366{
367 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
368 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
369 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
370 WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
371
372 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
373 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
374}
375
376/**
377 * i915_gem_shrinker_cleanup - Clean up i915 shrinker
378 * @dev_priv: i915 device
379 *
380 * This function unregisters the i915 shrinker and OOM handler.
381 */
382void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
383{
384 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
385 unregister_shrinker(&dev_priv->mm.shrinker);
386}
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/oom.h>
26#include <linux/shmem_fs.h>
27#include <linux/slab.h>
28#include <linux/swap.h>
29#include <linux/pci.h>
30#include <linux/dma-buf.h>
31#include <linux/vmalloc.h>
32#include <drm/drmP.h>
33#include <drm/i915_drm.h>
34
35#include "i915_drv.h"
36#include "i915_trace.h"
37
38static bool shrinker_lock(struct drm_i915_private *i915, bool *unlock)
39{
40 switch (mutex_trylock_recursive(&i915->drm.struct_mutex)) {
41 case MUTEX_TRYLOCK_RECURSIVE:
42 *unlock = false;
43 return true;
44
45 case MUTEX_TRYLOCK_FAILED:
46 *unlock = false;
47 preempt_disable();
48 do {
49 cpu_relax();
50 if (mutex_trylock(&i915->drm.struct_mutex)) {
51 *unlock = true;
52 break;
53 }
54 } while (!need_resched());
55 preempt_enable();
56 return *unlock;
57
58 case MUTEX_TRYLOCK_SUCCESS:
59 *unlock = true;
60 return true;
61 }
62
63 BUG();
64}
65
66static void shrinker_unlock(struct drm_i915_private *i915, bool unlock)
67{
68 if (!unlock)
69 return;
70
71 mutex_unlock(&i915->drm.struct_mutex);
72}
73
74static bool swap_available(void)
75{
76 return get_nr_swap_pages() > 0;
77}
78
79static bool can_release_pages(struct drm_i915_gem_object *obj)
80{
81 /* Consider only shrinkable ojects. */
82 if (!i915_gem_object_is_shrinkable(obj))
83 return false;
84
85 /* Only report true if by unbinding the object and putting its pages
86 * we can actually make forward progress towards freeing physical
87 * pages.
88 *
89 * If the pages are pinned for any other reason than being bound
90 * to the GPU, simply unbinding from the GPU is not going to succeed
91 * in releasing our pin count on the pages themselves.
92 */
93 if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
94 return false;
95
96 /* If any vma are "permanently" pinned, it will prevent us from
97 * reclaiming the obj->mm.pages. We only allow scanout objects to claim
98 * a permanent pin, along with a few others like the context objects.
99 * To simplify the scan, and to avoid walking the list of vma under the
100 * object, we just check the count of its permanently pinned.
101 */
102 if (READ_ONCE(obj->pin_global))
103 return false;
104
105 /* We can only return physical pages to the system if we can either
106 * discard the contents (because the user has marked them as being
107 * purgeable) or if we can move their contents out to swap.
108 */
109 return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
110}
111
112static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
113{
114 if (i915_gem_object_unbind(obj) == 0)
115 __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
116 return !i915_gem_object_has_pages(obj);
117}
118
119/**
120 * i915_gem_shrink - Shrink buffer object caches
121 * @i915: i915 device
122 * @target: amount of memory to make available, in pages
123 * @nr_scanned: optional output for number of pages scanned (incremental)
124 * @flags: control flags for selecting cache types
125 *
126 * This function is the main interface to the shrinker. It will try to release
127 * up to @target pages of main memory backing storage from buffer objects.
128 * Selection of the specific caches can be done with @flags. This is e.g. useful
129 * when purgeable objects should be removed from caches preferentially.
130 *
131 * Note that it's not guaranteed that released amount is actually available as
132 * free system memory - the pages might still be in-used to due to other reasons
133 * (like cpu mmaps) or the mm core has reused them before we could grab them.
134 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
135 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
136 *
137 * Also note that any kind of pinning (both per-vma address space pins and
138 * backing storage pins at the buffer object level) result in the shrinker code
139 * having to skip the object.
140 *
141 * Returns:
142 * The number of pages of backing storage actually released.
143 */
144unsigned long
145i915_gem_shrink(struct drm_i915_private *i915,
146 unsigned long target,
147 unsigned long *nr_scanned,
148 unsigned flags)
149{
150 const struct {
151 struct list_head *list;
152 unsigned int bit;
153 } phases[] = {
154 { &i915->mm.unbound_list, I915_SHRINK_UNBOUND },
155 { &i915->mm.bound_list, I915_SHRINK_BOUND },
156 { NULL, 0 },
157 }, *phase;
158 unsigned long count = 0;
159 unsigned long scanned = 0;
160 bool unlock;
161
162 if (!shrinker_lock(i915, &unlock))
163 return 0;
164
165 /*
166 * When shrinking the active list, also consider active contexts.
167 * Active contexts are pinned until they are retired, and so can
168 * not be simply unbound to retire and unpin their pages. To shrink
169 * the contexts, we must wait until the gpu is idle.
170 *
171 * We don't care about errors here; if we cannot wait upon the GPU,
172 * we will free as much as we can and hope to get a second chance.
173 */
174 if (flags & I915_SHRINK_ACTIVE)
175 i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
176
177 trace_i915_gem_shrink(i915, target, flags);
178 i915_retire_requests(i915);
179
180 /*
181 * Unbinding of objects will require HW access; Let us not wake the
182 * device just to recover a little memory. If absolutely necessary,
183 * we will force the wake during oom-notifier.
184 */
185 if ((flags & I915_SHRINK_BOUND) &&
186 !intel_runtime_pm_get_if_in_use(i915))
187 flags &= ~I915_SHRINK_BOUND;
188
189 /*
190 * As we may completely rewrite the (un)bound list whilst unbinding
191 * (due to retiring requests) we have to strictly process only
192 * one element of the list at the time, and recheck the list
193 * on every iteration.
194 *
195 * In particular, we must hold a reference whilst removing the
196 * object as we may end up waiting for and/or retiring the objects.
197 * This might release the final reference (held by the active list)
198 * and result in the object being freed from under us. This is
199 * similar to the precautions the eviction code must take whilst
200 * removing objects.
201 *
202 * Also note that although these lists do not hold a reference to
203 * the object we can safely grab one here: The final object
204 * unreferencing and the bound_list are both protected by the
205 * dev->struct_mutex and so we won't ever be able to observe an
206 * object on the bound_list with a reference count equals 0.
207 */
208 for (phase = phases; phase->list; phase++) {
209 struct list_head still_in_list;
210 struct drm_i915_gem_object *obj;
211
212 if ((flags & phase->bit) == 0)
213 continue;
214
215 INIT_LIST_HEAD(&still_in_list);
216
217 /*
218 * We serialize our access to unreferenced objects through
219 * the use of the struct_mutex. While the objects are not
220 * yet freed (due to RCU then a workqueue) we still want
221 * to be able to shrink their pages, so they remain on
222 * the unbound/bound list until actually freed.
223 */
224 spin_lock(&i915->mm.obj_lock);
225 while (count < target &&
226 (obj = list_first_entry_or_null(phase->list,
227 typeof(*obj),
228 mm.link))) {
229 list_move_tail(&obj->mm.link, &still_in_list);
230
231 if (flags & I915_SHRINK_PURGEABLE &&
232 obj->mm.madv != I915_MADV_DONTNEED)
233 continue;
234
235 if (flags & I915_SHRINK_VMAPS &&
236 !is_vmalloc_addr(obj->mm.mapping))
237 continue;
238
239 if (!(flags & I915_SHRINK_ACTIVE) &&
240 (i915_gem_object_is_active(obj) ||
241 i915_gem_object_is_framebuffer(obj)))
242 continue;
243
244 if (!can_release_pages(obj))
245 continue;
246
247 spin_unlock(&i915->mm.obj_lock);
248
249 if (unsafe_drop_pages(obj)) {
250 /* May arrive from get_pages on another bo */
251 mutex_lock_nested(&obj->mm.lock,
252 I915_MM_SHRINKER);
253 if (!i915_gem_object_has_pages(obj)) {
254 __i915_gem_object_invalidate(obj);
255 count += obj->base.size >> PAGE_SHIFT;
256 }
257 mutex_unlock(&obj->mm.lock);
258 }
259 scanned += obj->base.size >> PAGE_SHIFT;
260
261 spin_lock(&i915->mm.obj_lock);
262 }
263 list_splice_tail(&still_in_list, phase->list);
264 spin_unlock(&i915->mm.obj_lock);
265 }
266
267 if (flags & I915_SHRINK_BOUND)
268 intel_runtime_pm_put(i915);
269
270 i915_retire_requests(i915);
271
272 shrinker_unlock(i915, unlock);
273
274 if (nr_scanned)
275 *nr_scanned += scanned;
276 return count;
277}
278
279/**
280 * i915_gem_shrink_all - Shrink buffer object caches completely
281 * @i915: i915 device
282 *
283 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
284 * caches completely. It also first waits for and retires all outstanding
285 * requests to also be able to release backing storage for active objects.
286 *
287 * This should only be used in code to intentionally quiescent the gpu or as a
288 * last-ditch effort when memory seems to have run out.
289 *
290 * Returns:
291 * The number of pages of backing storage actually released.
292 */
293unsigned long i915_gem_shrink_all(struct drm_i915_private *i915)
294{
295 unsigned long freed;
296
297 intel_runtime_pm_get(i915);
298 freed = i915_gem_shrink(i915, -1UL, NULL,
299 I915_SHRINK_BOUND |
300 I915_SHRINK_UNBOUND |
301 I915_SHRINK_ACTIVE);
302 intel_runtime_pm_put(i915);
303
304 return freed;
305}
306
307static unsigned long
308i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
309{
310 struct drm_i915_private *i915 =
311 container_of(shrinker, struct drm_i915_private, mm.shrinker);
312 struct drm_i915_gem_object *obj;
313 unsigned long num_objects = 0;
314 unsigned long count = 0;
315
316 spin_lock(&i915->mm.obj_lock);
317 list_for_each_entry(obj, &i915->mm.unbound_list, mm.link)
318 if (can_release_pages(obj)) {
319 count += obj->base.size >> PAGE_SHIFT;
320 num_objects++;
321 }
322
323 list_for_each_entry(obj, &i915->mm.bound_list, mm.link)
324 if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) {
325 count += obj->base.size >> PAGE_SHIFT;
326 num_objects++;
327 }
328 spin_unlock(&i915->mm.obj_lock);
329
330 /* Update our preferred vmscan batch size for the next pass.
331 * Our rough guess for an effective batch size is roughly 2
332 * available GEM objects worth of pages. That is we don't want
333 * the shrinker to fire, until it is worth the cost of freeing an
334 * entire GEM object.
335 */
336 if (num_objects) {
337 unsigned long avg = 2 * count / num_objects;
338
339 i915->mm.shrinker.batch =
340 max((i915->mm.shrinker.batch + avg) >> 1,
341 128ul /* default SHRINK_BATCH */);
342 }
343
344 return count;
345}
346
347static unsigned long
348i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
349{
350 struct drm_i915_private *i915 =
351 container_of(shrinker, struct drm_i915_private, mm.shrinker);
352 unsigned long freed;
353 bool unlock;
354
355 sc->nr_scanned = 0;
356
357 if (!shrinker_lock(i915, &unlock))
358 return SHRINK_STOP;
359
360 freed = i915_gem_shrink(i915,
361 sc->nr_to_scan,
362 &sc->nr_scanned,
363 I915_SHRINK_BOUND |
364 I915_SHRINK_UNBOUND |
365 I915_SHRINK_PURGEABLE);
366 if (sc->nr_scanned < sc->nr_to_scan)
367 freed += i915_gem_shrink(i915,
368 sc->nr_to_scan - sc->nr_scanned,
369 &sc->nr_scanned,
370 I915_SHRINK_BOUND |
371 I915_SHRINK_UNBOUND);
372 if (sc->nr_scanned < sc->nr_to_scan && current_is_kswapd()) {
373 intel_runtime_pm_get(i915);
374 freed += i915_gem_shrink(i915,
375 sc->nr_to_scan - sc->nr_scanned,
376 &sc->nr_scanned,
377 I915_SHRINK_ACTIVE |
378 I915_SHRINK_BOUND |
379 I915_SHRINK_UNBOUND);
380 intel_runtime_pm_put(i915);
381 }
382
383 shrinker_unlock(i915, unlock);
384
385 return sc->nr_scanned ? freed : SHRINK_STOP;
386}
387
388static bool
389shrinker_lock_uninterruptible(struct drm_i915_private *i915, bool *unlock,
390 int timeout_ms)
391{
392 unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
393
394 do {
395 if (i915_gem_wait_for_idle(i915, 0) == 0 &&
396 shrinker_lock(i915, unlock))
397 break;
398
399 schedule_timeout_killable(1);
400 if (fatal_signal_pending(current))
401 return false;
402
403 if (time_after(jiffies, timeout)) {
404 pr_err("Unable to lock GPU to purge memory.\n");
405 return false;
406 }
407 } while (1);
408
409 return true;
410}
411
412static int
413i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
414{
415 struct drm_i915_private *i915 =
416 container_of(nb, struct drm_i915_private, mm.oom_notifier);
417 struct drm_i915_gem_object *obj;
418 unsigned long unevictable, bound, unbound, freed_pages;
419
420 freed_pages = i915_gem_shrink_all(i915);
421
422 /* Because we may be allocating inside our own driver, we cannot
423 * assert that there are no objects with pinned pages that are not
424 * being pointed to by hardware.
425 */
426 unbound = bound = unevictable = 0;
427 spin_lock(&i915->mm.obj_lock);
428 list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) {
429 if (!can_release_pages(obj))
430 unevictable += obj->base.size >> PAGE_SHIFT;
431 else
432 unbound += obj->base.size >> PAGE_SHIFT;
433 }
434 list_for_each_entry(obj, &i915->mm.bound_list, mm.link) {
435 if (!can_release_pages(obj))
436 unevictable += obj->base.size >> PAGE_SHIFT;
437 else
438 bound += obj->base.size >> PAGE_SHIFT;
439 }
440 spin_unlock(&i915->mm.obj_lock);
441
442 if (freed_pages || unbound || bound)
443 pr_info("Purging GPU memory, %lu pages freed, "
444 "%lu pages still pinned.\n",
445 freed_pages, unevictable);
446 if (unbound || bound)
447 pr_err("%lu and %lu pages still available in the "
448 "bound and unbound GPU page lists.\n",
449 bound, unbound);
450
451 *(unsigned long *)ptr += freed_pages;
452 return NOTIFY_DONE;
453}
454
455static int
456i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
457{
458 struct drm_i915_private *i915 =
459 container_of(nb, struct drm_i915_private, mm.vmap_notifier);
460 struct i915_vma *vma, *next;
461 unsigned long freed_pages = 0;
462 bool unlock;
463 int ret;
464
465 if (!shrinker_lock_uninterruptible(i915, &unlock, 5000))
466 return NOTIFY_DONE;
467
468 /* Force everything onto the inactive lists */
469 ret = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
470 if (ret)
471 goto out;
472
473 intel_runtime_pm_get(i915);
474 freed_pages += i915_gem_shrink(i915, -1UL, NULL,
475 I915_SHRINK_BOUND |
476 I915_SHRINK_UNBOUND |
477 I915_SHRINK_ACTIVE |
478 I915_SHRINK_VMAPS);
479 intel_runtime_pm_put(i915);
480
481 /* We also want to clear any cached iomaps as they wrap vmap */
482 list_for_each_entry_safe(vma, next,
483 &i915->ggtt.base.inactive_list, vm_link) {
484 unsigned long count = vma->node.size >> PAGE_SHIFT;
485 if (vma->iomap && i915_vma_unbind(vma) == 0)
486 freed_pages += count;
487 }
488
489out:
490 shrinker_unlock(i915, unlock);
491
492 *(unsigned long *)ptr += freed_pages;
493 return NOTIFY_DONE;
494}
495
496/**
497 * i915_gem_shrinker_register - Register the i915 shrinker
498 * @i915: i915 device
499 *
500 * This function registers and sets up the i915 shrinker and OOM handler.
501 */
502void i915_gem_shrinker_register(struct drm_i915_private *i915)
503{
504 i915->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
505 i915->mm.shrinker.count_objects = i915_gem_shrinker_count;
506 i915->mm.shrinker.seeks = DEFAULT_SEEKS;
507 i915->mm.shrinker.batch = 4096;
508 WARN_ON(register_shrinker(&i915->mm.shrinker));
509
510 i915->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
511 WARN_ON(register_oom_notifier(&i915->mm.oom_notifier));
512
513 i915->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
514 WARN_ON(register_vmap_purge_notifier(&i915->mm.vmap_notifier));
515}
516
517/**
518 * i915_gem_shrinker_unregister - Unregisters the i915 shrinker
519 * @i915: i915 device
520 *
521 * This function unregisters the i915 shrinker and OOM handler.
522 */
523void i915_gem_shrinker_unregister(struct drm_i915_private *i915)
524{
525 WARN_ON(unregister_vmap_purge_notifier(&i915->mm.vmap_notifier));
526 WARN_ON(unregister_oom_notifier(&i915->mm.oom_notifier));
527 unregister_shrinker(&i915->mm.shrinker);
528}