Loading...
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/oom.h>
26#include <linux/shmem_fs.h>
27#include <linux/slab.h>
28#include <linux/swap.h>
29#include <linux/pci.h>
30#include <linux/dma-buf.h>
31#include <drm/drmP.h>
32#include <drm/i915_drm.h>
33
34#include "i915_drv.h"
35#include "i915_trace.h"
36
37static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
38{
39 if (!mutex_is_locked(mutex))
40 return false;
41
42#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
43 return mutex->owner == task;
44#else
45 /* Since UP may be pre-empted, we cannot assume that we own the lock */
46 return false;
47#endif
48}
49
50static int num_vma_bound(struct drm_i915_gem_object *obj)
51{
52 struct i915_vma *vma;
53 int count = 0;
54
55 list_for_each_entry(vma, &obj->vma_list, obj_link) {
56 if (drm_mm_node_allocated(&vma->node))
57 count++;
58 if (vma->pin_count)
59 count++;
60 }
61
62 return count;
63}
64
65static bool swap_available(void)
66{
67 return get_nr_swap_pages() > 0;
68}
69
70static bool can_release_pages(struct drm_i915_gem_object *obj)
71{
72 /* Only report true if by unbinding the object and putting its pages
73 * we can actually make forward progress towards freeing physical
74 * pages.
75 *
76 * If the pages are pinned for any other reason than being bound
77 * to the GPU, simply unbinding from the GPU is not going to succeed
78 * in releasing our pin count on the pages themselves.
79 */
80 if (obj->pages_pin_count != num_vma_bound(obj))
81 return false;
82
83 /* We can only return physical pages to the system if we can either
84 * discard the contents (because the user has marked them as being
85 * purgeable) or if we can move their contents out to swap.
86 */
87 return swap_available() || obj->madv == I915_MADV_DONTNEED;
88}
89
90/**
91 * i915_gem_shrink - Shrink buffer object caches
92 * @dev_priv: i915 device
93 * @target: amount of memory to make available, in pages
94 * @flags: control flags for selecting cache types
95 *
96 * This function is the main interface to the shrinker. It will try to release
97 * up to @target pages of main memory backing storage from buffer objects.
98 * Selection of the specific caches can be done with @flags. This is e.g. useful
99 * when purgeable objects should be removed from caches preferentially.
100 *
101 * Note that it's not guaranteed that released amount is actually available as
102 * free system memory - the pages might still be in-used to due to other reasons
103 * (like cpu mmaps) or the mm core has reused them before we could grab them.
104 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
105 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
106 *
107 * Also note that any kind of pinning (both per-vma address space pins and
108 * backing storage pins at the buffer object level) result in the shrinker code
109 * having to skip the object.
110 *
111 * Returns:
112 * The number of pages of backing storage actually released.
113 */
114unsigned long
115i915_gem_shrink(struct drm_i915_private *dev_priv,
116 unsigned long target, unsigned flags)
117{
118 const struct {
119 struct list_head *list;
120 unsigned int bit;
121 } phases[] = {
122 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
123 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
124 { NULL, 0 },
125 }, *phase;
126 unsigned long count = 0;
127
128 trace_i915_gem_shrink(dev_priv, target, flags);
129 i915_gem_retire_requests(dev_priv->dev);
130
131 /*
132 * As we may completely rewrite the (un)bound list whilst unbinding
133 * (due to retiring requests) we have to strictly process only
134 * one element of the list at the time, and recheck the list
135 * on every iteration.
136 *
137 * In particular, we must hold a reference whilst removing the
138 * object as we may end up waiting for and/or retiring the objects.
139 * This might release the final reference (held by the active list)
140 * and result in the object being freed from under us. This is
141 * similar to the precautions the eviction code must take whilst
142 * removing objects.
143 *
144 * Also note that although these lists do not hold a reference to
145 * the object we can safely grab one here: The final object
146 * unreferencing and the bound_list are both protected by the
147 * dev->struct_mutex and so we won't ever be able to observe an
148 * object on the bound_list with a reference count equals 0.
149 */
150 for (phase = phases; phase->list; phase++) {
151 struct list_head still_in_list;
152
153 if ((flags & phase->bit) == 0)
154 continue;
155
156 INIT_LIST_HEAD(&still_in_list);
157 while (count < target && !list_empty(phase->list)) {
158 struct drm_i915_gem_object *obj;
159 struct i915_vma *vma, *v;
160
161 obj = list_first_entry(phase->list,
162 typeof(*obj), global_list);
163 list_move_tail(&obj->global_list, &still_in_list);
164
165 if (flags & I915_SHRINK_PURGEABLE &&
166 obj->madv != I915_MADV_DONTNEED)
167 continue;
168
169 if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active)
170 continue;
171
172 if (!can_release_pages(obj))
173 continue;
174
175 drm_gem_object_reference(&obj->base);
176
177 /* For the unbound phase, this should be a no-op! */
178 list_for_each_entry_safe(vma, v,
179 &obj->vma_list, obj_link)
180 if (i915_vma_unbind(vma))
181 break;
182
183 if (i915_gem_object_put_pages(obj) == 0)
184 count += obj->base.size >> PAGE_SHIFT;
185
186 drm_gem_object_unreference(&obj->base);
187 }
188 list_splice(&still_in_list, phase->list);
189 }
190
191 i915_gem_retire_requests(dev_priv->dev);
192
193 return count;
194}
195
196/**
197 * i915_gem_shrink_all - Shrink buffer object caches completely
198 * @dev_priv: i915 device
199 *
200 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
201 * caches completely. It also first waits for and retires all outstanding
202 * requests to also be able to release backing storage for active objects.
203 *
204 * This should only be used in code to intentionally quiescent the gpu or as a
205 * last-ditch effort when memory seems to have run out.
206 *
207 * Returns:
208 * The number of pages of backing storage actually released.
209 */
210unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
211{
212 return i915_gem_shrink(dev_priv, -1UL,
213 I915_SHRINK_BOUND |
214 I915_SHRINK_UNBOUND |
215 I915_SHRINK_ACTIVE);
216}
217
218static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
219{
220 if (!mutex_trylock(&dev->struct_mutex)) {
221 if (!mutex_is_locked_by(&dev->struct_mutex, current))
222 return false;
223
224 if (to_i915(dev)->mm.shrinker_no_lock_stealing)
225 return false;
226
227 *unlock = false;
228 } else
229 *unlock = true;
230
231 return true;
232}
233
234static unsigned long
235i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
236{
237 struct drm_i915_private *dev_priv =
238 container_of(shrinker, struct drm_i915_private, mm.shrinker);
239 struct drm_device *dev = dev_priv->dev;
240 struct drm_i915_gem_object *obj;
241 unsigned long count;
242 bool unlock;
243
244 if (!i915_gem_shrinker_lock(dev, &unlock))
245 return 0;
246
247 count = 0;
248 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
249 if (obj->pages_pin_count == 0)
250 count += obj->base.size >> PAGE_SHIFT;
251
252 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
253 if (!obj->active && can_release_pages(obj))
254 count += obj->base.size >> PAGE_SHIFT;
255 }
256
257 if (unlock)
258 mutex_unlock(&dev->struct_mutex);
259
260 return count;
261}
262
263static unsigned long
264i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
265{
266 struct drm_i915_private *dev_priv =
267 container_of(shrinker, struct drm_i915_private, mm.shrinker);
268 struct drm_device *dev = dev_priv->dev;
269 unsigned long freed;
270 bool unlock;
271
272 if (!i915_gem_shrinker_lock(dev, &unlock))
273 return SHRINK_STOP;
274
275 freed = i915_gem_shrink(dev_priv,
276 sc->nr_to_scan,
277 I915_SHRINK_BOUND |
278 I915_SHRINK_UNBOUND |
279 I915_SHRINK_PURGEABLE);
280 if (freed < sc->nr_to_scan)
281 freed += i915_gem_shrink(dev_priv,
282 sc->nr_to_scan - freed,
283 I915_SHRINK_BOUND |
284 I915_SHRINK_UNBOUND);
285 if (unlock)
286 mutex_unlock(&dev->struct_mutex);
287
288 return freed;
289}
290
291static int
292i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
293{
294 struct drm_i915_private *dev_priv =
295 container_of(nb, struct drm_i915_private, mm.oom_notifier);
296 struct drm_device *dev = dev_priv->dev;
297 struct drm_i915_gem_object *obj;
298 unsigned long timeout = msecs_to_jiffies(5000) + 1;
299 unsigned long pinned, bound, unbound, freed_pages;
300 bool was_interruptible;
301 bool unlock;
302
303 while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) {
304 schedule_timeout_killable(1);
305 if (fatal_signal_pending(current))
306 return NOTIFY_DONE;
307 }
308 if (timeout == 0) {
309 pr_err("Unable to purge GPU memory due lock contention.\n");
310 return NOTIFY_DONE;
311 }
312
313 was_interruptible = dev_priv->mm.interruptible;
314 dev_priv->mm.interruptible = false;
315
316 freed_pages = i915_gem_shrink_all(dev_priv);
317
318 dev_priv->mm.interruptible = was_interruptible;
319
320 /* Because we may be allocating inside our own driver, we cannot
321 * assert that there are no objects with pinned pages that are not
322 * being pointed to by hardware.
323 */
324 unbound = bound = pinned = 0;
325 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
326 if (!obj->base.filp) /* not backed by a freeable object */
327 continue;
328
329 if (obj->pages_pin_count)
330 pinned += obj->base.size;
331 else
332 unbound += obj->base.size;
333 }
334 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
335 if (!obj->base.filp)
336 continue;
337
338 if (obj->pages_pin_count)
339 pinned += obj->base.size;
340 else
341 bound += obj->base.size;
342 }
343
344 if (unlock)
345 mutex_unlock(&dev->struct_mutex);
346
347 if (freed_pages || unbound || bound)
348 pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
349 freed_pages << PAGE_SHIFT, pinned);
350 if (unbound || bound)
351 pr_err("%lu and %lu bytes still available in the "
352 "bound and unbound GPU page lists.\n",
353 bound, unbound);
354
355 *(unsigned long *)ptr += freed_pages;
356 return NOTIFY_DONE;
357}
358
359/**
360 * i915_gem_shrinker_init - Initialize i915 shrinker
361 * @dev_priv: i915 device
362 *
363 * This function registers and sets up the i915 shrinker and OOM handler.
364 */
365void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
366{
367 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
368 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
369 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
370 WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
371
372 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
373 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
374}
375
376/**
377 * i915_gem_shrinker_cleanup - Clean up i915 shrinker
378 * @dev_priv: i915 device
379 *
380 * This function unregisters the i915 shrinker and OOM handler.
381 */
382void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
383{
384 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
385 unregister_shrinker(&dev_priv->mm.shrinker);
386}
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/oom.h>
26#include <linux/shmem_fs.h>
27#include <linux/slab.h>
28#include <linux/swap.h>
29#include <linux/pci.h>
30#include <linux/dma-buf.h>
31#include <linux/vmalloc.h>
32#include <drm/drmP.h>
33#include <drm/i915_drm.h>
34
35#include "i915_drv.h"
36#include "i915_trace.h"
37
38static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
39{
40 switch (mutex_trylock_recursive(&dev->struct_mutex)) {
41 case MUTEX_TRYLOCK_FAILED:
42 return false;
43
44 case MUTEX_TRYLOCK_SUCCESS:
45 *unlock = true;
46 return true;
47
48 case MUTEX_TRYLOCK_RECURSIVE:
49 *unlock = false;
50 return true;
51 }
52
53 BUG();
54}
55
56static bool any_vma_pinned(struct drm_i915_gem_object *obj)
57{
58 struct i915_vma *vma;
59
60 list_for_each_entry(vma, &obj->vma_list, obj_link)
61 if (i915_vma_is_pinned(vma))
62 return true;
63
64 return false;
65}
66
67static bool swap_available(void)
68{
69 return get_nr_swap_pages() > 0;
70}
71
72static bool can_release_pages(struct drm_i915_gem_object *obj)
73{
74 if (!obj->mm.pages)
75 return false;
76
77 /* Consider only shrinkable ojects. */
78 if (!i915_gem_object_is_shrinkable(obj))
79 return false;
80
81 /* Only report true if by unbinding the object and putting its pages
82 * we can actually make forward progress towards freeing physical
83 * pages.
84 *
85 * If the pages are pinned for any other reason than being bound
86 * to the GPU, simply unbinding from the GPU is not going to succeed
87 * in releasing our pin count on the pages themselves.
88 */
89 if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
90 return false;
91
92 if (any_vma_pinned(obj))
93 return false;
94
95 /* We can only return physical pages to the system if we can either
96 * discard the contents (because the user has marked them as being
97 * purgeable) or if we can move their contents out to swap.
98 */
99 return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
100}
101
102static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
103{
104 if (i915_gem_object_unbind(obj) == 0)
105 __i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
106 return !READ_ONCE(obj->mm.pages);
107}
108
109/**
110 * i915_gem_shrink - Shrink buffer object caches
111 * @dev_priv: i915 device
112 * @target: amount of memory to make available, in pages
113 * @flags: control flags for selecting cache types
114 *
115 * This function is the main interface to the shrinker. It will try to release
116 * up to @target pages of main memory backing storage from buffer objects.
117 * Selection of the specific caches can be done with @flags. This is e.g. useful
118 * when purgeable objects should be removed from caches preferentially.
119 *
120 * Note that it's not guaranteed that released amount is actually available as
121 * free system memory - the pages might still be in-used to due to other reasons
122 * (like cpu mmaps) or the mm core has reused them before we could grab them.
123 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
124 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
125 *
126 * Also note that any kind of pinning (both per-vma address space pins and
127 * backing storage pins at the buffer object level) result in the shrinker code
128 * having to skip the object.
129 *
130 * Returns:
131 * The number of pages of backing storage actually released.
132 */
133unsigned long
134i915_gem_shrink(struct drm_i915_private *dev_priv,
135 unsigned long target, unsigned flags)
136{
137 const struct {
138 struct list_head *list;
139 unsigned int bit;
140 } phases[] = {
141 { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
142 { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
143 { NULL, 0 },
144 }, *phase;
145 unsigned long count = 0;
146 bool unlock;
147
148 if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock))
149 return 0;
150
151 trace_i915_gem_shrink(dev_priv, target, flags);
152 i915_gem_retire_requests(dev_priv);
153
154 /*
155 * Unbinding of objects will require HW access; Let us not wake the
156 * device just to recover a little memory. If absolutely necessary,
157 * we will force the wake during oom-notifier.
158 */
159 if ((flags & I915_SHRINK_BOUND) &&
160 !intel_runtime_pm_get_if_in_use(dev_priv))
161 flags &= ~I915_SHRINK_BOUND;
162
163 /*
164 * As we may completely rewrite the (un)bound list whilst unbinding
165 * (due to retiring requests) we have to strictly process only
166 * one element of the list at the time, and recheck the list
167 * on every iteration.
168 *
169 * In particular, we must hold a reference whilst removing the
170 * object as we may end up waiting for and/or retiring the objects.
171 * This might release the final reference (held by the active list)
172 * and result in the object being freed from under us. This is
173 * similar to the precautions the eviction code must take whilst
174 * removing objects.
175 *
176 * Also note that although these lists do not hold a reference to
177 * the object we can safely grab one here: The final object
178 * unreferencing and the bound_list are both protected by the
179 * dev->struct_mutex and so we won't ever be able to observe an
180 * object on the bound_list with a reference count equals 0.
181 */
182 for (phase = phases; phase->list; phase++) {
183 struct list_head still_in_list;
184 struct drm_i915_gem_object *obj;
185
186 if ((flags & phase->bit) == 0)
187 continue;
188
189 INIT_LIST_HEAD(&still_in_list);
190 while (count < target &&
191 (obj = list_first_entry_or_null(phase->list,
192 typeof(*obj),
193 global_link))) {
194 list_move_tail(&obj->global_link, &still_in_list);
195 if (!obj->mm.pages) {
196 list_del_init(&obj->global_link);
197 continue;
198 }
199
200 if (flags & I915_SHRINK_PURGEABLE &&
201 obj->mm.madv != I915_MADV_DONTNEED)
202 continue;
203
204 if (flags & I915_SHRINK_VMAPS &&
205 !is_vmalloc_addr(obj->mm.mapping))
206 continue;
207
208 if (!(flags & I915_SHRINK_ACTIVE) &&
209 (i915_gem_object_is_active(obj) ||
210 obj->framebuffer_references))
211 continue;
212
213 if (!can_release_pages(obj))
214 continue;
215
216 if (unsafe_drop_pages(obj)) {
217 /* May arrive from get_pages on another bo */
218 mutex_lock_nested(&obj->mm.lock,
219 I915_MM_SHRINKER);
220 if (!obj->mm.pages) {
221 __i915_gem_object_invalidate(obj);
222 list_del_init(&obj->global_link);
223 count += obj->base.size >> PAGE_SHIFT;
224 }
225 mutex_unlock(&obj->mm.lock);
226 }
227 }
228 list_splice_tail(&still_in_list, phase->list);
229 }
230
231 if (flags & I915_SHRINK_BOUND)
232 intel_runtime_pm_put(dev_priv);
233
234 i915_gem_retire_requests(dev_priv);
235 if (unlock)
236 mutex_unlock(&dev_priv->drm.struct_mutex);
237
238 /* expedite the RCU grace period to free some request slabs */
239 synchronize_rcu_expedited();
240
241 return count;
242}
243
244/**
245 * i915_gem_shrink_all - Shrink buffer object caches completely
246 * @dev_priv: i915 device
247 *
248 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
249 * caches completely. It also first waits for and retires all outstanding
250 * requests to also be able to release backing storage for active objects.
251 *
252 * This should only be used in code to intentionally quiescent the gpu or as a
253 * last-ditch effort when memory seems to have run out.
254 *
255 * Returns:
256 * The number of pages of backing storage actually released.
257 */
258unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
259{
260 unsigned long freed;
261
262 freed = i915_gem_shrink(dev_priv, -1UL,
263 I915_SHRINK_BOUND |
264 I915_SHRINK_UNBOUND |
265 I915_SHRINK_ACTIVE);
266 synchronize_rcu(); /* wait for our earlier RCU delayed slab frees */
267
268 return freed;
269}
270
271static unsigned long
272i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
273{
274 struct drm_i915_private *dev_priv =
275 container_of(shrinker, struct drm_i915_private, mm.shrinker);
276 struct drm_device *dev = &dev_priv->drm;
277 struct drm_i915_gem_object *obj;
278 unsigned long count;
279 bool unlock;
280
281 if (!i915_gem_shrinker_lock(dev, &unlock))
282 return 0;
283
284 i915_gem_retire_requests(dev_priv);
285
286 count = 0;
287 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link)
288 if (can_release_pages(obj))
289 count += obj->base.size >> PAGE_SHIFT;
290
291 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
292 if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
293 count += obj->base.size >> PAGE_SHIFT;
294 }
295
296 if (unlock)
297 mutex_unlock(&dev->struct_mutex);
298
299 return count;
300}
301
302static unsigned long
303i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
304{
305 struct drm_i915_private *dev_priv =
306 container_of(shrinker, struct drm_i915_private, mm.shrinker);
307 struct drm_device *dev = &dev_priv->drm;
308 unsigned long freed;
309 bool unlock;
310
311 if (!i915_gem_shrinker_lock(dev, &unlock))
312 return SHRINK_STOP;
313
314 freed = i915_gem_shrink(dev_priv,
315 sc->nr_to_scan,
316 I915_SHRINK_BOUND |
317 I915_SHRINK_UNBOUND |
318 I915_SHRINK_PURGEABLE);
319 if (freed < sc->nr_to_scan)
320 freed += i915_gem_shrink(dev_priv,
321 sc->nr_to_scan - freed,
322 I915_SHRINK_BOUND |
323 I915_SHRINK_UNBOUND);
324 if (unlock)
325 mutex_unlock(&dev->struct_mutex);
326
327 return freed;
328}
329
330struct shrinker_lock_uninterruptible {
331 bool was_interruptible;
332 bool unlock;
333};
334
335static bool
336i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
337 struct shrinker_lock_uninterruptible *slu,
338 int timeout_ms)
339{
340 unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
341
342 do {
343 if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
344 i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock))
345 break;
346
347 schedule_timeout_killable(1);
348 if (fatal_signal_pending(current))
349 return false;
350
351 if (time_after(jiffies, timeout)) {
352 pr_err("Unable to lock GPU to purge memory.\n");
353 return false;
354 }
355 } while (1);
356
357 slu->was_interruptible = dev_priv->mm.interruptible;
358 dev_priv->mm.interruptible = false;
359 return true;
360}
361
362static void
363i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
364 struct shrinker_lock_uninterruptible *slu)
365{
366 dev_priv->mm.interruptible = slu->was_interruptible;
367 if (slu->unlock)
368 mutex_unlock(&dev_priv->drm.struct_mutex);
369}
370
371static int
372i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
373{
374 struct drm_i915_private *dev_priv =
375 container_of(nb, struct drm_i915_private, mm.oom_notifier);
376 struct shrinker_lock_uninterruptible slu;
377 struct drm_i915_gem_object *obj;
378 unsigned long unevictable, bound, unbound, freed_pages;
379
380 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
381 return NOTIFY_DONE;
382
383 intel_runtime_pm_get(dev_priv);
384 freed_pages = i915_gem_shrink_all(dev_priv);
385 intel_runtime_pm_put(dev_priv);
386
387 /* Because we may be allocating inside our own driver, we cannot
388 * assert that there are no objects with pinned pages that are not
389 * being pointed to by hardware.
390 */
391 unbound = bound = unevictable = 0;
392 list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
393 if (!obj->mm.pages)
394 continue;
395
396 if (!can_release_pages(obj))
397 unevictable += obj->base.size >> PAGE_SHIFT;
398 else
399 unbound += obj->base.size >> PAGE_SHIFT;
400 }
401 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
402 if (!obj->mm.pages)
403 continue;
404
405 if (!can_release_pages(obj))
406 unevictable += obj->base.size >> PAGE_SHIFT;
407 else
408 bound += obj->base.size >> PAGE_SHIFT;
409 }
410
411 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
412
413 if (freed_pages || unbound || bound)
414 pr_info("Purging GPU memory, %lu pages freed, "
415 "%lu pages still pinned.\n",
416 freed_pages, unevictable);
417 if (unbound || bound)
418 pr_err("%lu and %lu pages still available in the "
419 "bound and unbound GPU page lists.\n",
420 bound, unbound);
421
422 *(unsigned long *)ptr += freed_pages;
423 return NOTIFY_DONE;
424}
425
426static int
427i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
428{
429 struct drm_i915_private *dev_priv =
430 container_of(nb, struct drm_i915_private, mm.vmap_notifier);
431 struct shrinker_lock_uninterruptible slu;
432 struct i915_vma *vma, *next;
433 unsigned long freed_pages = 0;
434 int ret;
435
436 if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
437 return NOTIFY_DONE;
438
439 /* Force everything onto the inactive lists */
440 ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
441 if (ret)
442 goto out;
443
444 intel_runtime_pm_get(dev_priv);
445 freed_pages += i915_gem_shrink(dev_priv, -1UL,
446 I915_SHRINK_BOUND |
447 I915_SHRINK_UNBOUND |
448 I915_SHRINK_ACTIVE |
449 I915_SHRINK_VMAPS);
450 intel_runtime_pm_put(dev_priv);
451
452 /* We also want to clear any cached iomaps as they wrap vmap */
453 list_for_each_entry_safe(vma, next,
454 &dev_priv->ggtt.base.inactive_list, vm_link) {
455 unsigned long count = vma->node.size >> PAGE_SHIFT;
456 if (vma->iomap && i915_vma_unbind(vma) == 0)
457 freed_pages += count;
458 }
459
460out:
461 i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
462
463 *(unsigned long *)ptr += freed_pages;
464 return NOTIFY_DONE;
465}
466
467/**
468 * i915_gem_shrinker_init - Initialize i915 shrinker
469 * @dev_priv: i915 device
470 *
471 * This function registers and sets up the i915 shrinker and OOM handler.
472 */
473void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
474{
475 dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
476 dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
477 dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
478 WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
479
480 dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
481 WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
482
483 dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
484 WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
485}
486
487/**
488 * i915_gem_shrinker_cleanup - Clean up i915 shrinker
489 * @dev_priv: i915 device
490 *
491 * This function unregisters the i915 shrinker and OOM handler.
492 */
493void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
494{
495 WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
496 WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
497 unregister_shrinker(&dev_priv->mm.shrinker);
498}