1/*
  2 * Copyright © 2008-2015 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include <linux/oom.h>
 26#include <linux/shmem_fs.h>
 27#include <linux/slab.h>
 28#include <linux/swap.h>
 29#include <linux/pci.h>
 30#include <linux/dma-buf.h>
 31#include <linux/vmalloc.h>
 32#include <drm/drmP.h>
 33#include <drm/i915_drm.h>
 34
 35#include "i915_drv.h"
 36#include "i915_trace.h"
 37
 38static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
 39{
 40	switch (mutex_trylock_recursive(&dev->struct_mutex)) {
 41	case MUTEX_TRYLOCK_FAILED:
 42		return false;
 43
 44	case MUTEX_TRYLOCK_SUCCESS:
 45		*unlock = true;
 46		return true;
 47
 48	case MUTEX_TRYLOCK_RECURSIVE:
 49		*unlock = false;
 50		return true;
 51	}
 52
 53	BUG();
 54}
 55
 56static bool any_vma_pinned(struct drm_i915_gem_object *obj)
 57{
 58	struct i915_vma *vma;
 59
 60	list_for_each_entry(vma, &obj->vma_list, obj_link)
 61		if (i915_vma_is_pinned(vma))
 62			return true;
 63
 64	return false;
 65}
 66
 67static bool swap_available(void)
 68{
 69	return get_nr_swap_pages() > 0;
 70}
 71
 72static bool can_release_pages(struct drm_i915_gem_object *obj)
 73{
 74	if (!obj->mm.pages)
 75		return false;
 76
 77	/* Consider only shrinkable ojects. */
 78	if (!i915_gem_object_is_shrinkable(obj))
 79		return false;
 80
 81	/* Only report true if by unbinding the object and putting its pages
 82	 * we can actually make forward progress towards freeing physical
 83	 * pages.
 84	 *
 85	 * If the pages are pinned for any other reason than being bound
 86	 * to the GPU, simply unbinding from the GPU is not going to succeed
 87	 * in releasing our pin count on the pages themselves.
 88	 */
 89	if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
 90		return false;
 91
 92	if (any_vma_pinned(obj))
 93		return false;
 94
 95	/* We can only return physical pages to the system if we can either
 96	 * discard the contents (because the user has marked them as being
 97	 * purgeable) or if we can move their contents out to swap.
 98	 */
 99	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
100}
101
102static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
103{
104	if (i915_gem_object_unbind(obj) == 0)
105		__i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
106	return !READ_ONCE(obj->mm.pages);
107}
108
109/**
110 * i915_gem_shrink - Shrink buffer object caches
111 * @dev_priv: i915 device
112 * @target: amount of memory to make available, in pages
113 * @flags: control flags for selecting cache types
114 *
115 * This function is the main interface to the shrinker. It will try to release
116 * up to @target pages of main memory backing storage from buffer objects.
117 * Selection of the specific caches can be done with @flags. This is e.g. useful
118 * when purgeable objects should be removed from caches preferentially.
119 *
120 * Note that it's not guaranteed that released amount is actually available as
121 * free system memory - the pages might still be in-used to due to other reasons
122 * (like cpu mmaps) or the mm core has reused them before we could grab them.
123 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
124 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
125 *
126 * Also note that any kind of pinning (both per-vma address space pins and
127 * backing storage pins at the buffer object level) result in the shrinker code
128 * having to skip the object.
129 *
130 * Returns:
131 * The number of pages of backing storage actually released.
132 */
133unsigned long
134i915_gem_shrink(struct drm_i915_private *dev_priv,
135		unsigned long target, unsigned flags)
136{
137	const struct {
138		struct list_head *list;
139		unsigned int bit;
140	} phases[] = {
141		{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
142		{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
143		{ NULL, 0 },
144	}, *phase;
145	unsigned long count = 0;
146	bool unlock;
147
148	if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock))
149		return 0;
150
151	trace_i915_gem_shrink(dev_priv, target, flags);
152	i915_gem_retire_requests(dev_priv);
153
154	/*
155	 * Unbinding of objects will require HW access; Let us not wake the
156	 * device just to recover a little memory. If absolutely necessary,
157	 * we will force the wake during oom-notifier.
158	 */
159	if ((flags & I915_SHRINK_BOUND) &&
160	    !intel_runtime_pm_get_if_in_use(dev_priv))
161		flags &= ~I915_SHRINK_BOUND;
162
163	/*
164	 * As we may completely rewrite the (un)bound list whilst unbinding
165	 * (due to retiring requests) we have to strictly process only
166	 * one element of the list at the time, and recheck the list
167	 * on every iteration.
168	 *
169	 * In particular, we must hold a reference whilst removing the
170	 * object as we may end up waiting for and/or retiring the objects.
171	 * This might release the final reference (held by the active list)
172	 * and result in the object being freed from under us. This is
173	 * similar to the precautions the eviction code must take whilst
174	 * removing objects.
175	 *
176	 * Also note that although these lists do not hold a reference to
177	 * the object we can safely grab one here: The final object
178	 * unreferencing and the bound_list are both protected by the
179	 * dev->struct_mutex and so we won't ever be able to observe an
180	 * object on the bound_list with a reference count equals 0.
181	 */
182	for (phase = phases; phase->list; phase++) {
183		struct list_head still_in_list;
184		struct drm_i915_gem_object *obj;
185
186		if ((flags & phase->bit) == 0)
187			continue;
188
189		INIT_LIST_HEAD(&still_in_list);
190		while (count < target &&
191		       (obj = list_first_entry_or_null(phase->list,
192						       typeof(*obj),
193						       global_link))) {
194			list_move_tail(&obj->global_link, &still_in_list);
195			if (!obj->mm.pages) {
196				list_del_init(&obj->global_link);
197				continue;
198			}
199
200			if (flags & I915_SHRINK_PURGEABLE &&
201			    obj->mm.madv != I915_MADV_DONTNEED)
202				continue;
203
204			if (flags & I915_SHRINK_VMAPS &&
205			    !is_vmalloc_addr(obj->mm.mapping))
206				continue;
207
208			if (!(flags & I915_SHRINK_ACTIVE) &&
209			    (i915_gem_object_is_active(obj) ||
210			     obj->framebuffer_references))
211				continue;
212
213			if (!can_release_pages(obj))
214				continue;
215
216			if (unsafe_drop_pages(obj)) {
217				/* May arrive from get_pages on another bo */
218				mutex_lock_nested(&obj->mm.lock,
219						  I915_MM_SHRINKER);
220				if (!obj->mm.pages) {
221					__i915_gem_object_invalidate(obj);
222					list_del_init(&obj->global_link);
223					count += obj->base.size >> PAGE_SHIFT;
224				}
225				mutex_unlock(&obj->mm.lock);
226			}
227		}
228		list_splice_tail(&still_in_list, phase->list);
229	}
230
231	if (flags & I915_SHRINK_BOUND)
232		intel_runtime_pm_put(dev_priv);
233
234	i915_gem_retire_requests(dev_priv);
235	if (unlock)
236		mutex_unlock(&dev_priv->drm.struct_mutex);
237
238	/* expedite the RCU grace period to free some request slabs */
239	synchronize_rcu_expedited();
240
241	return count;
242}
243
244/**
245 * i915_gem_shrink_all - Shrink buffer object caches completely
246 * @dev_priv: i915 device
247 *
248 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
249 * caches completely. It also first waits for and retires all outstanding
250 * requests to also be able to release backing storage for active objects.
251 *
252 * This should only be used in code to intentionally quiescent the gpu or as a
253 * last-ditch effort when memory seems to have run out.
254 *
255 * Returns:
256 * The number of pages of backing storage actually released.
257 */
258unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
259{
260	unsigned long freed;
261
262	freed = i915_gem_shrink(dev_priv, -1UL,
263				I915_SHRINK_BOUND |
264				I915_SHRINK_UNBOUND |
265				I915_SHRINK_ACTIVE);
266	synchronize_rcu(); /* wait for our earlier RCU delayed slab frees */
267
268	return freed;
269}
270
271static unsigned long
272i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
273{
274	struct drm_i915_private *dev_priv =
275		container_of(shrinker, struct drm_i915_private, mm.shrinker);
276	struct drm_device *dev = &dev_priv->drm;
277	struct drm_i915_gem_object *obj;
278	unsigned long count;
279	bool unlock;
280
281	if (!i915_gem_shrinker_lock(dev, &unlock))
282		return 0;
283
284	i915_gem_retire_requests(dev_priv);
285
286	count = 0;
287	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link)
288		if (can_release_pages(obj))
289			count += obj->base.size >> PAGE_SHIFT;
290
291	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
292		if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
293			count += obj->base.size >> PAGE_SHIFT;
294	}
295
296	if (unlock)
297		mutex_unlock(&dev->struct_mutex);
298
299	return count;
300}
301
302static unsigned long
303i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
304{
305	struct drm_i915_private *dev_priv =
306		container_of(shrinker, struct drm_i915_private, mm.shrinker);
307	struct drm_device *dev = &dev_priv->drm;
308	unsigned long freed;
309	bool unlock;
310
311	if (!i915_gem_shrinker_lock(dev, &unlock))
312		return SHRINK_STOP;
313
314	freed = i915_gem_shrink(dev_priv,
315				sc->nr_to_scan,
316				I915_SHRINK_BOUND |
317				I915_SHRINK_UNBOUND |
318				I915_SHRINK_PURGEABLE);
319	if (freed < sc->nr_to_scan)
320		freed += i915_gem_shrink(dev_priv,
321					 sc->nr_to_scan - freed,
322					 I915_SHRINK_BOUND |
323					 I915_SHRINK_UNBOUND);
324	if (unlock)
325		mutex_unlock(&dev->struct_mutex);
326
327	return freed;
328}
329
330struct shrinker_lock_uninterruptible {
331	bool was_interruptible;
332	bool unlock;
333};
334
335static bool
336i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
337				       struct shrinker_lock_uninterruptible *slu,
338				       int timeout_ms)
339{
340	unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
341
342	do {
343		if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
344		    i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock))
345			break;
346
347		schedule_timeout_killable(1);
348		if (fatal_signal_pending(current))
349			return false;
350
351		if (time_after(jiffies, timeout)) {
352			pr_err("Unable to lock GPU to purge memory.\n");
353			return false;
354		}
355	} while (1);
356
357	slu->was_interruptible = dev_priv->mm.interruptible;
358	dev_priv->mm.interruptible = false;
359	return true;
360}
361
362static void
363i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
364					 struct shrinker_lock_uninterruptible *slu)
365{
366	dev_priv->mm.interruptible = slu->was_interruptible;
367	if (slu->unlock)
368		mutex_unlock(&dev_priv->drm.struct_mutex);
369}
370
371static int
372i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
373{
374	struct drm_i915_private *dev_priv =
375		container_of(nb, struct drm_i915_private, mm.oom_notifier);
376	struct shrinker_lock_uninterruptible slu;
377	struct drm_i915_gem_object *obj;
378	unsigned long unevictable, bound, unbound, freed_pages;
379
380	if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
381		return NOTIFY_DONE;
382
383	intel_runtime_pm_get(dev_priv);
384	freed_pages = i915_gem_shrink_all(dev_priv);
385	intel_runtime_pm_put(dev_priv);
386
387	/* Because we may be allocating inside our own driver, we cannot
388	 * assert that there are no objects with pinned pages that are not
389	 * being pointed to by hardware.
390	 */
391	unbound = bound = unevictable = 0;
392	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
393		if (!obj->mm.pages)
394			continue;
395
396		if (!can_release_pages(obj))
397			unevictable += obj->base.size >> PAGE_SHIFT;
398		else
399			unbound += obj->base.size >> PAGE_SHIFT;
400	}
401	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
402		if (!obj->mm.pages)
403			continue;
404
405		if (!can_release_pages(obj))
406			unevictable += obj->base.size >> PAGE_SHIFT;
407		else
408			bound += obj->base.size >> PAGE_SHIFT;
409	}
410
411	i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
412
413	if (freed_pages || unbound || bound)
414		pr_info("Purging GPU memory, %lu pages freed, "
415			"%lu pages still pinned.\n",
416			freed_pages, unevictable);
417	if (unbound || bound)
418		pr_err("%lu and %lu pages still available in the "
419		       "bound and unbound GPU page lists.\n",
420		       bound, unbound);
421
422	*(unsigned long *)ptr += freed_pages;
423	return NOTIFY_DONE;
424}
425
426static int
427i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
428{
429	struct drm_i915_private *dev_priv =
430		container_of(nb, struct drm_i915_private, mm.vmap_notifier);
431	struct shrinker_lock_uninterruptible slu;
432	struct i915_vma *vma, *next;
433	unsigned long freed_pages = 0;
434	int ret;
435
436	if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
437		return NOTIFY_DONE;
438
439	/* Force everything onto the inactive lists */
440	ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
441	if (ret)
442		goto out;
443
444	intel_runtime_pm_get(dev_priv);
445	freed_pages += i915_gem_shrink(dev_priv, -1UL,
446				       I915_SHRINK_BOUND |
447				       I915_SHRINK_UNBOUND |
448				       I915_SHRINK_ACTIVE |
449				       I915_SHRINK_VMAPS);
450	intel_runtime_pm_put(dev_priv);
451
452	/* We also want to clear any cached iomaps as they wrap vmap */
453	list_for_each_entry_safe(vma, next,
454				 &dev_priv->ggtt.base.inactive_list, vm_link) {
455		unsigned long count = vma->node.size >> PAGE_SHIFT;
456		if (vma->iomap && i915_vma_unbind(vma) == 0)
457			freed_pages += count;
458	}
459
460out:
461	i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
462
463	*(unsigned long *)ptr += freed_pages;
464	return NOTIFY_DONE;
465}
466
467/**
468 * i915_gem_shrinker_init - Initialize i915 shrinker
469 * @dev_priv: i915 device
470 *
471 * This function registers and sets up the i915 shrinker and OOM handler.
472 */
473void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
474{
475	dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
476	dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
477	dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
478	WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
479
480	dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
481	WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
482
483	dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
484	WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
485}
486
487/**
488 * i915_gem_shrinker_cleanup - Clean up i915 shrinker
489 * @dev_priv: i915 device
490 *
491 * This function unregisters the i915 shrinker and OOM handler.
492 */
493void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
494{
495	WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
496	WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
497	unregister_shrinker(&dev_priv->mm.shrinker);
498}