1/*
  2 * Copyright © 2008-2015 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include <linux/oom.h>
 26#include <linux/shmem_fs.h>
 27#include <linux/slab.h>
 28#include <linux/swap.h>
 29#include <linux/pci.h>
 30#include <linux/dma-buf.h>
 31#include <drm/drmP.h>
 32#include <drm/i915_drm.h>
 33
 34#include "i915_drv.h"
 35#include "i915_trace.h"
 36
 37static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
 38{
 39	if (!mutex_is_locked(mutex))
 40		return false;
 41
 42#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
 43	return mutex->owner == task;
 44#else
 45	/* Since UP may be pre-empted, we cannot assume that we own the lock */
 46	return false;
 47#endif
 48}
 49
 50static int num_vma_bound(struct drm_i915_gem_object *obj)
 51{
 52	struct i915_vma *vma;
 53	int count = 0;
 54
 55	list_for_each_entry(vma, &obj->vma_list, obj_link) {
 56		if (drm_mm_node_allocated(&vma->node))
 57			count++;
 58		if (vma->pin_count)
 59			count++;
 60	}
 61
 62	return count;
 63}
 64
 65static bool swap_available(void)
 66{
 67	return get_nr_swap_pages() > 0;
 68}
 69
 70static bool can_release_pages(struct drm_i915_gem_object *obj)
 71{
 72	/* Only report true if by unbinding the object and putting its pages
 73	 * we can actually make forward progress towards freeing physical
 74	 * pages.
 75	 *
 76	 * If the pages are pinned for any other reason than being bound
 77	 * to the GPU, simply unbinding from the GPU is not going to succeed
 78	 * in releasing our pin count on the pages themselves.
 79	 */
 80	if (obj->pages_pin_count != num_vma_bound(obj))
 81		return false;
 82
 83	/* We can only return physical pages to the system if we can either
 84	 * discard the contents (because the user has marked them as being
 85	 * purgeable) or if we can move their contents out to swap.
 86	 */
 87	return swap_available() || obj->madv == I915_MADV_DONTNEED;
 88}
 89
 90/**
 91 * i915_gem_shrink - Shrink buffer object caches
 92 * @dev_priv: i915 device
 93 * @target: amount of memory to make available, in pages
 94 * @flags: control flags for selecting cache types
 95 *
 96 * This function is the main interface to the shrinker. It will try to release
 97 * up to @target pages of main memory backing storage from buffer objects.
 98 * Selection of the specific caches can be done with @flags. This is e.g. useful
 99 * when purgeable objects should be removed from caches preferentially.
100 *
101 * Note that it's not guaranteed that released amount is actually available as
102 * free system memory - the pages might still be in-used to due to other reasons
103 * (like cpu mmaps) or the mm core has reused them before we could grab them.
104 * Therefore code that needs to explicitly shrink buffer objects caches (e.g. to
105 * avoid deadlocks in memory reclaim) must fall back to i915_gem_shrink_all().
106 *
107 * Also note that any kind of pinning (both per-vma address space pins and
108 * backing storage pins at the buffer object level) result in the shrinker code
109 * having to skip the object.
110 *
111 * Returns:
112 * The number of pages of backing storage actually released.
113 */
114unsigned long
115i915_gem_shrink(struct drm_i915_private *dev_priv,
116		unsigned long target, unsigned flags)
117{
118	const struct {
119		struct list_head *list;
120		unsigned int bit;
121	} phases[] = {
122		{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
123		{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
124		{ NULL, 0 },
125	}, *phase;
126	unsigned long count = 0;
127
128	trace_i915_gem_shrink(dev_priv, target, flags);
129	i915_gem_retire_requests(dev_priv->dev);
130
131	/*
132	 * As we may completely rewrite the (un)bound list whilst unbinding
133	 * (due to retiring requests) we have to strictly process only
134	 * one element of the list at the time, and recheck the list
135	 * on every iteration.
136	 *
137	 * In particular, we must hold a reference whilst removing the
138	 * object as we may end up waiting for and/or retiring the objects.
139	 * This might release the final reference (held by the active list)
140	 * and result in the object being freed from under us. This is
141	 * similar to the precautions the eviction code must take whilst
142	 * removing objects.
143	 *
144	 * Also note that although these lists do not hold a reference to
145	 * the object we can safely grab one here: The final object
146	 * unreferencing and the bound_list are both protected by the
147	 * dev->struct_mutex and so we won't ever be able to observe an
148	 * object on the bound_list with a reference count equals 0.
149	 */
150	for (phase = phases; phase->list; phase++) {
151		struct list_head still_in_list;
152
153		if ((flags & phase->bit) == 0)
154			continue;
155
156		INIT_LIST_HEAD(&still_in_list);
157		while (count < target && !list_empty(phase->list)) {
158			struct drm_i915_gem_object *obj;
159			struct i915_vma *vma, *v;
160
161			obj = list_first_entry(phase->list,
162					       typeof(*obj), global_list);
163			list_move_tail(&obj->global_list, &still_in_list);
164
165			if (flags & I915_SHRINK_PURGEABLE &&
166			    obj->madv != I915_MADV_DONTNEED)
167				continue;
168
169			if ((flags & I915_SHRINK_ACTIVE) == 0 && obj->active)
170				continue;
171
172			if (!can_release_pages(obj))
173				continue;
174
175			drm_gem_object_reference(&obj->base);
176
177			/* For the unbound phase, this should be a no-op! */
178			list_for_each_entry_safe(vma, v,
179						 &obj->vma_list, obj_link)
180				if (i915_vma_unbind(vma))
181					break;
182
183			if (i915_gem_object_put_pages(obj) == 0)
184				count += obj->base.size >> PAGE_SHIFT;
185
186			drm_gem_object_unreference(&obj->base);
187		}
188		list_splice(&still_in_list, phase->list);
189	}
190
191	i915_gem_retire_requests(dev_priv->dev);
192
193	return count;
194}
195
196/**
197 * i915_gem_shrink_all - Shrink buffer object caches completely
198 * @dev_priv: i915 device
199 *
200 * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
201 * caches completely. It also first waits for and retires all outstanding
202 * requests to also be able to release backing storage for active objects.
203 *
204 * This should only be used in code to intentionally quiescent the gpu or as a
205 * last-ditch effort when memory seems to have run out.
206 *
207 * Returns:
208 * The number of pages of backing storage actually released.
209 */
210unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
211{
212	return i915_gem_shrink(dev_priv, -1UL,
213			       I915_SHRINK_BOUND |
214			       I915_SHRINK_UNBOUND |
215			       I915_SHRINK_ACTIVE);
216}
217
218static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
219{
220	if (!mutex_trylock(&dev->struct_mutex)) {
221		if (!mutex_is_locked_by(&dev->struct_mutex, current))
222			return false;
223
224		if (to_i915(dev)->mm.shrinker_no_lock_stealing)
225			return false;
226
227		*unlock = false;
228	} else
229		*unlock = true;
230
231	return true;
232}
233
234static unsigned long
235i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
236{
237	struct drm_i915_private *dev_priv =
238		container_of(shrinker, struct drm_i915_private, mm.shrinker);
239	struct drm_device *dev = dev_priv->dev;
240	struct drm_i915_gem_object *obj;
241	unsigned long count;
242	bool unlock;
243
244	if (!i915_gem_shrinker_lock(dev, &unlock))
245		return 0;
246
247	count = 0;
248	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
249		if (obj->pages_pin_count == 0)
250			count += obj->base.size >> PAGE_SHIFT;
251
252	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
253		if (!obj->active && can_release_pages(obj))
254			count += obj->base.size >> PAGE_SHIFT;
255	}
256
257	if (unlock)
258		mutex_unlock(&dev->struct_mutex);
259
260	return count;
261}
262
263static unsigned long
264i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
265{
266	struct drm_i915_private *dev_priv =
267		container_of(shrinker, struct drm_i915_private, mm.shrinker);
268	struct drm_device *dev = dev_priv->dev;
269	unsigned long freed;
270	bool unlock;
271
272	if (!i915_gem_shrinker_lock(dev, &unlock))
273		return SHRINK_STOP;
274
275	freed = i915_gem_shrink(dev_priv,
276				sc->nr_to_scan,
277				I915_SHRINK_BOUND |
278				I915_SHRINK_UNBOUND |
279				I915_SHRINK_PURGEABLE);
280	if (freed < sc->nr_to_scan)
281		freed += i915_gem_shrink(dev_priv,
282					 sc->nr_to_scan - freed,
283					 I915_SHRINK_BOUND |
284					 I915_SHRINK_UNBOUND);
285	if (unlock)
286		mutex_unlock(&dev->struct_mutex);
287
288	return freed;
289}
290
291static int
292i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
293{
294	struct drm_i915_private *dev_priv =
295		container_of(nb, struct drm_i915_private, mm.oom_notifier);
296	struct drm_device *dev = dev_priv->dev;
297	struct drm_i915_gem_object *obj;
298	unsigned long timeout = msecs_to_jiffies(5000) + 1;
299	unsigned long pinned, bound, unbound, freed_pages;
300	bool was_interruptible;
301	bool unlock;
302
303	while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) {
304		schedule_timeout_killable(1);
305		if (fatal_signal_pending(current))
306			return NOTIFY_DONE;
307	}
308	if (timeout == 0) {
309		pr_err("Unable to purge GPU memory due lock contention.\n");
310		return NOTIFY_DONE;
311	}
312
313	was_interruptible = dev_priv->mm.interruptible;
314	dev_priv->mm.interruptible = false;
315
316	freed_pages = i915_gem_shrink_all(dev_priv);
317
318	dev_priv->mm.interruptible = was_interruptible;
319
320	/* Because we may be allocating inside our own driver, we cannot
321	 * assert that there are no objects with pinned pages that are not
322	 * being pointed to by hardware.
323	 */
324	unbound = bound = pinned = 0;
325	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
326		if (!obj->base.filp) /* not backed by a freeable object */
327			continue;
328
329		if (obj->pages_pin_count)
330			pinned += obj->base.size;
331		else
332			unbound += obj->base.size;
333	}
334	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
335		if (!obj->base.filp)
336			continue;
337
338		if (obj->pages_pin_count)
339			pinned += obj->base.size;
340		else
341			bound += obj->base.size;
342	}
343
344	if (unlock)
345		mutex_unlock(&dev->struct_mutex);
346
347	if (freed_pages || unbound || bound)
348		pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
349			freed_pages << PAGE_SHIFT, pinned);
350	if (unbound || bound)
351		pr_err("%lu and %lu bytes still available in the "
352		       "bound and unbound GPU page lists.\n",
353		       bound, unbound);
354
355	*(unsigned long *)ptr += freed_pages;
356	return NOTIFY_DONE;
357}
358
359/**
360 * i915_gem_shrinker_init - Initialize i915 shrinker
361 * @dev_priv: i915 device
362 *
363 * This function registers and sets up the i915 shrinker and OOM handler.
364 */
365void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
366{
367	dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
368	dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
369	dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
370	WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
371
372	dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
373	WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
374}
375
376/**
377 * i915_gem_shrinker_cleanup - Clean up i915 shrinker
378 * @dev_priv: i915 device
379 *
380 * This function unregisters the i915 shrinker and OOM handler.
381 */
382void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
383{
384	WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
385	unregister_shrinker(&dev_priv->mm.shrinker);
386}