Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2016 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include "msm_drv.h"
8#include "msm_gem.h"
9#include "msm_gpu.h"
10#include "msm_gpu_trace.h"
11
12/* Default disabled for now until it has some more testing on the different
13 * iommu combinations that can be paired with the driver:
14 */
15bool enable_eviction = false;
16MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers");
17module_param(enable_eviction, bool, 0600);
18
19static bool can_swap(void)
20{
21 return enable_eviction && get_nr_swap_pages() > 0;
22}
23
24static unsigned long
25msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
26{
27 struct msm_drm_private *priv =
28 container_of(shrinker, struct msm_drm_private, shrinker);
29 unsigned count = priv->shrinkable_count;
30
31 if (can_swap())
32 count += priv->evictable_count;
33
34 return count;
35}
36
37static bool
38purge(struct msm_gem_object *msm_obj)
39{
40 if (!is_purgeable(msm_obj))
41 return false;
42
43 /*
44 * This will move the obj out of still_in_list to
45 * the purged list
46 */
47 msm_gem_purge(&msm_obj->base);
48
49 return true;
50}
51
52static bool
53evict(struct msm_gem_object *msm_obj)
54{
55 if (is_unevictable(msm_obj))
56 return false;
57
58 msm_gem_evict(&msm_obj->base);
59
60 return true;
61}
62
63static unsigned long
64scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list,
65 bool (*shrink)(struct msm_gem_object *msm_obj))
66{
67 unsigned freed = 0;
68 struct list_head still_in_list;
69
70 INIT_LIST_HEAD(&still_in_list);
71
72 mutex_lock(&priv->mm_lock);
73
74 while (freed < nr_to_scan) {
75 struct msm_gem_object *msm_obj = list_first_entry_or_null(
76 list, typeof(*msm_obj), mm_list);
77
78 if (!msm_obj)
79 break;
80
81 list_move_tail(&msm_obj->mm_list, &still_in_list);
82
83 /*
84 * If it is in the process of being freed, msm_gem_free_object
85 * can be blocked on mm_lock waiting to remove it. So just
86 * skip it.
87 */
88 if (!kref_get_unless_zero(&msm_obj->base.refcount))
89 continue;
90
91 /*
92 * Now that we own a reference, we can drop mm_lock for the
93 * rest of the loop body, to reduce contention with the
94 * retire_submit path (which could make more objects purgeable)
95 */
96
97 mutex_unlock(&priv->mm_lock);
98
99 /*
100 * Note that this still needs to be trylock, since we can
101 * hit shrinker in response to trying to get backing pages
102 * for this obj (ie. while it's lock is already held)
103 */
104 if (!msm_gem_trylock(&msm_obj->base))
105 goto tail;
106
107 if (shrink(msm_obj))
108 freed += msm_obj->base.size >> PAGE_SHIFT;
109
110 msm_gem_unlock(&msm_obj->base);
111
112tail:
113 drm_gem_object_put(&msm_obj->base);
114 mutex_lock(&priv->mm_lock);
115 }
116
117 list_splice_tail(&still_in_list, list);
118 mutex_unlock(&priv->mm_lock);
119
120 return freed;
121}
122
123static unsigned long
124msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
125{
126 struct msm_drm_private *priv =
127 container_of(shrinker, struct msm_drm_private, shrinker);
128 unsigned long freed;
129
130 freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge);
131
132 if (freed > 0)
133 trace_msm_gem_purge(freed << PAGE_SHIFT);
134
135 if (can_swap() && freed < sc->nr_to_scan) {
136 int evicted = scan(priv, sc->nr_to_scan - freed,
137 &priv->inactive_willneed, evict);
138
139 if (evicted > 0)
140 trace_msm_gem_evict(evicted << PAGE_SHIFT);
141
142 freed += evicted;
143 }
144
145 return (freed > 0) ? freed : SHRINK_STOP;
146}
147
148#ifdef CONFIG_DEBUG_FS
149unsigned long
150msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
151{
152 struct msm_drm_private *priv = dev->dev_private;
153 struct shrink_control sc = {
154 .nr_to_scan = nr_to_scan,
155 };
156 int ret;
157
158 fs_reclaim_acquire(GFP_KERNEL);
159 ret = msm_gem_shrinker_scan(&priv->shrinker, &sc);
160 fs_reclaim_release(GFP_KERNEL);
161
162 return ret;
163}
164#endif
165
166/* since we don't know any better, lets bail after a few
167 * and if necessary the shrinker will be invoked again.
168 * Seems better than unmapping *everything*
169 */
170static const int vmap_shrink_limit = 15;
171
172static bool
173vmap_shrink(struct msm_gem_object *msm_obj)
174{
175 if (!is_vunmapable(msm_obj))
176 return false;
177
178 msm_gem_vunmap(&msm_obj->base);
179
180 return true;
181}
182
183static int
184msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
185{
186 struct msm_drm_private *priv =
187 container_of(nb, struct msm_drm_private, vmap_notifier);
188 struct list_head *mm_lists[] = {
189 &priv->inactive_dontneed,
190 &priv->inactive_willneed,
191 priv->gpu ? &priv->gpu->active_list : NULL,
192 NULL,
193 };
194 unsigned idx, unmapped = 0;
195
196 for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) {
197 unmapped += scan(priv, vmap_shrink_limit - unmapped,
198 mm_lists[idx], vmap_shrink);
199 }
200
201 *(unsigned long *)ptr += unmapped;
202
203 if (unmapped > 0)
204 trace_msm_gem_purge_vmaps(unmapped);
205
206 return NOTIFY_DONE;
207}
208
209/**
210 * msm_gem_shrinker_init - Initialize msm shrinker
211 * @dev: drm device
212 *
213 * This function registers and sets up the msm shrinker.
214 */
215void msm_gem_shrinker_init(struct drm_device *dev)
216{
217 struct msm_drm_private *priv = dev->dev_private;
218 priv->shrinker.count_objects = msm_gem_shrinker_count;
219 priv->shrinker.scan_objects = msm_gem_shrinker_scan;
220 priv->shrinker.seeks = DEFAULT_SEEKS;
221 WARN_ON(register_shrinker(&priv->shrinker));
222
223 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
224 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
225}
226
227/**
228 * msm_gem_shrinker_cleanup - Clean up msm shrinker
229 * @dev: drm device
230 *
231 * This function unregisters the msm shrinker.
232 */
233void msm_gem_shrinker_cleanup(struct drm_device *dev)
234{
235 struct msm_drm_private *priv = dev->dev_private;
236
237 if (priv->shrinker.nr_deferred) {
238 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
239 unregister_shrinker(&priv->shrinker);
240 }
241}
1/*
2 * Copyright (C) 2016 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include "msm_drv.h"
19#include "msm_gem.h"
20
21static bool msm_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
22{
23 /* NOTE: we are *closer* to being able to get rid of
24 * mutex_trylock_recursive().. the msm_gem code itself does
25 * not need struct_mutex, although codepaths that can trigger
26 * shrinker are still called in code-paths that hold the
27 * struct_mutex.
28 *
29 * Also, msm_obj->madv is protected by struct_mutex.
30 *
31 * The next step is probably split out a seperate lock for
32 * protecting inactive_list, so that shrinker does not need
33 * struct_mutex.
34 */
35 switch (mutex_trylock_recursive(&dev->struct_mutex)) {
36 case MUTEX_TRYLOCK_FAILED:
37 return false;
38
39 case MUTEX_TRYLOCK_SUCCESS:
40 *unlock = true;
41 return true;
42
43 case MUTEX_TRYLOCK_RECURSIVE:
44 *unlock = false;
45 return true;
46 }
47
48 BUG();
49}
50
51static unsigned long
52msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
53{
54 struct msm_drm_private *priv =
55 container_of(shrinker, struct msm_drm_private, shrinker);
56 struct drm_device *dev = priv->dev;
57 struct msm_gem_object *msm_obj;
58 unsigned long count = 0;
59 bool unlock;
60
61 if (!msm_gem_shrinker_lock(dev, &unlock))
62 return 0;
63
64 list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
65 if (is_purgeable(msm_obj))
66 count += msm_obj->base.size >> PAGE_SHIFT;
67 }
68
69 if (unlock)
70 mutex_unlock(&dev->struct_mutex);
71
72 return count;
73}
74
75static unsigned long
76msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
77{
78 struct msm_drm_private *priv =
79 container_of(shrinker, struct msm_drm_private, shrinker);
80 struct drm_device *dev = priv->dev;
81 struct msm_gem_object *msm_obj;
82 unsigned long freed = 0;
83 bool unlock;
84
85 if (!msm_gem_shrinker_lock(dev, &unlock))
86 return SHRINK_STOP;
87
88 list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
89 if (freed >= sc->nr_to_scan)
90 break;
91 if (is_purgeable(msm_obj)) {
92 msm_gem_purge(&msm_obj->base, OBJ_LOCK_SHRINKER);
93 freed += msm_obj->base.size >> PAGE_SHIFT;
94 }
95 }
96
97 if (unlock)
98 mutex_unlock(&dev->struct_mutex);
99
100 if (freed > 0)
101 pr_info_ratelimited("Purging %lu bytes\n", freed << PAGE_SHIFT);
102
103 return freed;
104}
105
106static int
107msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
108{
109 struct msm_drm_private *priv =
110 container_of(nb, struct msm_drm_private, vmap_notifier);
111 struct drm_device *dev = priv->dev;
112 struct msm_gem_object *msm_obj;
113 unsigned unmapped = 0;
114 bool unlock;
115
116 if (!msm_gem_shrinker_lock(dev, &unlock))
117 return NOTIFY_DONE;
118
119 list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
120 if (is_vunmapable(msm_obj)) {
121 msm_gem_vunmap(&msm_obj->base, OBJ_LOCK_SHRINKER);
122 /* since we don't know any better, lets bail after a few
123 * and if necessary the shrinker will be invoked again.
124 * Seems better than unmapping *everything*
125 */
126 if (++unmapped >= 15)
127 break;
128 }
129 }
130
131 if (unlock)
132 mutex_unlock(&dev->struct_mutex);
133
134 *(unsigned long *)ptr += unmapped;
135
136 if (unmapped > 0)
137 pr_info_ratelimited("Purging %u vmaps\n", unmapped);
138
139 return NOTIFY_DONE;
140}
141
142/**
143 * msm_gem_shrinker_init - Initialize msm shrinker
144 * @dev_priv: msm device
145 *
146 * This function registers and sets up the msm shrinker.
147 */
148void msm_gem_shrinker_init(struct drm_device *dev)
149{
150 struct msm_drm_private *priv = dev->dev_private;
151 priv->shrinker.count_objects = msm_gem_shrinker_count;
152 priv->shrinker.scan_objects = msm_gem_shrinker_scan;
153 priv->shrinker.seeks = DEFAULT_SEEKS;
154 WARN_ON(register_shrinker(&priv->shrinker));
155
156 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
157 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
158}
159
160/**
161 * msm_gem_shrinker_cleanup - Clean up msm shrinker
162 * @dev_priv: msm device
163 *
164 * This function unregisters the msm shrinker.
165 */
166void msm_gem_shrinker_cleanup(struct drm_device *dev)
167{
168 struct msm_drm_private *priv = dev->dev_private;
169
170 if (priv->shrinker.nr_deferred) {
171 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
172 unregister_shrinker(&priv->shrinker);
173 }
174}