Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2016 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include <linux/vmalloc.h>
8#include <linux/sched/mm.h>
9
10#include "msm_drv.h"
11#include "msm_gem.h"
12#include "msm_gpu.h"
13#include "msm_gpu_trace.h"
14
15/* Default disabled for now until it has some more testing on the different
16 * iommu combinations that can be paired with the driver:
17 */
18static bool enable_eviction = true;
19MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers");
20module_param(enable_eviction, bool, 0600);
21
22static bool can_swap(void)
23{
24 return enable_eviction && get_nr_swap_pages() > 0;
25}
26
27static bool can_block(struct shrink_control *sc)
28{
29 if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM))
30 return false;
31 return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM);
32}
33
34static unsigned long
35msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
36{
37 struct msm_drm_private *priv = shrinker->private_data;
38 unsigned count = priv->lru.dontneed.count;
39
40 if (can_swap())
41 count += priv->lru.willneed.count;
42
43 return count;
44}
45
46static bool
47purge(struct drm_gem_object *obj)
48{
49 if (!is_purgeable(to_msm_bo(obj)))
50 return false;
51
52 if (msm_gem_active(obj))
53 return false;
54
55 msm_gem_purge(obj);
56
57 return true;
58}
59
60static bool
61evict(struct drm_gem_object *obj)
62{
63 if (is_unevictable(to_msm_bo(obj)))
64 return false;
65
66 if (msm_gem_active(obj))
67 return false;
68
69 msm_gem_evict(obj);
70
71 return true;
72}
73
74static bool
75wait_for_idle(struct drm_gem_object *obj)
76{
77 enum dma_resv_usage usage = dma_resv_usage_rw(true);
78 return dma_resv_wait_timeout(obj->resv, usage, false, 10) > 0;
79}
80
81static bool
82active_purge(struct drm_gem_object *obj)
83{
84 if (!wait_for_idle(obj))
85 return false;
86
87 return purge(obj);
88}
89
90static bool
91active_evict(struct drm_gem_object *obj)
92{
93 if (!wait_for_idle(obj))
94 return false;
95
96 return evict(obj);
97}
98
99static unsigned long
100msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
101{
102 struct msm_drm_private *priv = shrinker->private_data;
103 struct {
104 struct drm_gem_lru *lru;
105 bool (*shrink)(struct drm_gem_object *obj);
106 bool cond;
107 unsigned long freed;
108 unsigned long remaining;
109 } stages[] = {
110 /* Stages of progressively more aggressive/expensive reclaim: */
111 { &priv->lru.dontneed, purge, true },
112 { &priv->lru.willneed, evict, can_swap() },
113 { &priv->lru.dontneed, active_purge, can_block(sc) },
114 { &priv->lru.willneed, active_evict, can_swap() && can_block(sc) },
115 };
116 long nr = sc->nr_to_scan;
117 unsigned long freed = 0;
118 unsigned long remaining = 0;
119
120 for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
121 if (!stages[i].cond)
122 continue;
123 stages[i].freed =
124 drm_gem_lru_scan(stages[i].lru, nr,
125 &stages[i].remaining,
126 stages[i].shrink);
127 nr -= stages[i].freed;
128 freed += stages[i].freed;
129 remaining += stages[i].remaining;
130 }
131
132 if (freed) {
133 trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed,
134 stages[1].freed, stages[2].freed,
135 stages[3].freed);
136 }
137
138 return (freed > 0 && remaining > 0) ? freed : SHRINK_STOP;
139}
140
141#ifdef CONFIG_DEBUG_FS
142unsigned long
143msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
144{
145 struct msm_drm_private *priv = dev->dev_private;
146 struct shrink_control sc = {
147 .nr_to_scan = nr_to_scan,
148 };
149 unsigned long ret = SHRINK_STOP;
150
151 fs_reclaim_acquire(GFP_KERNEL);
152 if (priv->shrinker)
153 ret = msm_gem_shrinker_scan(priv->shrinker, &sc);
154 fs_reclaim_release(GFP_KERNEL);
155
156 return ret;
157}
158#endif
159
160/* since we don't know any better, lets bail after a few
161 * and if necessary the shrinker will be invoked again.
162 * Seems better than unmapping *everything*
163 */
164static const int vmap_shrink_limit = 15;
165
166static bool
167vmap_shrink(struct drm_gem_object *obj)
168{
169 if (!is_vunmapable(to_msm_bo(obj)))
170 return false;
171
172 msm_gem_vunmap(obj);
173
174 return true;
175}
176
177static int
178msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
179{
180 struct msm_drm_private *priv =
181 container_of(nb, struct msm_drm_private, vmap_notifier);
182 struct drm_gem_lru *lrus[] = {
183 &priv->lru.dontneed,
184 &priv->lru.willneed,
185 &priv->lru.pinned,
186 NULL,
187 };
188 unsigned idx, unmapped = 0;
189 unsigned long remaining = 0;
190
191 for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
192 unmapped += drm_gem_lru_scan(lrus[idx],
193 vmap_shrink_limit - unmapped,
194 &remaining,
195 vmap_shrink);
196 }
197
198 *(unsigned long *)ptr += unmapped;
199
200 if (unmapped > 0)
201 trace_msm_gem_purge_vmaps(unmapped);
202
203 return NOTIFY_DONE;
204}
205
206/**
207 * msm_gem_shrinker_init - Initialize msm shrinker
208 * @dev: drm device
209 *
210 * This function registers and sets up the msm shrinker.
211 */
212int msm_gem_shrinker_init(struct drm_device *dev)
213{
214 struct msm_drm_private *priv = dev->dev_private;
215
216 priv->shrinker = shrinker_alloc(0, "drm-msm_gem");
217 if (!priv->shrinker)
218 return -ENOMEM;
219
220 priv->shrinker->count_objects = msm_gem_shrinker_count;
221 priv->shrinker->scan_objects = msm_gem_shrinker_scan;
222 priv->shrinker->private_data = priv;
223
224 shrinker_register(priv->shrinker);
225
226 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
227 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
228
229 return 0;
230}
231
232/**
233 * msm_gem_shrinker_cleanup - Clean up msm shrinker
234 * @dev: drm device
235 *
236 * This function unregisters the msm shrinker.
237 */
238void msm_gem_shrinker_cleanup(struct drm_device *dev)
239{
240 struct msm_drm_private *priv = dev->dev_private;
241
242 if (priv->shrinker) {
243 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
244 shrinker_free(priv->shrinker);
245 }
246}
1/*
2 * Copyright (C) 2016 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include "msm_drv.h"
19#include "msm_gem.h"
20
21static bool msm_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
22{
23 switch (mutex_trylock_recursive(&dev->struct_mutex)) {
24 case MUTEX_TRYLOCK_FAILED:
25 return false;
26
27 case MUTEX_TRYLOCK_SUCCESS:
28 *unlock = true;
29 return true;
30
31 case MUTEX_TRYLOCK_RECURSIVE:
32 *unlock = false;
33 return true;
34 }
35
36 BUG();
37}
38
39static unsigned long
40msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
41{
42 struct msm_drm_private *priv =
43 container_of(shrinker, struct msm_drm_private, shrinker);
44 struct drm_device *dev = priv->dev;
45 struct msm_gem_object *msm_obj;
46 unsigned long count = 0;
47 bool unlock;
48
49 if (!msm_gem_shrinker_lock(dev, &unlock))
50 return 0;
51
52 list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
53 if (is_purgeable(msm_obj))
54 count += msm_obj->base.size >> PAGE_SHIFT;
55 }
56
57 if (unlock)
58 mutex_unlock(&dev->struct_mutex);
59
60 return count;
61}
62
63static unsigned long
64msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
65{
66 struct msm_drm_private *priv =
67 container_of(shrinker, struct msm_drm_private, shrinker);
68 struct drm_device *dev = priv->dev;
69 struct msm_gem_object *msm_obj;
70 unsigned long freed = 0;
71 bool unlock;
72
73 if (!msm_gem_shrinker_lock(dev, &unlock))
74 return SHRINK_STOP;
75
76 list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
77 if (freed >= sc->nr_to_scan)
78 break;
79 if (is_purgeable(msm_obj)) {
80 msm_gem_purge(&msm_obj->base);
81 freed += msm_obj->base.size >> PAGE_SHIFT;
82 }
83 }
84
85 if (unlock)
86 mutex_unlock(&dev->struct_mutex);
87
88 if (freed > 0)
89 pr_info_ratelimited("Purging %lu bytes\n", freed << PAGE_SHIFT);
90
91 return freed;
92}
93
94static int
95msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
96{
97 struct msm_drm_private *priv =
98 container_of(nb, struct msm_drm_private, vmap_notifier);
99 struct drm_device *dev = priv->dev;
100 struct msm_gem_object *msm_obj;
101 unsigned unmapped = 0;
102 bool unlock;
103
104 if (!msm_gem_shrinker_lock(dev, &unlock))
105 return NOTIFY_DONE;
106
107 list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
108 if (is_vunmapable(msm_obj)) {
109 msm_gem_vunmap(&msm_obj->base);
110 /* since we don't know any better, lets bail after a few
111 * and if necessary the shrinker will be invoked again.
112 * Seems better than unmapping *everything*
113 */
114 if (++unmapped >= 15)
115 break;
116 }
117 }
118
119 if (unlock)
120 mutex_unlock(&dev->struct_mutex);
121
122 *(unsigned long *)ptr += unmapped;
123
124 if (unmapped > 0)
125 pr_info_ratelimited("Purging %u vmaps\n", unmapped);
126
127 return NOTIFY_DONE;
128}
129
130/**
131 * msm_gem_shrinker_init - Initialize msm shrinker
132 * @dev_priv: msm device
133 *
134 * This function registers and sets up the msm shrinker.
135 */
136void msm_gem_shrinker_init(struct drm_device *dev)
137{
138 struct msm_drm_private *priv = dev->dev_private;
139 priv->shrinker.count_objects = msm_gem_shrinker_count;
140 priv->shrinker.scan_objects = msm_gem_shrinker_scan;
141 priv->shrinker.seeks = DEFAULT_SEEKS;
142 WARN_ON(register_shrinker(&priv->shrinker));
143
144 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
145 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
146}
147
148/**
149 * msm_gem_shrinker_cleanup - Clean up msm shrinker
150 * @dev_priv: msm device
151 *
152 * This function unregisters the msm shrinker.
153 */
154void msm_gem_shrinker_cleanup(struct drm_device *dev)
155{
156 struct msm_drm_private *priv = dev->dev_private;
157
158 if (priv->shrinker.nr_deferred) {
159 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
160 unregister_shrinker(&priv->shrinker);
161 }
162}