Loading...
1/*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#include <drm/drmP.h>
26#include "amdgpu.h"
27
28struct amdgpu_vram_mgr {
29 struct drm_mm mm;
30 spinlock_t lock;
31 atomic64_t usage;
32 atomic64_t vis_usage;
33};
34
35/**
36 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
37 *
38 * @man: TTM memory type manager
39 * @p_size: maximum size of VRAM
40 *
41 * Allocate and initialize the VRAM manager.
42 */
43static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
44 unsigned long p_size)
45{
46 struct amdgpu_vram_mgr *mgr;
47
48 mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
49 if (!mgr)
50 return -ENOMEM;
51
52 drm_mm_init(&mgr->mm, 0, p_size);
53 spin_lock_init(&mgr->lock);
54 man->priv = mgr;
55 return 0;
56}
57
58/**
59 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
60 *
61 * @man: TTM memory type manager
62 *
63 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
64 * allocated inside it.
65 */
66static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
67{
68 struct amdgpu_vram_mgr *mgr = man->priv;
69
70 spin_lock(&mgr->lock);
71 drm_mm_takedown(&mgr->mm);
72 spin_unlock(&mgr->lock);
73 kfree(mgr);
74 man->priv = NULL;
75 return 0;
76}
77
78/**
79 * amdgpu_vram_mgr_vis_size - Calculate visible node size
80 *
81 * @adev: amdgpu device structure
82 * @node: MM node structure
83 *
84 * Calculate how many bytes of the MM node are inside visible VRAM
85 */
86static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
87 struct drm_mm_node *node)
88{
89 uint64_t start = node->start << PAGE_SHIFT;
90 uint64_t end = (node->size + node->start) << PAGE_SHIFT;
91
92 if (start >= adev->gmc.visible_vram_size)
93 return 0;
94
95 return (end > adev->gmc.visible_vram_size ?
96 adev->gmc.visible_vram_size : end) - start;
97}
98
99/**
100 * amdgpu_vram_mgr_new - allocate new ranges
101 *
102 * @man: TTM memory type manager
103 * @tbo: TTM BO we need this range for
104 * @place: placement flags and restrictions
105 * @mem: the resulting mem object
106 *
107 * Allocate VRAM for the given BO.
108 */
109static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
110 struct ttm_buffer_object *tbo,
111 const struct ttm_place *place,
112 struct ttm_mem_reg *mem)
113{
114 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
115 struct amdgpu_vram_mgr *mgr = man->priv;
116 struct drm_mm *mm = &mgr->mm;
117 struct drm_mm_node *nodes;
118 enum drm_mm_insert_mode mode;
119 unsigned long lpfn, num_nodes, pages_per_node, pages_left;
120 uint64_t usage = 0, vis_usage = 0;
121 unsigned i;
122 int r;
123
124 lpfn = place->lpfn;
125 if (!lpfn)
126 lpfn = man->size;
127
128 if (place->flags & TTM_PL_FLAG_CONTIGUOUS ||
129 amdgpu_vram_page_split == -1) {
130 pages_per_node = ~0ul;
131 num_nodes = 1;
132 } else {
133 pages_per_node = max((uint32_t)amdgpu_vram_page_split,
134 mem->page_alignment);
135 num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
136 }
137
138 nodes = kcalloc(num_nodes, sizeof(*nodes), GFP_KERNEL);
139 if (!nodes)
140 return -ENOMEM;
141
142 mode = DRM_MM_INSERT_BEST;
143 if (place->flags & TTM_PL_FLAG_TOPDOWN)
144 mode = DRM_MM_INSERT_HIGH;
145
146 mem->start = 0;
147 pages_left = mem->num_pages;
148
149 spin_lock(&mgr->lock);
150 for (i = 0; i < num_nodes; ++i) {
151 unsigned long pages = min(pages_left, pages_per_node);
152 uint32_t alignment = mem->page_alignment;
153 unsigned long start;
154
155 if (pages == pages_per_node)
156 alignment = pages_per_node;
157
158 r = drm_mm_insert_node_in_range(mm, &nodes[i],
159 pages, alignment, 0,
160 place->fpfn, lpfn,
161 mode);
162 if (unlikely(r))
163 goto error;
164
165 usage += nodes[i].size << PAGE_SHIFT;
166 vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
167
168 /* Calculate a virtual BO start address to easily check if
169 * everything is CPU accessible.
170 */
171 start = nodes[i].start + nodes[i].size;
172 if (start > mem->num_pages)
173 start -= mem->num_pages;
174 else
175 start = 0;
176 mem->start = max(mem->start, start);
177 pages_left -= pages;
178 }
179 spin_unlock(&mgr->lock);
180
181 atomic64_add(usage, &mgr->usage);
182 atomic64_add(vis_usage, &mgr->vis_usage);
183
184 mem->mm_node = nodes;
185
186 return 0;
187
188error:
189 while (i--)
190 drm_mm_remove_node(&nodes[i]);
191 spin_unlock(&mgr->lock);
192
193 kfree(nodes);
194 return r == -ENOSPC ? 0 : r;
195}
196
197/**
198 * amdgpu_vram_mgr_del - free ranges
199 *
200 * @man: TTM memory type manager
201 * @tbo: TTM BO we need this range for
202 * @place: placement flags and restrictions
203 * @mem: TTM memory object
204 *
205 * Free the allocated VRAM again.
206 */
207static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
208 struct ttm_mem_reg *mem)
209{
210 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
211 struct amdgpu_vram_mgr *mgr = man->priv;
212 struct drm_mm_node *nodes = mem->mm_node;
213 uint64_t usage = 0, vis_usage = 0;
214 unsigned pages = mem->num_pages;
215
216 if (!mem->mm_node)
217 return;
218
219 spin_lock(&mgr->lock);
220 while (pages) {
221 pages -= nodes->size;
222 drm_mm_remove_node(nodes);
223 usage += nodes->size << PAGE_SHIFT;
224 vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
225 ++nodes;
226 }
227 spin_unlock(&mgr->lock);
228
229 atomic64_sub(usage, &mgr->usage);
230 atomic64_sub(vis_usage, &mgr->vis_usage);
231
232 kfree(mem->mm_node);
233 mem->mm_node = NULL;
234}
235
236/**
237 * amdgpu_vram_mgr_usage - how many bytes are used in this domain
238 *
239 * @man: TTM memory type manager
240 *
241 * Returns how many bytes are used in this domain.
242 */
243uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man)
244{
245 struct amdgpu_vram_mgr *mgr = man->priv;
246
247 return atomic64_read(&mgr->usage);
248}
249
250/**
251 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
252 *
253 * @man: TTM memory type manager
254 *
255 * Returns how many bytes are used in the visible part of VRAM
256 */
257uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man)
258{
259 struct amdgpu_vram_mgr *mgr = man->priv;
260
261 return atomic64_read(&mgr->vis_usage);
262}
263
264/**
265 * amdgpu_vram_mgr_debug - dump VRAM table
266 *
267 * @man: TTM memory type manager
268 * @printer: DRM printer to use
269 *
270 * Dump the table content using printk.
271 */
272static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
273 struct drm_printer *printer)
274{
275 struct amdgpu_vram_mgr *mgr = man->priv;
276
277 spin_lock(&mgr->lock);
278 drm_mm_print(&mgr->mm, printer);
279 spin_unlock(&mgr->lock);
280
281 drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
282 man->size, amdgpu_vram_mgr_usage(man) >> 20,
283 amdgpu_vram_mgr_vis_usage(man) >> 20);
284}
285
286const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
287 .init = amdgpu_vram_mgr_init,
288 .takedown = amdgpu_vram_mgr_fini,
289 .get_node = amdgpu_vram_mgr_new,
290 .put_node = amdgpu_vram_mgr_del,
291 .debug = amdgpu_vram_mgr_debug
292};
1/*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#include <drm/drmP.h>
26#include "amdgpu.h"
27
28struct amdgpu_vram_mgr {
29 struct drm_mm mm;
30 spinlock_t lock;
31};
32
33/**
34 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
35 *
36 * @man: TTM memory type manager
37 * @p_size: maximum size of VRAM
38 *
39 * Allocate and initialize the VRAM manager.
40 */
41static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
42 unsigned long p_size)
43{
44 struct amdgpu_vram_mgr *mgr;
45
46 mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
47 if (!mgr)
48 return -ENOMEM;
49
50 drm_mm_init(&mgr->mm, 0, p_size);
51 spin_lock_init(&mgr->lock);
52 man->priv = mgr;
53 return 0;
54}
55
56/**
57 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
58 *
59 * @man: TTM memory type manager
60 *
61 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
62 * allocated inside it.
63 */
64static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
65{
66 struct amdgpu_vram_mgr *mgr = man->priv;
67
68 spin_lock(&mgr->lock);
69 if (!drm_mm_clean(&mgr->mm)) {
70 spin_unlock(&mgr->lock);
71 return -EBUSY;
72 }
73
74 drm_mm_takedown(&mgr->mm);
75 spin_unlock(&mgr->lock);
76 kfree(mgr);
77 man->priv = NULL;
78 return 0;
79}
80
81/**
82 * amdgpu_vram_mgr_new - allocate new ranges
83 *
84 * @man: TTM memory type manager
85 * @tbo: TTM BO we need this range for
86 * @place: placement flags and restrictions
87 * @mem: the resulting mem object
88 *
89 * Allocate VRAM for the given BO.
90 */
91static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
92 struct ttm_buffer_object *tbo,
93 const struct ttm_place *place,
94 struct ttm_mem_reg *mem)
95{
96 struct amdgpu_bo *bo = container_of(tbo, struct amdgpu_bo, tbo);
97 struct amdgpu_vram_mgr *mgr = man->priv;
98 struct drm_mm *mm = &mgr->mm;
99 struct drm_mm_node *nodes;
100 enum drm_mm_search_flags sflags = DRM_MM_SEARCH_DEFAULT;
101 enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT;
102 unsigned long lpfn, num_nodes, pages_per_node, pages_left;
103 unsigned i;
104 int r;
105
106 lpfn = place->lpfn;
107 if (!lpfn)
108 lpfn = man->size;
109
110 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS ||
111 place->lpfn || amdgpu_vram_page_split == -1) {
112 pages_per_node = ~0ul;
113 num_nodes = 1;
114 } else {
115 pages_per_node = max((uint32_t)amdgpu_vram_page_split,
116 mem->page_alignment);
117 num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
118 }
119
120 nodes = kcalloc(num_nodes, sizeof(*nodes), GFP_KERNEL);
121 if (!nodes)
122 return -ENOMEM;
123
124 if (place->flags & TTM_PL_FLAG_TOPDOWN) {
125 sflags = DRM_MM_SEARCH_BELOW;
126 aflags = DRM_MM_CREATE_TOP;
127 }
128
129 pages_left = mem->num_pages;
130
131 spin_lock(&mgr->lock);
132 for (i = 0; i < num_nodes; ++i) {
133 unsigned long pages = min(pages_left, pages_per_node);
134 uint32_t alignment = mem->page_alignment;
135
136 if (pages == pages_per_node)
137 alignment = pages_per_node;
138 else
139 sflags |= DRM_MM_SEARCH_BEST;
140
141 r = drm_mm_insert_node_in_range_generic(mm, &nodes[i], pages,
142 alignment, 0,
143 place->fpfn, lpfn,
144 sflags, aflags);
145 if (unlikely(r))
146 goto error;
147
148 pages_left -= pages;
149 }
150 spin_unlock(&mgr->lock);
151
152 mem->start = num_nodes == 1 ? nodes[0].start : AMDGPU_BO_INVALID_OFFSET;
153 mem->mm_node = nodes;
154
155 return 0;
156
157error:
158 while (i--)
159 drm_mm_remove_node(&nodes[i]);
160 spin_unlock(&mgr->lock);
161
162 kfree(nodes);
163 return r == -ENOSPC ? 0 : r;
164}
165
166/**
167 * amdgpu_vram_mgr_del - free ranges
168 *
169 * @man: TTM memory type manager
170 * @tbo: TTM BO we need this range for
171 * @place: placement flags and restrictions
172 * @mem: TTM memory object
173 *
174 * Free the allocated VRAM again.
175 */
176static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
177 struct ttm_mem_reg *mem)
178{
179 struct amdgpu_vram_mgr *mgr = man->priv;
180 struct drm_mm_node *nodes = mem->mm_node;
181 unsigned pages = mem->num_pages;
182
183 if (!mem->mm_node)
184 return;
185
186 spin_lock(&mgr->lock);
187 while (pages) {
188 pages -= nodes->size;
189 drm_mm_remove_node(nodes);
190 ++nodes;
191 }
192 spin_unlock(&mgr->lock);
193
194 kfree(mem->mm_node);
195 mem->mm_node = NULL;
196}
197
198/**
199 * amdgpu_vram_mgr_debug - dump VRAM table
200 *
201 * @man: TTM memory type manager
202 * @prefix: text prefix
203 *
204 * Dump the table content using printk.
205 */
206static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
207 const char *prefix)
208{
209 struct amdgpu_vram_mgr *mgr = man->priv;
210
211 spin_lock(&mgr->lock);
212 drm_mm_debug_table(&mgr->mm, prefix);
213 spin_unlock(&mgr->lock);
214}
215
216const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
217 amdgpu_vram_mgr_init,
218 amdgpu_vram_mgr_fini,
219 amdgpu_vram_mgr_new,
220 amdgpu_vram_mgr_del,
221 amdgpu_vram_mgr_debug
222};