Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include <linux/slab.h>
7
8#include <drm/ttm/ttm_bo_driver.h>
9#include <drm/ttm/ttm_placement.h>
10
11#include <drm/drm_buddy.h>
12
13#include "i915_ttm_buddy_manager.h"
14
15#include "i915_gem.h"
16
17struct i915_ttm_buddy_manager {
18 struct ttm_resource_manager manager;
19 struct drm_buddy mm;
20 struct list_head reserved;
21 struct mutex lock;
22 unsigned long visible_size;
23 unsigned long visible_avail;
24 unsigned long visible_reserved;
25 u64 default_page_size;
26};
27
28static struct i915_ttm_buddy_manager *
29to_buddy_manager(struct ttm_resource_manager *man)
30{
31 return container_of(man, struct i915_ttm_buddy_manager, manager);
32}
33
34static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
35 struct ttm_buffer_object *bo,
36 const struct ttm_place *place,
37 struct ttm_resource **res)
38{
39 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
40 struct i915_ttm_buddy_resource *bman_res;
41 struct drm_buddy *mm = &bman->mm;
42 unsigned long n_pages, lpfn;
43 u64 min_page_size;
44 u64 size;
45 int err;
46
47 lpfn = place->lpfn;
48 if (!lpfn)
49 lpfn = man->size;
50
51 bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL);
52 if (!bman_res)
53 return -ENOMEM;
54
55 ttm_resource_init(bo, place, &bman_res->base);
56 INIT_LIST_HEAD(&bman_res->blocks);
57 bman_res->mm = mm;
58
59 if (place->flags & TTM_PL_FLAG_TOPDOWN)
60 bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
61
62 if (place->fpfn || lpfn != man->size)
63 bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
64
65 GEM_BUG_ON(!bman_res->base.size);
66 size = bman_res->base.size;
67
68 min_page_size = bman->default_page_size;
69 if (bo->page_alignment)
70 min_page_size = bo->page_alignment << PAGE_SHIFT;
71
72 GEM_BUG_ON(min_page_size < mm->chunk_size);
73 GEM_BUG_ON(!IS_ALIGNED(size, min_page_size));
74
75 if (place->fpfn + PFN_UP(bman_res->base.size) != place->lpfn &&
76 place->flags & TTM_PL_FLAG_CONTIGUOUS) {
77 unsigned long pages;
78
79 size = roundup_pow_of_two(size);
80 min_page_size = size;
81
82 pages = size >> ilog2(mm->chunk_size);
83 if (pages > lpfn)
84 lpfn = pages;
85 }
86
87 if (size > lpfn << PAGE_SHIFT) {
88 err = -E2BIG;
89 goto err_free_res;
90 }
91
92 n_pages = size >> ilog2(mm->chunk_size);
93
94 mutex_lock(&bman->lock);
95 if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) {
96 mutex_unlock(&bman->lock);
97 err = -ENOSPC;
98 goto err_free_res;
99 }
100
101 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
102 (u64)lpfn << PAGE_SHIFT,
103 (u64)n_pages << PAGE_SHIFT,
104 min_page_size,
105 &bman_res->blocks,
106 bman_res->flags);
107 if (unlikely(err))
108 goto err_free_blocks;
109
110 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
111 u64 original_size = (u64)bman_res->base.size;
112
113 drm_buddy_block_trim(mm,
114 original_size,
115 &bman_res->blocks);
116 }
117
118 if (lpfn <= bman->visible_size) {
119 bman_res->used_visible_size = PFN_UP(bman_res->base.size);
120 } else {
121 struct drm_buddy_block *block;
122
123 list_for_each_entry(block, &bman_res->blocks, link) {
124 unsigned long start =
125 drm_buddy_block_offset(block) >> PAGE_SHIFT;
126
127 if (start < bman->visible_size) {
128 unsigned long end = start +
129 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
130
131 bman_res->used_visible_size +=
132 min(end, bman->visible_size) - start;
133 }
134 }
135 }
136
137 if (bman_res->used_visible_size)
138 bman->visible_avail -= bman_res->used_visible_size;
139
140 mutex_unlock(&bman->lock);
141
142 if (place->lpfn - place->fpfn == n_pages)
143 bman_res->base.start = place->fpfn;
144 else if (lpfn <= bman->visible_size)
145 bman_res->base.start = 0;
146 else
147 bman_res->base.start = bman->visible_size;
148
149 *res = &bman_res->base;
150 return 0;
151
152err_free_blocks:
153 drm_buddy_free_list(mm, &bman_res->blocks);
154 mutex_unlock(&bman->lock);
155err_free_res:
156 ttm_resource_fini(man, &bman_res->base);
157 kfree(bman_res);
158 return err;
159}
160
161static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
162 struct ttm_resource *res)
163{
164 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
165 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
166
167 mutex_lock(&bman->lock);
168 drm_buddy_free_list(&bman->mm, &bman_res->blocks);
169 bman->visible_avail += bman_res->used_visible_size;
170 mutex_unlock(&bman->lock);
171
172 ttm_resource_fini(man, res);
173 kfree(bman_res);
174}
175
176static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
177 struct ttm_resource *res,
178 const struct ttm_place *place,
179 size_t size)
180{
181 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
182 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
183 struct drm_buddy *mm = &bman->mm;
184 struct drm_buddy_block *block;
185
186 if (!place->fpfn && !place->lpfn)
187 return true;
188
189 GEM_BUG_ON(!place->lpfn);
190
191 /*
192 * If we just want something mappable then we can quickly check
193 * if the current victim resource is using any of the CPU
194 * visible portion.
195 */
196 if (!place->fpfn &&
197 place->lpfn == i915_ttm_buddy_man_visible_size(man))
198 return bman_res->used_visible_size > 0;
199
200 /* Check each drm buddy block individually */
201 list_for_each_entry(block, &bman_res->blocks, link) {
202 unsigned long fpfn =
203 drm_buddy_block_offset(block) >> PAGE_SHIFT;
204 unsigned long lpfn = fpfn +
205 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
206
207 if (place->fpfn < lpfn && place->lpfn > fpfn)
208 return true;
209 }
210
211 return false;
212}
213
214static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
215 struct ttm_resource *res,
216 const struct ttm_place *place,
217 size_t size)
218{
219 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
220 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
221 struct drm_buddy *mm = &bman->mm;
222 struct drm_buddy_block *block;
223
224 if (!place->fpfn && !place->lpfn)
225 return true;
226
227 GEM_BUG_ON(!place->lpfn);
228
229 if (!place->fpfn &&
230 place->lpfn == i915_ttm_buddy_man_visible_size(man))
231 return bman_res->used_visible_size == PFN_UP(res->size);
232
233 /* Check each drm buddy block individually */
234 list_for_each_entry(block, &bman_res->blocks, link) {
235 unsigned long fpfn =
236 drm_buddy_block_offset(block) >> PAGE_SHIFT;
237 unsigned long lpfn = fpfn +
238 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
239
240 if (fpfn < place->fpfn || lpfn > place->lpfn)
241 return false;
242 }
243
244 return true;
245}
246
247static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
248 struct drm_printer *printer)
249{
250 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
251 struct drm_buddy_block *block;
252
253 mutex_lock(&bman->lock);
254 drm_printf(printer, "default_page_size: %lluKiB\n",
255 bman->default_page_size >> 10);
256 drm_printf(printer, "visible_avail: %lluMiB\n",
257 (u64)bman->visible_avail << PAGE_SHIFT >> 20);
258 drm_printf(printer, "visible_size: %lluMiB\n",
259 (u64)bman->visible_size << PAGE_SHIFT >> 20);
260 drm_printf(printer, "visible_reserved: %lluMiB\n",
261 (u64)bman->visible_reserved << PAGE_SHIFT >> 20);
262
263 drm_buddy_print(&bman->mm, printer);
264
265 drm_printf(printer, "reserved:\n");
266 list_for_each_entry(block, &bman->reserved, link)
267 drm_buddy_block_print(&bman->mm, block, printer);
268 mutex_unlock(&bman->lock);
269}
270
271static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = {
272 .alloc = i915_ttm_buddy_man_alloc,
273 .free = i915_ttm_buddy_man_free,
274 .intersects = i915_ttm_buddy_man_intersects,
275 .compatible = i915_ttm_buddy_man_compatible,
276 .debug = i915_ttm_buddy_man_debug,
277};
278
279/**
280 * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager
281 * @bdev: The ttm device
282 * @type: Memory type we want to manage
283 * @use_tt: Set use_tt for the manager
284 * @size: The size in bytes to manage
285 * @visible_size: The CPU visible size in bytes to manage
286 * @default_page_size: The default minimum page size in bytes for allocations,
287 * this must be at least as large as @chunk_size, and can be overridden by
288 * setting the BO page_alignment, to be larger or smaller as needed.
289 * @chunk_size: The minimum page size in bytes for our allocations i.e
290 * order-zero
291 *
292 * Note that the starting address is assumed to be zero here, since this
293 * simplifies keeping the property where allocated blocks having natural
294 * power-of-two alignment. So long as the real starting address is some large
295 * power-of-two, or naturally start from zero, then this should be fine. Also
296 * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment
297 * if say there is some unusable range from the start of the region. We can
298 * revisit this in the future and make the interface accept an actual starting
299 * offset and let it take care of the rest.
300 *
301 * Note that if the @size is not aligned to the @chunk_size then we perform the
302 * required rounding to get the usable size. The final size in pages can be
303 * taken from &ttm_resource_manager.size.
304 *
305 * Return: 0 on success, negative error code on failure.
306 */
307int i915_ttm_buddy_man_init(struct ttm_device *bdev,
308 unsigned int type, bool use_tt,
309 u64 size, u64 visible_size, u64 default_page_size,
310 u64 chunk_size)
311{
312 struct ttm_resource_manager *man;
313 struct i915_ttm_buddy_manager *bman;
314 int err;
315
316 bman = kzalloc(sizeof(*bman), GFP_KERNEL);
317 if (!bman)
318 return -ENOMEM;
319
320 err = drm_buddy_init(&bman->mm, size, chunk_size);
321 if (err)
322 goto err_free_bman;
323
324 mutex_init(&bman->lock);
325 INIT_LIST_HEAD(&bman->reserved);
326 GEM_BUG_ON(default_page_size < chunk_size);
327 bman->default_page_size = default_page_size;
328 bman->visible_size = visible_size >> PAGE_SHIFT;
329 bman->visible_avail = bman->visible_size;
330
331 man = &bman->manager;
332 man->use_tt = use_tt;
333 man->func = &i915_ttm_buddy_manager_func;
334 ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT);
335
336 ttm_resource_manager_set_used(man, true);
337 ttm_set_driver_manager(bdev, type, man);
338
339 return 0;
340
341err_free_bman:
342 kfree(bman);
343 return err;
344}
345
346/**
347 * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager
348 * @bdev: The ttm device
349 * @type: Memory type we want to manage
350 *
351 * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will
352 * also be freed for us here.
353 *
354 * Return: 0 on success, negative error code on failure.
355 */
356int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
357{
358 struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
359 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
360 struct drm_buddy *mm = &bman->mm;
361 int ret;
362
363 ttm_resource_manager_set_used(man, false);
364
365 ret = ttm_resource_manager_evict_all(bdev, man);
366 if (ret)
367 return ret;
368
369 ttm_set_driver_manager(bdev, type, NULL);
370
371 mutex_lock(&bman->lock);
372 drm_buddy_free_list(mm, &bman->reserved);
373 drm_buddy_fini(mm);
374 bman->visible_avail += bman->visible_reserved;
375 WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
376 mutex_unlock(&bman->lock);
377
378 ttm_resource_manager_cleanup(man);
379 kfree(bman);
380
381 return 0;
382}
383
384/**
385 * i915_ttm_buddy_man_reserve - Reserve address range
386 * @man: The buddy allocator ttm manager
387 * @start: The offset in bytes, where the region start is assumed to be zero
388 * @size: The size in bytes
389 *
390 * Note that the starting address for the region is always assumed to be zero.
391 *
392 * Return: 0 on success, negative error code on failure.
393 */
394int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
395 u64 start, u64 size)
396{
397 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
398 struct drm_buddy *mm = &bman->mm;
399 unsigned long fpfn = start >> PAGE_SHIFT;
400 unsigned long flags = 0;
401 int ret;
402
403 flags |= DRM_BUDDY_RANGE_ALLOCATION;
404
405 mutex_lock(&bman->lock);
406 ret = drm_buddy_alloc_blocks(mm, start,
407 start + size,
408 size, mm->chunk_size,
409 &bman->reserved,
410 flags);
411
412 if (fpfn < bman->visible_size) {
413 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT);
414 unsigned long visible = min(lpfn, bman->visible_size) - fpfn;
415
416 bman->visible_reserved += visible;
417 bman->visible_avail -= visible;
418 }
419 mutex_unlock(&bman->lock);
420
421 return ret;
422}
423
424/**
425 * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion
426 * in pages.
427 * @man: The buddy allocator ttm manager
428 */
429u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man)
430{
431 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
432
433 return bman->visible_size;
434}
435
436/**
437 * i915_ttm_buddy_man_avail - Query the avail tracking for the manager.
438 *
439 * @man: The buddy allocator ttm manager
440 * @avail: The total available memory in pages for the entire manager.
441 * @visible_avail: The total available memory in pages for the CPU visible
442 * portion. Note that this will always give the same value as @avail on
443 * configurations that don't have a small BAR.
444 */
445void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man,
446 u64 *avail, u64 *visible_avail)
447{
448 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
449
450 mutex_lock(&bman->lock);
451 *avail = bman->mm.avail >> PAGE_SHIFT;
452 *visible_avail = bman->visible_avail;
453 mutex_unlock(&bman->lock);
454}
455
456#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
457void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man,
458 u64 size)
459{
460 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
461
462 bman->visible_size = size;
463}
464#endif