Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021-2022 Intel Corporation
4 * Copyright (C) 2021-2002 Red Hat
5 */
6
7#include <drm/drm_managed.h>
8
9#include <drm/ttm/ttm_placement.h>
10#include <drm/ttm/ttm_range_manager.h>
11
12#include "xe_bo.h"
13#include "xe_device.h"
14#include "xe_gt.h"
15#include "xe_res_cursor.h"
16#include "xe_ttm_vram_mgr.h"
17
18static inline struct drm_buddy_block *
19xe_ttm_vram_mgr_first_block(struct list_head *list)
20{
21 return list_first_entry_or_null(list, struct drm_buddy_block, link);
22}
23
24static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
25 struct list_head *head)
26{
27 struct drm_buddy_block *block;
28 u64 start, size;
29
30 block = xe_ttm_vram_mgr_first_block(head);
31 if (!block)
32 return false;
33
34 while (head != block->link.next) {
35 start = drm_buddy_block_offset(block);
36 size = drm_buddy_block_size(mm, block);
37
38 block = list_entry(block->link.next, struct drm_buddy_block,
39 link);
40 if (start + size != drm_buddy_block_offset(block))
41 return false;
42 }
43
44 return true;
45}
46
47static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
48 struct ttm_buffer_object *tbo,
49 const struct ttm_place *place,
50 struct ttm_resource **res)
51{
52 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
53 struct xe_ttm_vram_mgr_resource *vres;
54 struct drm_buddy *mm = &mgr->mm;
55 u64 size, remaining_size, min_page_size;
56 unsigned long lpfn;
57 int err;
58
59 lpfn = place->lpfn;
60 if (!lpfn || lpfn > man->size >> PAGE_SHIFT)
61 lpfn = man->size >> PAGE_SHIFT;
62
63 if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn))
64 return -E2BIG; /* don't trigger eviction for the impossible */
65
66 vres = kzalloc(sizeof(*vres), GFP_KERNEL);
67 if (!vres)
68 return -ENOMEM;
69
70 ttm_resource_init(tbo, place, &vres->base);
71
72 /* bail out quickly if there's likely not enough VRAM for this BO */
73 if (ttm_resource_manager_usage(man) > man->size) {
74 err = -ENOSPC;
75 goto error_fini;
76 }
77
78 INIT_LIST_HEAD(&vres->blocks);
79
80 if (place->flags & TTM_PL_FLAG_TOPDOWN)
81 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
82
83 if (place->fpfn || lpfn != man->size >> PAGE_SHIFT)
84 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
85
86 if (WARN_ON(!vres->base.size)) {
87 err = -EINVAL;
88 goto error_fini;
89 }
90 size = vres->base.size;
91
92 min_page_size = mgr->default_page_size;
93 if (tbo->page_alignment)
94 min_page_size = tbo->page_alignment << PAGE_SHIFT;
95
96 if (WARN_ON(min_page_size < mm->chunk_size)) {
97 err = -EINVAL;
98 goto error_fini;
99 }
100
101 if (WARN_ON(min_page_size > SZ_2G)) { /* FIXME: sg limit */
102 err = -EINVAL;
103 goto error_fini;
104 }
105
106 if (WARN_ON((size > SZ_2G &&
107 (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS)))) {
108 err = -EINVAL;
109 goto error_fini;
110 }
111
112 if (WARN_ON(!IS_ALIGNED(size, min_page_size))) {
113 err = -EINVAL;
114 goto error_fini;
115 }
116
117 mutex_lock(&mgr->lock);
118 if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) {
119 mutex_unlock(&mgr->lock);
120 err = -ENOSPC;
121 goto error_fini;
122 }
123
124 if (place->fpfn + (size >> PAGE_SHIFT) != place->lpfn &&
125 place->flags & TTM_PL_FLAG_CONTIGUOUS) {
126 size = roundup_pow_of_two(size);
127 min_page_size = size;
128
129 lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn);
130 }
131
132 remaining_size = size;
133 do {
134 /*
135 * Limit maximum size to 2GiB due to SG table limitations.
136 * FIXME: Should maybe be handled as part of sg construction.
137 */
138 u64 alloc_size = min_t(u64, remaining_size, SZ_2G);
139
140 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
141 (u64)lpfn << PAGE_SHIFT,
142 alloc_size,
143 min_page_size,
144 &vres->blocks,
145 vres->flags);
146 if (err)
147 goto error_free_blocks;
148
149 remaining_size -= alloc_size;
150 } while (remaining_size);
151
152 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
153 if (!drm_buddy_block_trim(mm, vres->base.size, &vres->blocks))
154 size = vres->base.size;
155 }
156
157 if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {
158 vres->used_visible_size = size;
159 } else {
160 struct drm_buddy_block *block;
161
162 list_for_each_entry(block, &vres->blocks, link) {
163 u64 start = drm_buddy_block_offset(block);
164
165 if (start < mgr->visible_size) {
166 u64 end = start + drm_buddy_block_size(mm, block);
167
168 vres->used_visible_size +=
169 min(end, mgr->visible_size) - start;
170 }
171 }
172 }
173
174 mgr->visible_avail -= vres->used_visible_size;
175 mutex_unlock(&mgr->lock);
176
177 if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) &&
178 xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks))
179 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
180
181 /*
182 * For some kernel objects we still rely on the start when io mapping
183 * the object.
184 */
185 if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) {
186 struct drm_buddy_block *block = list_first_entry(&vres->blocks,
187 typeof(*block),
188 link);
189
190 vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT;
191 } else {
192 vres->base.start = XE_BO_INVALID_OFFSET;
193 }
194
195 *res = &vres->base;
196 return 0;
197
198error_free_blocks:
199 drm_buddy_free_list(mm, &vres->blocks);
200 mutex_unlock(&mgr->lock);
201error_fini:
202 ttm_resource_fini(man, &vres->base);
203 kfree(vres);
204
205 return err;
206}
207
208static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
209 struct ttm_resource *res)
210{
211 struct xe_ttm_vram_mgr_resource *vres =
212 to_xe_ttm_vram_mgr_resource(res);
213 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
214 struct drm_buddy *mm = &mgr->mm;
215
216 mutex_lock(&mgr->lock);
217 drm_buddy_free_list(mm, &vres->blocks);
218 mgr->visible_avail += vres->used_visible_size;
219 mutex_unlock(&mgr->lock);
220
221 ttm_resource_fini(man, res);
222
223 kfree(vres);
224}
225
226static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,
227 struct drm_printer *printer)
228{
229 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
230 struct drm_buddy *mm = &mgr->mm;
231
232 mutex_lock(&mgr->lock);
233 drm_printf(printer, "default_page_size: %lluKiB\n",
234 mgr->default_page_size >> 10);
235 drm_printf(printer, "visible_avail: %lluMiB\n",
236 (u64)mgr->visible_avail >> 20);
237 drm_printf(printer, "visible_size: %lluMiB\n",
238 (u64)mgr->visible_size >> 20);
239
240 drm_buddy_print(mm, printer);
241 mutex_unlock(&mgr->lock);
242 drm_printf(printer, "man size:%llu\n", man->size);
243}
244
245static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man,
246 struct ttm_resource *res,
247 const struct ttm_place *place,
248 size_t size)
249{
250 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
251 struct xe_ttm_vram_mgr_resource *vres =
252 to_xe_ttm_vram_mgr_resource(res);
253 struct drm_buddy *mm = &mgr->mm;
254 struct drm_buddy_block *block;
255
256 if (!place->fpfn && !place->lpfn)
257 return true;
258
259 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT)
260 return vres->used_visible_size > 0;
261
262 list_for_each_entry(block, &vres->blocks, link) {
263 unsigned long fpfn =
264 drm_buddy_block_offset(block) >> PAGE_SHIFT;
265 unsigned long lpfn = fpfn +
266 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
267
268 if (place->fpfn < lpfn && place->lpfn > fpfn)
269 return true;
270 }
271
272 return false;
273}
274
275static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man,
276 struct ttm_resource *res,
277 const struct ttm_place *place,
278 size_t size)
279{
280 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
281 struct xe_ttm_vram_mgr_resource *vres =
282 to_xe_ttm_vram_mgr_resource(res);
283 struct drm_buddy *mm = &mgr->mm;
284 struct drm_buddy_block *block;
285
286 if (!place->fpfn && !place->lpfn)
287 return true;
288
289 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT)
290 return vres->used_visible_size == size;
291
292 list_for_each_entry(block, &vres->blocks, link) {
293 unsigned long fpfn =
294 drm_buddy_block_offset(block) >> PAGE_SHIFT;
295 unsigned long lpfn = fpfn +
296 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
297
298 if (fpfn < place->fpfn || lpfn > place->lpfn)
299 return false;
300 }
301
302 return true;
303}
304
305static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = {
306 .alloc = xe_ttm_vram_mgr_new,
307 .free = xe_ttm_vram_mgr_del,
308 .intersects = xe_ttm_vram_mgr_intersects,
309 .compatible = xe_ttm_vram_mgr_compatible,
310 .debug = xe_ttm_vram_mgr_debug
311};
312
313static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
314{
315 struct xe_device *xe = to_xe_device(dev);
316 struct xe_ttm_vram_mgr *mgr = arg;
317 struct ttm_resource_manager *man = &mgr->manager;
318
319 ttm_resource_manager_set_used(man, false);
320
321 if (ttm_resource_manager_evict_all(&xe->ttm, man))
322 return;
323
324 WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size);
325
326 drm_buddy_fini(&mgr->mm);
327
328 ttm_resource_manager_cleanup(&mgr->manager);
329
330 ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL);
331
332 mutex_destroy(&mgr->lock);
333}
334
335int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
336 u32 mem_type, u64 size, u64 io_size,
337 u64 default_page_size)
338{
339 struct ttm_resource_manager *man = &mgr->manager;
340 int err;
341
342 man->func = &xe_ttm_vram_mgr_func;
343 mgr->mem_type = mem_type;
344 mutex_init(&mgr->lock);
345 mgr->default_page_size = default_page_size;
346 mgr->visible_size = io_size;
347 mgr->visible_avail = io_size;
348
349 ttm_resource_manager_init(man, &xe->ttm, size);
350 err = drm_buddy_init(&mgr->mm, man->size, default_page_size);
351 if (err)
352 return err;
353
354 ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager);
355 ttm_resource_manager_set_used(&mgr->manager, true);
356
357 return drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr);
358}
359
360int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr)
361{
362 struct xe_device *xe = tile_to_xe(tile);
363 struct xe_mem_region *vram = &tile->mem.vram;
364
365 mgr->vram = vram;
366 return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + tile->id,
367 vram->usable_size, vram->io_size,
368 PAGE_SIZE);
369}
370
371int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
372 struct ttm_resource *res,
373 u64 offset, u64 length,
374 struct device *dev,
375 enum dma_data_direction dir,
376 struct sg_table **sgt)
377{
378 struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0];
379 struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res);
380 struct xe_res_cursor cursor;
381 struct scatterlist *sg;
382 int num_entries = 0;
383 int i, r;
384
385 if (vres->used_visible_size < res->size)
386 return -EOPNOTSUPP;
387
388 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
389 if (!*sgt)
390 return -ENOMEM;
391
392 /* Determine the number of DRM_BUDDY blocks to export */
393 xe_res_first(res, offset, length, &cursor);
394 while (cursor.remaining) {
395 num_entries++;
396 xe_res_next(&cursor, cursor.size);
397 }
398
399 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
400 if (r)
401 goto error_free;
402
403 /* Initialize scatterlist nodes of sg_table */
404 for_each_sgtable_sg((*sgt), sg, i)
405 sg->length = 0;
406
407 /*
408 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
409 * @note: Use iterator api to get first the DRM_BUDDY block
410 * and the number of bytes from it. Access the following
411 * DRM_BUDDY block(s) if more buffer needs to exported
412 */
413 xe_res_first(res, offset, length, &cursor);
414 for_each_sgtable_sg((*sgt), sg, i) {
415 phys_addr_t phys = cursor.start + tile->mem.vram.io_start;
416 size_t size = cursor.size;
417 dma_addr_t addr;
418
419 addr = dma_map_resource(dev, phys, size, dir,
420 DMA_ATTR_SKIP_CPU_SYNC);
421 r = dma_mapping_error(dev, addr);
422 if (r)
423 goto error_unmap;
424
425 sg_set_page(sg, NULL, size, 0);
426 sg_dma_address(sg) = addr;
427 sg_dma_len(sg) = size;
428
429 xe_res_next(&cursor, cursor.size);
430 }
431
432 return 0;
433
434error_unmap:
435 for_each_sgtable_sg((*sgt), sg, i) {
436 if (!sg->length)
437 continue;
438
439 dma_unmap_resource(dev, sg->dma_address,
440 sg->length, dir,
441 DMA_ATTR_SKIP_CPU_SYNC);
442 }
443 sg_free_table(*sgt);
444
445error_free:
446 kfree(*sgt);
447 return r;
448}
449
450void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,
451 struct sg_table *sgt)
452{
453 struct scatterlist *sg;
454 int i;
455
456 for_each_sgtable_sg(sgt, sg, i)
457 dma_unmap_resource(dev, sg->dma_address,
458 sg->length, dir,
459 DMA_ATTR_SKIP_CPU_SYNC);
460 sg_free_table(sgt);
461 kfree(sgt);
462}
463
464u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man)
465{
466 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
467
468 return mgr->visible_size;
469}
470
471void xe_ttm_vram_get_used(struct ttm_resource_manager *man,
472 u64 *used, u64 *used_visible)
473{
474 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
475
476 mutex_lock(&mgr->lock);
477 *used = mgr->mm.size - mgr->mm.avail;
478 *used_visible = mgr->visible_size - mgr->visible_avail;
479 mutex_unlock(&mgr->lock);
480}