Loading...
1/*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#ifndef _TTM_RESOURCE_H_
26#define _TTM_RESOURCE_H_
27
28#include <linux/types.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/iosys-map.h>
32#include <linux/dma-fence.h>
33
34#include <drm/drm_print.h>
35#include <drm/ttm/ttm_caching.h>
36#include <drm/ttm/ttm_kmap_iter.h>
37
38#define TTM_MAX_BO_PRIORITY 4U
39#define TTM_NUM_MEM_TYPES 8
40
41struct ttm_device;
42struct ttm_resource_manager;
43struct ttm_resource;
44struct ttm_place;
45struct ttm_buffer_object;
46struct ttm_placement;
47struct iosys_map;
48struct io_mapping;
49struct sg_table;
50struct scatterlist;
51
52struct ttm_resource_manager_func {
53 /**
54 * struct ttm_resource_manager_func member alloc
55 *
56 * @man: Pointer to a memory type manager.
57 * @bo: Pointer to the buffer object we're allocating space for.
58 * @place: Placement details.
59 * @res: Resulting pointer to the ttm_resource.
60 *
61 * This function should allocate space in the memory type managed
62 * by @man. Placement details if applicable are given by @place. If
63 * successful, a filled in ttm_resource object should be returned in
64 * @res. @res::start should be set to a value identifying the beginning
65 * of the range allocated, and the function should return zero.
66 * If the manager can't fulfill the request -ENOSPC should be returned.
67 * If a system error occurred, preventing the request to be fulfilled,
68 * the function should return a negative error code.
69 *
70 * This function may not be called from within atomic context and needs
71 * to take care of its own locking to protect any data structures
72 * managing the space.
73 */
74 int (*alloc)(struct ttm_resource_manager *man,
75 struct ttm_buffer_object *bo,
76 const struct ttm_place *place,
77 struct ttm_resource **res);
78
79 /**
80 * struct ttm_resource_manager_func member free
81 *
82 * @man: Pointer to a memory type manager.
83 * @res: Pointer to a struct ttm_resource to be freed.
84 *
85 * This function frees memory type resources previously allocated.
86 * May not be called from within atomic context.
87 */
88 void (*free)(struct ttm_resource_manager *man,
89 struct ttm_resource *res);
90
91 /**
92 * struct ttm_resource_manager_func member intersects
93 *
94 * @man: Pointer to a memory type manager.
95 * @res: Pointer to a struct ttm_resource to be checked.
96 * @place: Placement to check against.
97 * @size: Size of the check.
98 *
99 * Test if @res intersects with @place + @size. Used to judge if
100 * evictions are valueable or not.
101 */
102 bool (*intersects)(struct ttm_resource_manager *man,
103 struct ttm_resource *res,
104 const struct ttm_place *place,
105 size_t size);
106
107 /**
108 * struct ttm_resource_manager_func member compatible
109 *
110 * @man: Pointer to a memory type manager.
111 * @res: Pointer to a struct ttm_resource to be checked.
112 * @place: Placement to check against.
113 * @size: Size of the check.
114 *
115 * Test if @res compatible with @place + @size. Used to check of
116 * the need to move the backing store or not.
117 */
118 bool (*compatible)(struct ttm_resource_manager *man,
119 struct ttm_resource *res,
120 const struct ttm_place *place,
121 size_t size);
122
123 /**
124 * struct ttm_resource_manager_func member debug
125 *
126 * @man: Pointer to a memory type manager.
127 * @printer: Prefix to be used in printout to identify the caller.
128 *
129 * This function is called to print out the state of the memory
130 * type manager to aid debugging of out-of-memory conditions.
131 * It may not be called from within atomic context.
132 */
133 void (*debug)(struct ttm_resource_manager *man,
134 struct drm_printer *printer);
135};
136
137/**
138 * struct ttm_resource_manager
139 *
140 * @use_type: The memory type is enabled.
141 * @use_tt: If a TT object should be used for the backing store.
142 * @size: Size of the managed region.
143 * @bdev: ttm device this manager belongs to
144 * @func: structure pointer implementing the range manager. See above
145 * @move_lock: lock for move fence
146 * @move: The fence of the last pipelined move operation.
147 * @lru: The lru list for this memory type.
148 *
149 * This structure is used to identify and manage memory types for a device.
150 */
151struct ttm_resource_manager {
152 /*
153 * No protection. Constant from start.
154 */
155 bool use_type;
156 bool use_tt;
157 struct ttm_device *bdev;
158 uint64_t size;
159 const struct ttm_resource_manager_func *func;
160 spinlock_t move_lock;
161
162 /*
163 * Protected by @move_lock.
164 */
165 struct dma_fence *move;
166
167 /*
168 * Protected by the bdev->lru_lock.
169 */
170 struct list_head lru[TTM_MAX_BO_PRIORITY];
171
172 /**
173 * @usage: How much of the resources are used, protected by the
174 * bdev->lru_lock.
175 */
176 uint64_t usage;
177};
178
179/**
180 * struct ttm_bus_placement
181 *
182 * @addr: mapped virtual address
183 * @offset: physical addr
184 * @is_iomem: is this io memory ?
185 * @caching: See enum ttm_caching
186 *
187 * Structure indicating the bus placement of an object.
188 */
189struct ttm_bus_placement {
190 void *addr;
191 phys_addr_t offset;
192 bool is_iomem;
193 enum ttm_caching caching;
194};
195
196/**
197 * struct ttm_resource
198 *
199 * @start: Start of the allocation.
200 * @size: Actual size of resource in bytes.
201 * @mem_type: Resource type of the allocation.
202 * @placement: Placement flags.
203 * @bus: Placement on io bus accessible to the CPU
204 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
205 *
206 * Structure indicating the placement and space resources used by a
207 * buffer object.
208 */
209struct ttm_resource {
210 unsigned long start;
211 size_t size;
212 uint32_t mem_type;
213 uint32_t placement;
214 struct ttm_bus_placement bus;
215 struct ttm_buffer_object *bo;
216
217 /**
218 * @lru: Least recently used list, see &ttm_resource_manager.lru
219 */
220 struct list_head lru;
221};
222
223/**
224 * struct ttm_resource_cursor
225 *
226 * @priority: the current priority
227 *
228 * Cursor to iterate over the resources in a manager.
229 */
230struct ttm_resource_cursor {
231 unsigned int priority;
232};
233
234/**
235 * struct ttm_lru_bulk_move_pos
236 *
237 * @first: first res in the bulk move range
238 * @last: last res in the bulk move range
239 *
240 * Range of resources for a lru bulk move.
241 */
242struct ttm_lru_bulk_move_pos {
243 struct ttm_resource *first;
244 struct ttm_resource *last;
245};
246
247/**
248 * struct ttm_lru_bulk_move
249 *
250 * @pos: first/last lru entry for resources in the each domain/priority
251 *
252 * Container for the current bulk move state. Should be used with
253 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
254 */
255struct ttm_lru_bulk_move {
256 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
257};
258
259/**
260 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
261 * struct sg_table backed struct ttm_resource.
262 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
263 * @iomap: struct io_mapping representing the underlying linear io_memory.
264 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
265 * @start: Offset that needs to be subtracted from @st to make
266 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
267 * @cache: Scatterlist traversal cache for fast lookups.
268 * @cache.sg: Pointer to the currently cached scatterlist segment.
269 * @cache.i: First index of @sg. PAGE_SIZE granularity.
270 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
271 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
272 */
273struct ttm_kmap_iter_iomap {
274 struct ttm_kmap_iter base;
275 struct io_mapping *iomap;
276 struct sg_table *st;
277 resource_size_t start;
278 struct {
279 struct scatterlist *sg;
280 pgoff_t i;
281 pgoff_t end;
282 pgoff_t offs;
283 } cache;
284};
285
286/**
287 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
288 * @base: The base iterator
289 * @dmap: Points to the starting address of the region
290 * @needs_unmap: Whether we need to unmap on fini
291 */
292struct ttm_kmap_iter_linear_io {
293 struct ttm_kmap_iter base;
294 struct iosys_map dmap;
295 bool needs_unmap;
296};
297
298/**
299 * ttm_resource_manager_set_used
300 *
301 * @man: A memory manager object.
302 * @used: usage state to set.
303 *
304 * Set the manager in use flag. If disabled the manager is no longer
305 * used for object placement.
306 */
307static inline void
308ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
309{
310 int i;
311
312 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
313 WARN_ON(!list_empty(&man->lru[i]));
314 man->use_type = used;
315}
316
317/**
318 * ttm_resource_manager_used
319 *
320 * @man: Manager to get used state for
321 *
322 * Get the in use flag for a manager.
323 * Returns:
324 * true is used, false if not.
325 */
326static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
327{
328 return man->use_type;
329}
330
331/**
332 * ttm_resource_manager_cleanup
333 *
334 * @man: A memory manager object.
335 *
336 * Cleanup the move fences from the memory manager object.
337 */
338static inline void
339ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
340{
341 dma_fence_put(man->move);
342 man->move = NULL;
343}
344
345void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
346void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
347
348void ttm_resource_add_bulk_move(struct ttm_resource *res,
349 struct ttm_buffer_object *bo);
350void ttm_resource_del_bulk_move(struct ttm_resource *res,
351 struct ttm_buffer_object *bo);
352void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
353
354void ttm_resource_init(struct ttm_buffer_object *bo,
355 const struct ttm_place *place,
356 struct ttm_resource *res);
357void ttm_resource_fini(struct ttm_resource_manager *man,
358 struct ttm_resource *res);
359
360int ttm_resource_alloc(struct ttm_buffer_object *bo,
361 const struct ttm_place *place,
362 struct ttm_resource **res);
363void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
364bool ttm_resource_intersects(struct ttm_device *bdev,
365 struct ttm_resource *res,
366 const struct ttm_place *place,
367 size_t size);
368bool ttm_resource_compatible(struct ttm_device *bdev,
369 struct ttm_resource *res,
370 const struct ttm_place *place,
371 size_t size);
372bool ttm_resource_compat(struct ttm_resource *res,
373 struct ttm_placement *placement);
374void ttm_resource_set_bo(struct ttm_resource *res,
375 struct ttm_buffer_object *bo);
376
377void ttm_resource_manager_init(struct ttm_resource_manager *man,
378 struct ttm_device *bdev,
379 uint64_t size);
380
381int ttm_resource_manager_evict_all(struct ttm_device *bdev,
382 struct ttm_resource_manager *man);
383
384uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
385void ttm_resource_manager_debug(struct ttm_resource_manager *man,
386 struct drm_printer *p);
387
388struct ttm_resource *
389ttm_resource_manager_first(struct ttm_resource_manager *man,
390 struct ttm_resource_cursor *cursor);
391struct ttm_resource *
392ttm_resource_manager_next(struct ttm_resource_manager *man,
393 struct ttm_resource_cursor *cursor,
394 struct ttm_resource *res);
395
396/**
397 * ttm_resource_manager_for_each_res - iterate over all resources
398 * @man: the resource manager
399 * @cursor: struct ttm_resource_cursor for the current position
400 * @res: the current resource
401 *
402 * Iterate over all the evictable resources in a resource manager.
403 */
404#define ttm_resource_manager_for_each_res(man, cursor, res) \
405 for (res = ttm_resource_manager_first(man, cursor); res; \
406 res = ttm_resource_manager_next(man, cursor, res))
407
408struct ttm_kmap_iter *
409ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
410 struct io_mapping *iomap,
411 struct sg_table *st,
412 resource_size_t start);
413
414struct ttm_kmap_iter_linear_io;
415
416struct ttm_kmap_iter *
417ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
418 struct ttm_device *bdev,
419 struct ttm_resource *mem);
420
421void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
422 struct ttm_device *bdev,
423 struct ttm_resource *mem);
424
425void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
426 struct dentry * parent,
427 const char *name);
428#endif
1/*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#ifndef _TTM_RESOURCE_H_
26#define _TTM_RESOURCE_H_
27
28#include <linux/types.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/iosys-map.h>
32#include <linux/dma-fence.h>
33
34#include <drm/drm_print.h>
35#include <drm/ttm/ttm_caching.h>
36#include <drm/ttm/ttm_kmap_iter.h>
37
38#define TTM_MAX_BO_PRIORITY 4U
39#define TTM_NUM_MEM_TYPES 8
40
41struct ttm_device;
42struct ttm_resource_manager;
43struct ttm_resource;
44struct ttm_place;
45struct ttm_buffer_object;
46struct ttm_placement;
47struct iosys_map;
48struct io_mapping;
49struct sg_table;
50struct scatterlist;
51
52/**
53 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses
54 */
55enum ttm_lru_item_type {
56 /** @TTM_LRU_RESOURCE: The resource subclass */
57 TTM_LRU_RESOURCE,
58 /** @TTM_LRU_HITCH: The iterator hitch subclass */
59 TTM_LRU_HITCH
60};
61
62/**
63 * struct ttm_lru_item - The TTM lru list node base class
64 * @link: The list link
65 * @type: The subclass type
66 */
67struct ttm_lru_item {
68 struct list_head link;
69 enum ttm_lru_item_type type;
70};
71
72/**
73 * ttm_lru_item_init() - initialize a struct ttm_lru_item
74 * @item: The item to initialize
75 * @type: The subclass type
76 */
77static inline void ttm_lru_item_init(struct ttm_lru_item *item,
78 enum ttm_lru_item_type type)
79{
80 item->type = type;
81 INIT_LIST_HEAD(&item->link);
82}
83
84static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item)
85{
86 return item->type == TTM_LRU_RESOURCE;
87}
88
89struct ttm_resource_manager_func {
90 /**
91 * struct ttm_resource_manager_func member alloc
92 *
93 * @man: Pointer to a memory type manager.
94 * @bo: Pointer to the buffer object we're allocating space for.
95 * @place: Placement details.
96 * @res: Resulting pointer to the ttm_resource.
97 *
98 * This function should allocate space in the memory type managed
99 * by @man. Placement details if applicable are given by @place. If
100 * successful, a filled in ttm_resource object should be returned in
101 * @res. @res::start should be set to a value identifying the beginning
102 * of the range allocated, and the function should return zero.
103 * If the manager can't fulfill the request -ENOSPC should be returned.
104 * If a system error occurred, preventing the request to be fulfilled,
105 * the function should return a negative error code.
106 *
107 * This function may not be called from within atomic context and needs
108 * to take care of its own locking to protect any data structures
109 * managing the space.
110 */
111 int (*alloc)(struct ttm_resource_manager *man,
112 struct ttm_buffer_object *bo,
113 const struct ttm_place *place,
114 struct ttm_resource **res);
115
116 /**
117 * struct ttm_resource_manager_func member free
118 *
119 * @man: Pointer to a memory type manager.
120 * @res: Pointer to a struct ttm_resource to be freed.
121 *
122 * This function frees memory type resources previously allocated.
123 * May not be called from within atomic context.
124 */
125 void (*free)(struct ttm_resource_manager *man,
126 struct ttm_resource *res);
127
128 /**
129 * struct ttm_resource_manager_func member intersects
130 *
131 * @man: Pointer to a memory type manager.
132 * @res: Pointer to a struct ttm_resource to be checked.
133 * @place: Placement to check against.
134 * @size: Size of the check.
135 *
136 * Test if @res intersects with @place + @size. Used to judge if
137 * evictions are valueable or not.
138 */
139 bool (*intersects)(struct ttm_resource_manager *man,
140 struct ttm_resource *res,
141 const struct ttm_place *place,
142 size_t size);
143
144 /**
145 * struct ttm_resource_manager_func member compatible
146 *
147 * @man: Pointer to a memory type manager.
148 * @res: Pointer to a struct ttm_resource to be checked.
149 * @place: Placement to check against.
150 * @size: Size of the check.
151 *
152 * Test if @res compatible with @place + @size. Used to check of
153 * the need to move the backing store or not.
154 */
155 bool (*compatible)(struct ttm_resource_manager *man,
156 struct ttm_resource *res,
157 const struct ttm_place *place,
158 size_t size);
159
160 /**
161 * struct ttm_resource_manager_func member debug
162 *
163 * @man: Pointer to a memory type manager.
164 * @printer: Prefix to be used in printout to identify the caller.
165 *
166 * This function is called to print out the state of the memory
167 * type manager to aid debugging of out-of-memory conditions.
168 * It may not be called from within atomic context.
169 */
170 void (*debug)(struct ttm_resource_manager *man,
171 struct drm_printer *printer);
172};
173
174/**
175 * struct ttm_resource_manager
176 *
177 * @use_type: The memory type is enabled.
178 * @use_tt: If a TT object should be used for the backing store.
179 * @size: Size of the managed region.
180 * @bdev: ttm device this manager belongs to
181 * @func: structure pointer implementing the range manager. See above
182 * @move_lock: lock for move fence
183 * @move: The fence of the last pipelined move operation.
184 * @lru: The lru list for this memory type.
185 *
186 * This structure is used to identify and manage memory types for a device.
187 */
188struct ttm_resource_manager {
189 /*
190 * No protection. Constant from start.
191 */
192 bool use_type;
193 bool use_tt;
194 struct ttm_device *bdev;
195 uint64_t size;
196 const struct ttm_resource_manager_func *func;
197 spinlock_t move_lock;
198
199 /*
200 * Protected by @move_lock.
201 */
202 struct dma_fence *move;
203
204 /*
205 * Protected by the bdev->lru_lock.
206 */
207 struct list_head lru[TTM_MAX_BO_PRIORITY];
208
209 /**
210 * @usage: How much of the resources are used, protected by the
211 * bdev->lru_lock.
212 */
213 uint64_t usage;
214};
215
216/**
217 * struct ttm_bus_placement
218 *
219 * @addr: mapped virtual address
220 * @offset: physical addr
221 * @is_iomem: is this io memory ?
222 * @caching: See enum ttm_caching
223 *
224 * Structure indicating the bus placement of an object.
225 */
226struct ttm_bus_placement {
227 void *addr;
228 phys_addr_t offset;
229 bool is_iomem;
230 enum ttm_caching caching;
231};
232
233/**
234 * struct ttm_resource
235 *
236 * @start: Start of the allocation.
237 * @size: Actual size of resource in bytes.
238 * @mem_type: Resource type of the allocation.
239 * @placement: Placement flags.
240 * @bus: Placement on io bus accessible to the CPU
241 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
242 *
243 * Structure indicating the placement and space resources used by a
244 * buffer object.
245 */
246struct ttm_resource {
247 unsigned long start;
248 size_t size;
249 uint32_t mem_type;
250 uint32_t placement;
251 struct ttm_bus_placement bus;
252 struct ttm_buffer_object *bo;
253
254 /**
255 * @lru: Least recently used list, see &ttm_resource_manager.lru
256 */
257 struct ttm_lru_item lru;
258};
259
260/**
261 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource
262 * @item: The struct ttm_lru_item to downcast
263 *
264 * Return: Pointer to the embedding struct ttm_resource
265 */
266static inline struct ttm_resource *
267ttm_lru_item_to_res(struct ttm_lru_item *item)
268{
269 return container_of(item, struct ttm_resource, lru);
270}
271
272/**
273 * struct ttm_lru_bulk_move_pos
274 *
275 * @first: first res in the bulk move range
276 * @last: last res in the bulk move range
277 *
278 * Range of resources for a lru bulk move.
279 */
280struct ttm_lru_bulk_move_pos {
281 struct ttm_resource *first;
282 struct ttm_resource *last;
283};
284
285/**
286 * struct ttm_lru_bulk_move
287 * @pos: first/last lru entry for resources in the each domain/priority
288 * @cursor_list: The list of cursors currently traversing any of
289 * the sublists of @pos. Protected by the ttm device's lru_lock.
290 *
291 * Container for the current bulk move state. Should be used with
292 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
293 * All BOs in a bulk_move structure need to share the same reservation object to
294 * ensure that the bulk as a whole is locked for eviction even if only one BO of
295 * the bulk is evicted.
296 */
297struct ttm_lru_bulk_move {
298 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
299 struct list_head cursor_list;
300};
301
302/**
303 * struct ttm_resource_cursor
304 * @man: The resource manager currently being iterated over
305 * @hitch: A hitch list node inserted before the next resource
306 * to iterate over.
307 * @bulk_link: A list link for the list of cursors traversing the
308 * bulk sublist of @bulk. Protected by the ttm device's lru_lock.
309 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is
310 * inserted to. NULL if none. Never dereference this pointer since
311 * the struct ttm_lru_bulk_move object pointed to might have been
312 * freed. The pointer is only for comparison.
313 * @mem_type: The memory type of the LRU list being traversed.
314 * This field is valid iff @bulk != NULL.
315 * @priority: the current priority
316 *
317 * Cursor to iterate over the resources in a manager.
318 */
319struct ttm_resource_cursor {
320 struct ttm_resource_manager *man;
321 struct ttm_lru_item hitch;
322 struct list_head bulk_link;
323 struct ttm_lru_bulk_move *bulk;
324 unsigned int mem_type;
325 unsigned int priority;
326};
327
328void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);
329
330/**
331 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
332 * struct sg_table backed struct ttm_resource.
333 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
334 * @iomap: struct io_mapping representing the underlying linear io_memory.
335 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
336 * @start: Offset that needs to be subtracted from @st to make
337 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
338 * @cache: Scatterlist traversal cache for fast lookups.
339 * @cache.sg: Pointer to the currently cached scatterlist segment.
340 * @cache.i: First index of @sg. PAGE_SIZE granularity.
341 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
342 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
343 */
344struct ttm_kmap_iter_iomap {
345 struct ttm_kmap_iter base;
346 struct io_mapping *iomap;
347 struct sg_table *st;
348 resource_size_t start;
349 struct {
350 struct scatterlist *sg;
351 pgoff_t i;
352 pgoff_t end;
353 pgoff_t offs;
354 } cache;
355};
356
357/**
358 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
359 * @base: The base iterator
360 * @dmap: Points to the starting address of the region
361 * @needs_unmap: Whether we need to unmap on fini
362 */
363struct ttm_kmap_iter_linear_io {
364 struct ttm_kmap_iter base;
365 struct iosys_map dmap;
366 bool needs_unmap;
367};
368
369/**
370 * ttm_resource_manager_set_used
371 *
372 * @man: A memory manager object.
373 * @used: usage state to set.
374 *
375 * Set the manager in use flag. If disabled the manager is no longer
376 * used for object placement.
377 */
378static inline void
379ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
380{
381 int i;
382
383 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
384 WARN_ON(!list_empty(&man->lru[i]));
385 man->use_type = used;
386}
387
388/**
389 * ttm_resource_manager_used
390 *
391 * @man: Manager to get used state for
392 *
393 * Get the in use flag for a manager.
394 * Returns:
395 * true is used, false if not.
396 */
397static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
398{
399 return man->use_type;
400}
401
402/**
403 * ttm_resource_manager_cleanup
404 *
405 * @man: A memory manager object.
406 *
407 * Cleanup the move fences from the memory manager object.
408 */
409static inline void
410ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
411{
412 dma_fence_put(man->move);
413 man->move = NULL;
414}
415
416void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
417void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
418void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
419 struct ttm_lru_bulk_move *bulk);
420
421void ttm_resource_add_bulk_move(struct ttm_resource *res,
422 struct ttm_buffer_object *bo);
423void ttm_resource_del_bulk_move(struct ttm_resource *res,
424 struct ttm_buffer_object *bo);
425void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
426
427void ttm_resource_init(struct ttm_buffer_object *bo,
428 const struct ttm_place *place,
429 struct ttm_resource *res);
430void ttm_resource_fini(struct ttm_resource_manager *man,
431 struct ttm_resource *res);
432
433int ttm_resource_alloc(struct ttm_buffer_object *bo,
434 const struct ttm_place *place,
435 struct ttm_resource **res);
436void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
437bool ttm_resource_intersects(struct ttm_device *bdev,
438 struct ttm_resource *res,
439 const struct ttm_place *place,
440 size_t size);
441bool ttm_resource_compatible(struct ttm_resource *res,
442 struct ttm_placement *placement,
443 bool evicting);
444void ttm_resource_set_bo(struct ttm_resource *res,
445 struct ttm_buffer_object *bo);
446
447void ttm_resource_manager_init(struct ttm_resource_manager *man,
448 struct ttm_device *bdev,
449 uint64_t size);
450
451int ttm_resource_manager_evict_all(struct ttm_device *bdev,
452 struct ttm_resource_manager *man);
453
454uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
455void ttm_resource_manager_debug(struct ttm_resource_manager *man,
456 struct drm_printer *p);
457
458struct ttm_resource *
459ttm_resource_manager_first(struct ttm_resource_manager *man,
460 struct ttm_resource_cursor *cursor);
461struct ttm_resource *
462ttm_resource_manager_next(struct ttm_resource_cursor *cursor);
463
464struct ttm_resource *
465ttm_lru_first_res_or_null(struct list_head *head);
466
467/**
468 * ttm_resource_manager_for_each_res - iterate over all resources
469 * @man: the resource manager
470 * @cursor: struct ttm_resource_cursor for the current position
471 * @res: the current resource
472 *
473 * Iterate over all the evictable resources in a resource manager.
474 */
475#define ttm_resource_manager_for_each_res(man, cursor, res) \
476 for (res = ttm_resource_manager_first(man, cursor); res; \
477 res = ttm_resource_manager_next(cursor))
478
479struct ttm_kmap_iter *
480ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
481 struct io_mapping *iomap,
482 struct sg_table *st,
483 resource_size_t start);
484
485struct ttm_kmap_iter_linear_io;
486
487struct ttm_kmap_iter *
488ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
489 struct ttm_device *bdev,
490 struct ttm_resource *mem);
491
492void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
493 struct ttm_device *bdev,
494 struct ttm_resource *mem);
495
496void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
497 struct dentry * parent,
498 const char *name);
499#endif