1/*
  2 * Copyright 2020 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 * Authors: Christian König
 23 */
 24
 25#include <linux/debugfs.h>
 26#include <linux/io-mapping.h>
 27#include <linux/iosys-map.h>
 
 28#include <linux/scatterlist.h>
 29
 30#include <drm/ttm/ttm_bo.h>
 31#include <drm/ttm/ttm_placement.h>
 32#include <drm/ttm/ttm_resource.h>
 33#include <drm/ttm/ttm_tt.h>
 34
 35#include <drm/drm_util.h>
 36
 37/* Detach the cursor from the bulk move list*/
 38static void
 39ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
 40{
 41	lockdep_assert_held(&cursor->man->bdev->lru_lock);
 42
 43	cursor->bulk = NULL;
 44	list_del_init(&cursor->bulk_link);
 45}
 46
 47/* Move the cursor to the end of the bulk move list it's in */
 48static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk,
 49					       struct ttm_resource_cursor *cursor)
 50{
 51	struct ttm_lru_bulk_move_pos *pos;
 52
 53	lockdep_assert_held(&cursor->man->bdev->lru_lock);
 54
 55	if (WARN_ON_ONCE(bulk != cursor->bulk)) {
 56		list_del_init(&cursor->bulk_link);
 57		return;
 58	}
 59
 60	pos = &bulk->pos[cursor->mem_type][cursor->priority];
 61	if (pos->last)
 62		list_move(&cursor->hitch.link, &pos->last->lru.link);
 63	ttm_resource_cursor_clear_bulk(cursor);
 64}
 65
 66/* Move all cursors attached to a bulk move to its end */
 67static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk)
 68{
 69	struct ttm_resource_cursor *cursor, *next;
 70
 71	list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
 72		ttm_resource_cursor_move_bulk_tail(bulk, cursor);
 73}
 74
 75/* Remove a cursor from an empty bulk move list */
 76static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk)
 77{
 78	struct ttm_resource_cursor *cursor, *next;
 79
 80	list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
 81		ttm_resource_cursor_clear_bulk(cursor);
 82}
 83
 84/**
 85 * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
 86 * @cursor: The struct ttm_resource_cursor to finalize.
 87 *
 88 * The function pulls the LRU list cursor off any lists it was previusly
 89 * attached to. Needs to be called with the LRU lock held. The function
 90 * can be called multiple times after eachother.
 91 */
 92void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
 93{
 94	lockdep_assert_held(&cursor->man->bdev->lru_lock);
 95	list_del_init(&cursor->hitch.link);
 96	ttm_resource_cursor_clear_bulk(cursor);
 97}
 98
 99/**
100 * ttm_lru_bulk_move_init - initialize a bulk move structure
101 * @bulk: the structure to init
102 *
103 * For now just memset the structure to zero.
104 */
105void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
106{
107	memset(bulk, 0, sizeof(*bulk));
108	INIT_LIST_HEAD(&bulk->cursor_list);
109}
110EXPORT_SYMBOL(ttm_lru_bulk_move_init);
111
112/**
113 * ttm_lru_bulk_move_fini - finalize a bulk move structure
114 * @bdev: The struct ttm_device
115 * @bulk: the structure to finalize
116 *
117 * Sanity checks that bulk moves don't have any
118 * resources left and hence no cursors attached.
119 */
120void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
121			    struct ttm_lru_bulk_move *bulk)
122{
123	spin_lock(&bdev->lru_lock);
124	ttm_bulk_move_drop_cursors(bulk);
125	spin_unlock(&bdev->lru_lock);
126}
127EXPORT_SYMBOL(ttm_lru_bulk_move_fini);
128
129/**
130 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
131 *
132 * @bulk: bulk move structure
133 *
134 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
135 * resource order never changes. Should be called with &ttm_device.lru_lock held.
136 */
137void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
138{
139	unsigned i, j;
140
141	ttm_bulk_move_adjust_cursors(bulk);
142	for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
143		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
144			struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
145			struct ttm_resource_manager *man;
146
147			if (!pos->first)
148				continue;
149
150			lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
151			dma_resv_assert_held(pos->first->bo->base.resv);
152			dma_resv_assert_held(pos->last->bo->base.resv);
153
154			man = ttm_manager_type(pos->first->bo->bdev, i);
155			list_bulk_move_tail(&man->lru[j], &pos->first->lru.link,
156					    &pos->last->lru.link);
157		}
158	}
159}
160EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
161
162/* Return the bulk move pos object for this resource */
163static struct ttm_lru_bulk_move_pos *
164ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
165{
166	return &bulk->pos[res->mem_type][res->bo->priority];
167}
168
169/* Return the previous resource on the list (skip over non-resource list items) */
170static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)
171{
172	struct ttm_lru_item *lru = &cur->lru;
173
174	do {
175		lru = list_prev_entry(lru, link);
176	} while (!ttm_lru_item_is_res(lru));
177
178	return ttm_lru_item_to_res(lru);
179}
180
181/* Return the next resource on the list (skip over non-resource list items) */
182static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)
183{
184	struct ttm_lru_item *lru = &cur->lru;
185
186	do {
187		lru = list_next_entry(lru, link);
188	} while (!ttm_lru_item_is_res(lru));
189
190	return ttm_lru_item_to_res(lru);
191}
192
193/* Move the resource to the tail of the bulk move range */
194static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
195				       struct ttm_resource *res)
196{
197	if (pos->last != res) {
198		if (pos->first == res)
199			pos->first = ttm_lru_next_res(res);
200		list_move(&res->lru.link, &pos->last->lru.link);
201		pos->last = res;
202	}
203}
204
205/* Add the resource to a bulk_move cursor */
206static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
207				  struct ttm_resource *res)
208{
209	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
210
211	if (!pos->first) {
212		pos->first = res;
213		pos->last = res;
214	} else {
215		WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);
216		ttm_lru_bulk_move_pos_tail(pos, res);
217	}
218}
219
220/* Remove the resource from a bulk_move range */
221static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
222				  struct ttm_resource *res)
223{
224	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
225
226	if (unlikely(WARN_ON(!pos->first || !pos->last) ||
227		     (pos->first == res && pos->last == res))) {
228		pos->first = NULL;
229		pos->last = NULL;
230	} else if (pos->first == res) {
231		pos->first = ttm_lru_next_res(res);
232	} else if (pos->last == res) {
233		pos->last = ttm_lru_prev_res(res);
234	} else {
235		list_move(&res->lru.link, &pos->last->lru.link);
236	}
237}
238
239static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_object *bo)
240{
241	/*
242	 * Take care when creating a new resource for a bo, that it is not considered
243	 * swapped if it's not the current resource for the bo and is thus logically
244	 * associated with the ttm_tt. Think a VRAM resource created to move a
245	 * swapped-out bo to VRAM.
246	 */
247	if (bo->resource != res || !bo->ttm)
248		return false;
249
250	dma_resv_assert_held(bo->base.resv);
251	return ttm_tt_is_swapped(bo->ttm);
252}
253
254/* Add the resource to a bulk move if the BO is configured for it */
255void ttm_resource_add_bulk_move(struct ttm_resource *res,
256				struct ttm_buffer_object *bo)
257{
258	if (bo->bulk_move && !bo->pin_count && !ttm_resource_is_swapped(res, bo))
259		ttm_lru_bulk_move_add(bo->bulk_move, res);
260}
261
262/* Remove the resource from a bulk move if the BO is configured for it */
263void ttm_resource_del_bulk_move(struct ttm_resource *res,
264				struct ttm_buffer_object *bo)
265{
266	if (bo->bulk_move && !bo->pin_count && !ttm_resource_is_swapped(res, bo))
267		ttm_lru_bulk_move_del(bo->bulk_move, res);
268}
269
270/* Move a resource to the LRU or bulk tail */
271void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
272{
273	struct ttm_buffer_object *bo = res->bo;
274	struct ttm_device *bdev = bo->bdev;
275
276	lockdep_assert_held(&bo->bdev->lru_lock);
277
278	if (bo->pin_count || ttm_resource_is_swapped(res, bo)) {
279		list_move_tail(&res->lru.link, &bdev->unevictable);
280
281	} else	if (bo->bulk_move) {
282		struct ttm_lru_bulk_move_pos *pos =
283			ttm_lru_bulk_move_pos(bo->bulk_move, res);
284
285		ttm_lru_bulk_move_pos_tail(pos, res);
286	} else {
287		struct ttm_resource_manager *man;
288
289		man = ttm_manager_type(bdev, res->mem_type);
290		list_move_tail(&res->lru.link, &man->lru[bo->priority]);
291	}
292}
293
294/**
295 * ttm_resource_init - resource object constructure
296 * @bo: buffer object this resources is allocated for
297 * @place: placement of the resource
298 * @res: the resource object to inistilize
299 *
300 * Initialize a new resource object. Counterpart of ttm_resource_fini().
301 */
302void ttm_resource_init(struct ttm_buffer_object *bo,
303                       const struct ttm_place *place,
304                       struct ttm_resource *res)
305{
306	struct ttm_resource_manager *man;
307
308	res->start = 0;
309	res->size = bo->base.size;
310	res->mem_type = place->mem_type;
311	res->placement = place->flags;
312	res->bus.addr = NULL;
313	res->bus.offset = 0;
314	res->bus.is_iomem = false;
315	res->bus.caching = ttm_cached;
316	res->bo = bo;
317
318	man = ttm_manager_type(bo->bdev, place->mem_type);
319	spin_lock(&bo->bdev->lru_lock);
320	if (bo->pin_count || ttm_resource_is_swapped(res, bo))
321		list_add_tail(&res->lru.link, &bo->bdev->unevictable);
322	else
323		list_add_tail(&res->lru.link, &man->lru[bo->priority]);
324	man->usage += res->size;
325	spin_unlock(&bo->bdev->lru_lock);
326}
327EXPORT_SYMBOL(ttm_resource_init);
328
329/**
330 * ttm_resource_fini - resource destructor
331 * @man: the resource manager this resource belongs to
332 * @res: the resource to clean up
333 *
334 * Should be used by resource manager backends to clean up the TTM resource
335 * objects before freeing the underlying structure. Makes sure the resource is
336 * removed from the LRU before destruction.
337 * Counterpart of ttm_resource_init().
338 */
339void ttm_resource_fini(struct ttm_resource_manager *man,
340		       struct ttm_resource *res)
341{
342	struct ttm_device *bdev = man->bdev;
343
344	spin_lock(&bdev->lru_lock);
345	list_del_init(&res->lru.link);
346	man->usage -= res->size;
347	spin_unlock(&bdev->lru_lock);
348}
349EXPORT_SYMBOL(ttm_resource_fini);
350
351int ttm_resource_alloc(struct ttm_buffer_object *bo,
352		       const struct ttm_place *place,
353		       struct ttm_resource **res_ptr)
354{
355	struct ttm_resource_manager *man =
356		ttm_manager_type(bo->bdev, place->mem_type);
357	int ret;
358
359	ret = man->func->alloc(man, bo, place, res_ptr);
360	if (ret)
361		return ret;
362
363	spin_lock(&bo->bdev->lru_lock);
364	ttm_resource_add_bulk_move(*res_ptr, bo);
365	spin_unlock(&bo->bdev->lru_lock);
366	return 0;
367}
368EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);
369
370void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
371{
372	struct ttm_resource_manager *man;
373
374	if (!*res)
375		return;
376
377	spin_lock(&bo->bdev->lru_lock);
378	ttm_resource_del_bulk_move(*res, bo);
379	spin_unlock(&bo->bdev->lru_lock);
380	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
381	man->func->free(man, *res);
382	*res = NULL;
383}
384EXPORT_SYMBOL(ttm_resource_free);
385
386/**
387 * ttm_resource_intersects - test for intersection
388 *
389 * @bdev: TTM device structure
390 * @res: The resource to test
391 * @place: The placement to test
392 * @size: How many bytes the new allocation needs.
393 *
394 * Test if @res intersects with @place and @size. Used for testing if evictions
395 * are valueable or not.
396 *
397 * Returns true if the res placement intersects with @place and @size.
398 */
399bool ttm_resource_intersects(struct ttm_device *bdev,
400			     struct ttm_resource *res,
401			     const struct ttm_place *place,
402			     size_t size)
403{
404	struct ttm_resource_manager *man;
405
406	if (!res)
407		return false;
408
409	man = ttm_manager_type(bdev, res->mem_type);
410	if (!place || !man->func->intersects)
411		return true;
412
413	return man->func->intersects(man, res, place, size);
414}
415
416/**
417 * ttm_resource_compatible - check if resource is compatible with placement
418 *
419 * @res: the resource to check
420 * @placement: the placement to check against
421 * @evicting: true if the caller is doing evictions
 
422 *
423 * Returns true if the placement is compatible.
 
 
424 */
425bool ttm_resource_compatible(struct ttm_resource *res,
426			     struct ttm_placement *placement,
427			     bool evicting)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
428{
429	struct ttm_buffer_object *bo = res->bo;
430	struct ttm_device *bdev = bo->bdev;
431	unsigned i;
432
433	if (res->placement & TTM_PL_FLAG_TEMPORARY)
434		return false;
435
436	for (i = 0; i < placement->num_placement; i++) {
437		const struct ttm_place *place = &placement->placement[i];
438		struct ttm_resource_manager *man;
439
440		if (res->mem_type != place->mem_type)
441			continue;
442
443		if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :
444				    TTM_PL_FLAG_FALLBACK))
445			continue;
446
447		if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&
448		    !(res->placement & TTM_PL_FLAG_CONTIGUOUS))
449			continue;
 
 
 
 
450
451		man = ttm_manager_type(bdev, res->mem_type);
452		if (man->func->compatible &&
453		    !man->func->compatible(man, res, place, bo->base.size))
454			continue;
 
 
 
 
 
 
 
 
 
 
455
 
 
 
 
456		return true;
457	}
458	return false;
459}
 
460
461void ttm_resource_set_bo(struct ttm_resource *res,
462			 struct ttm_buffer_object *bo)
463{
464	spin_lock(&bo->bdev->lru_lock);
465	res->bo = bo;
466	spin_unlock(&bo->bdev->lru_lock);
467}
468
469/**
470 * ttm_resource_manager_init
471 *
472 * @man: memory manager object to init
473 * @bdev: ttm device this manager belongs to
474 * @size: size of managed resources in arbitrary units
475 *
476 * Initialise core parts of a manager object.
477 */
478void ttm_resource_manager_init(struct ttm_resource_manager *man,
479			       struct ttm_device *bdev,
480			       uint64_t size)
481{
482	unsigned i;
483
484	spin_lock_init(&man->move_lock);
485	man->bdev = bdev;
486	man->size = size;
487	man->usage = 0;
488
489	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
490		INIT_LIST_HEAD(&man->lru[i]);
491	man->move = NULL;
492}
493EXPORT_SYMBOL(ttm_resource_manager_init);
494
495/*
496 * ttm_resource_manager_evict_all
497 *
498 * @bdev - device to use
499 * @man - manager to use
500 *
501 * Evict all the objects out of a memory manager until it is empty.
502 * Part of memory manager cleanup sequence.
503 */
504int ttm_resource_manager_evict_all(struct ttm_device *bdev,
505				   struct ttm_resource_manager *man)
506{
507	struct ttm_operation_ctx ctx = {
508		.interruptible = false,
509		.no_wait_gpu = false,
510		.force_alloc = true
511	};
512	struct dma_fence *fence;
513	int ret;
 
514
515	do {
516		ret = ttm_bo_evict_first(bdev, man, &ctx);
517		cond_resched();
518	} while (!ret);
 
 
 
 
 
 
 
 
 
 
 
 
519
520	spin_lock(&man->move_lock);
521	fence = dma_fence_get(man->move);
522	spin_unlock(&man->move_lock);
523
524	if (fence) {
525		ret = dma_fence_wait(fence, false);
526		dma_fence_put(fence);
527		if (ret)
528			return ret;
529	}
530
531	return 0;
532}
533EXPORT_SYMBOL(ttm_resource_manager_evict_all);
534
535/**
536 * ttm_resource_manager_usage
537 *
538 * @man: A memory manager object.
539 *
540 * Return how many resources are currently used.
541 */
542uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
543{
544	uint64_t usage;
545
546	spin_lock(&man->bdev->lru_lock);
547	usage = man->usage;
548	spin_unlock(&man->bdev->lru_lock);
549	return usage;
550}
551EXPORT_SYMBOL(ttm_resource_manager_usage);
552
553/**
554 * ttm_resource_manager_debug
555 *
556 * @man: manager type to dump.
557 * @p: printer to use for debug.
558 */
559void ttm_resource_manager_debug(struct ttm_resource_manager *man,
560				struct drm_printer *p)
561{
562	drm_printf(p, "  use_type: %d\n", man->use_type);
563	drm_printf(p, "  use_tt: %d\n", man->use_tt);
564	drm_printf(p, "  size: %llu\n", man->size);
565	drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
566	if (man->func->debug)
567		man->func->debug(man, p);
568}
569EXPORT_SYMBOL(ttm_resource_manager_debug);
570
571static void
572ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
573			       struct ttm_lru_item *next_lru)
574{
575	struct ttm_resource *next = ttm_lru_item_to_res(next_lru);
576	struct ttm_lru_bulk_move *bulk = NULL;
577	struct ttm_buffer_object *bo = next->bo;
578
579	lockdep_assert_held(&cursor->man->bdev->lru_lock);
580	bulk = bo->bulk_move;
581
582	if (cursor->bulk != bulk) {
583		if (bulk) {
584			list_move_tail(&cursor->bulk_link, &bulk->cursor_list);
585			cursor->mem_type = next->mem_type;
586		} else {
587			list_del_init(&cursor->bulk_link);
588		}
589		cursor->bulk = bulk;
590	}
591}
592
593/**
594 * ttm_resource_manager_first() - Start iterating over the resources
595 * of a resource manager
596 * @man: resource manager to iterate over
597 * @cursor: cursor to record the position
598 *
599 * Initializes the cursor and starts iterating. When done iterating,
600 * the caller must explicitly call ttm_resource_cursor_fini().
601 *
602 * Return: The first resource from the resource manager.
603 */
604struct ttm_resource *
605ttm_resource_manager_first(struct ttm_resource_manager *man,
606			   struct ttm_resource_cursor *cursor)
607{
 
 
608	lockdep_assert_held(&man->bdev->lru_lock);
609
610	cursor->priority = 0;
611	cursor->man = man;
612	ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH);
613	INIT_LIST_HEAD(&cursor->bulk_link);
614	list_add(&cursor->hitch.link, &man->lru[cursor->priority]);
615
616	return ttm_resource_manager_next(cursor);
617}
618
619/**
620 * ttm_resource_manager_next() - Continue iterating over the resource manager
621 * resources
 
622 * @cursor: cursor to record the position
 
623 *
624 * Return: the next resource from the resource manager.
625 */
626struct ttm_resource *
627ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
 
 
628{
629	struct ttm_resource_manager *man = cursor->man;
630	struct ttm_lru_item *lru;
631
632	lockdep_assert_held(&man->bdev->lru_lock);
633
634	for (;;) {
635		lru = &cursor->hitch;
636		list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
637			if (ttm_lru_item_is_res(lru)) {
638				ttm_resource_cursor_check_bulk(cursor, lru);
639				list_move(&cursor->hitch.link, &lru->link);
640				return ttm_lru_item_to_res(lru);
641			}
642		}
643
644		if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
645			break;
646
647		list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
648		ttm_resource_cursor_clear_bulk(cursor);
649	}
650
651	ttm_resource_cursor_fini(cursor);
652
653	return NULL;
654}
655
656/**
657 * ttm_lru_first_res_or_null() - Return the first resource on an lru list
658 * @head: The list head of the lru list.
659 *
660 * Return: Pointer to the first resource on the lru list or NULL if
661 * there is none.
662 */
663struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head)
664{
665	struct ttm_lru_item *lru;
666
667	list_for_each_entry(lru, head, link) {
668		if (ttm_lru_item_is_res(lru))
669			return ttm_lru_item_to_res(lru);
670	}
671
672	return NULL;
673}
674
675static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
676					  struct iosys_map *dmap,
677					  pgoff_t i)
678{
679	struct ttm_kmap_iter_iomap *iter_io =
680		container_of(iter, typeof(*iter_io), base);
681	void __iomem *addr;
682
683retry:
684	while (i >= iter_io->cache.end) {
685		iter_io->cache.sg = iter_io->cache.sg ?
686			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
687		iter_io->cache.i = iter_io->cache.end;
688		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
689			PAGE_SHIFT;
690		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
691			iter_io->start;
692	}
693
694	if (i < iter_io->cache.i) {
695		iter_io->cache.end = 0;
696		iter_io->cache.sg = NULL;
697		goto retry;
698	}
699
700	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
701				       (((resource_size_t)i - iter_io->cache.i)
702					<< PAGE_SHIFT));
703	iosys_map_set_vaddr_iomem(dmap, addr);
704}
705
706static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
707					    struct iosys_map *map)
708{
709	io_mapping_unmap_local(map->vaddr_iomem);
710}
711
712static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
713	.map_local =  ttm_kmap_iter_iomap_map_local,
714	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
715	.maps_tt = false,
716};
717
718/**
719 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
720 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
721 * @iomap: The struct io_mapping representing the underlying linear io_memory.
722 * @st: sg_table into @iomap, representing the memory of the struct
723 * ttm_resource.
724 * @start: Offset that needs to be subtracted from @st to make
725 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
726 *
727 * Return: Pointer to the embedded struct ttm_kmap_iter.
728 */
729struct ttm_kmap_iter *
730ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
731			 struct io_mapping *iomap,
732			 struct sg_table *st,
733			 resource_size_t start)
734{
735	iter_io->base.ops = &ttm_kmap_iter_io_ops;
736	iter_io->iomap = iomap;
737	iter_io->st = st;
738	iter_io->start = start;
739	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
740
741	return &iter_io->base;
742}
743EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
744
745/**
746 * DOC: Linear io iterator
747 *
748 * This code should die in the not too near future. Best would be if we could
749 * make io-mapping use memremap for all io memory, and have memremap
750 * implement a kmap_local functionality. We could then strip a huge amount of
751 * code. These linear io iterators are implemented to mimic old functionality,
752 * and they don't use kmap_local semantics at all internally. Rather ioremap or
753 * friends, and at least on 32-bit they add global TLB flushes and points
754 * of failure.
755 */
756
757static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
758					      struct iosys_map *dmap,
759					      pgoff_t i)
760{
761	struct ttm_kmap_iter_linear_io *iter_io =
762		container_of(iter, typeof(*iter_io), base);
763
764	*dmap = iter_io->dmap;
765	iosys_map_incr(dmap, i * PAGE_SIZE);
766}
767
768static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
769	.map_local =  ttm_kmap_iter_linear_io_map_local,
770	.maps_tt = false,
771};
772
773/**
774 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
775 * @iter_io: The iterator to initialize
776 * @bdev: The TTM device
777 * @mem: The ttm resource representing the iomap.
778 *
779 * This function is for internal TTM use only. It sets up a memcpy kmap iterator
780 * pointing at a linear chunk of io memory.
781 *
782 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
783 * failure.
784 */
785struct ttm_kmap_iter *
786ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
787			     struct ttm_device *bdev,
788			     struct ttm_resource *mem)
789{
790	int ret;
791
792	ret = ttm_mem_io_reserve(bdev, mem);
793	if (ret)
794		goto out_err;
795	if (!mem->bus.is_iomem) {
796		ret = -EINVAL;
797		goto out_io_free;
798	}
799
800	if (mem->bus.addr) {
801		iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
802		iter_io->needs_unmap = false;
803	} else {
804		iter_io->needs_unmap = true;
805		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
806		if (mem->bus.caching == ttm_write_combined)
807			iosys_map_set_vaddr_iomem(&iter_io->dmap,
808						  ioremap_wc(mem->bus.offset,
809							     mem->size));
810		else if (mem->bus.caching == ttm_cached)
811			iosys_map_set_vaddr(&iter_io->dmap,
812					    memremap(mem->bus.offset, mem->size,
813						     MEMREMAP_WB |
814						     MEMREMAP_WT |
815						     MEMREMAP_WC));
816
817		/* If uncached requested or if mapping cached or wc failed */
818		if (iosys_map_is_null(&iter_io->dmap))
819			iosys_map_set_vaddr_iomem(&iter_io->dmap,
820						  ioremap(mem->bus.offset,
821							  mem->size));
822
823		if (iosys_map_is_null(&iter_io->dmap)) {
824			ret = -ENOMEM;
825			goto out_io_free;
826		}
827	}
828
829	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
830	return &iter_io->base;
831
832out_io_free:
833	ttm_mem_io_free(bdev, mem);
834out_err:
835	return ERR_PTR(ret);
836}
837
838/**
839 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
840 * @iter_io: The iterator to initialize
841 * @bdev: The TTM device
842 * @mem: The ttm resource representing the iomap.
843 *
844 * This function is for internal TTM use only. It cleans up a memcpy kmap
845 * iterator initialized by ttm_kmap_iter_linear_io_init.
846 */
847void
848ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
849			     struct ttm_device *bdev,
850			     struct ttm_resource *mem)
851{
852	if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
853		if (iter_io->dmap.is_iomem)
854			iounmap(iter_io->dmap.vaddr_iomem);
855		else
856			memunmap(iter_io->dmap.vaddr);
857	}
858
859	ttm_mem_io_free(bdev, mem);
860}
861
862#if defined(CONFIG_DEBUG_FS)
863
864static int ttm_resource_manager_show(struct seq_file *m, void *unused)
865{
866	struct ttm_resource_manager *man =
867		(struct ttm_resource_manager *)m->private;
868	struct drm_printer p = drm_seq_file_printer(m);
869	ttm_resource_manager_debug(man, &p);
870	return 0;
871}
872DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
873
874#endif
875
876/**
877 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
878 * resource manager.
879 * @man: The TTM resource manager for which the debugfs stats file be creates
880 * @parent: debugfs directory in which the file will reside
881 * @name: The filename to create.
882 *
883 * This function setups up a debugfs file that can be used to look
884 * at debug statistics of the specified ttm_resource_manager.
885 */
886void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
887					 struct dentry * parent,
888					 const char *name)
889{
890#if defined(CONFIG_DEBUG_FS)
891	debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
892#endif
893}
894EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);