1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * Copyright (C) 2023 Loongson Technology Corporation Limited
  4 */
  5
  6#include <drm/drm_drv.h>
  7#include <drm/drm_file.h>
  8#include <drm/drm_gem.h>
  9#include <drm/drm_managed.h>
 10#include <drm/drm_prime.h>
 11
 12#include "lsdc_drv.h"
 13#include "lsdc_ttm.h"
 14
 15const char *lsdc_mem_type_to_str(uint32_t mem_type)
 16{
 17	switch (mem_type) {
 18	case TTM_PL_VRAM:
 19		return "VRAM";
 20	case TTM_PL_TT:
 21		return "GTT";
 22	case TTM_PL_SYSTEM:
 23		return "SYSTEM";
 24	default:
 25		break;
 26	}
 27
 28	return "Unknown";
 29}
 30
 31const char *lsdc_domain_to_str(u32 domain)
 32{
 33	switch (domain) {
 34	case LSDC_GEM_DOMAIN_VRAM:
 35		return "VRAM";
 36	case LSDC_GEM_DOMAIN_GTT:
 37		return "GTT";
 38	case LSDC_GEM_DOMAIN_SYSTEM:
 39		return "SYSTEM";
 40	default:
 41		break;
 42	}
 43
 44	return "Unknown";
 45}
 46
 47static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
 48{
 49	u32 c = 0;
 50	u32 pflags = 0;
 51	u32 i;
 52
 53	if (lbo->tbo.base.size <= PAGE_SIZE)
 54		pflags |= TTM_PL_FLAG_TOPDOWN;
 55
 56	lbo->placement.placement = lbo->placements;
 57
 58	if (domain & LSDC_GEM_DOMAIN_VRAM) {
 59		lbo->placements[c].mem_type = TTM_PL_VRAM;
 60		lbo->placements[c++].flags = pflags;
 61	}
 62
 63	if (domain & LSDC_GEM_DOMAIN_GTT) {
 64		lbo->placements[c].mem_type = TTM_PL_TT;
 65		lbo->placements[c++].flags = pflags;
 66	}
 67
 68	if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
 69		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
 70		lbo->placements[c++].flags = 0;
 71	}
 72
 73	if (!c) {
 74		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
 75		lbo->placements[c++].flags = 0;
 76	}
 77
 78	lbo->placement.num_placement = c;
 79
 80	for (i = 0; i < c; ++i) {
 81		lbo->placements[i].fpfn = 0;
 82		lbo->placements[i].lpfn = 0;
 83	}
 84}
 85
 86static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
 87{
 88	ttm_tt_fini(tt);
 89	kfree(tt);
 90}
 91
 92static struct ttm_tt *
 93lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
 94{
 95	struct ttm_tt *tt;
 96	int ret;
 97
 98	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
 99	if (!tt)
100		return NULL;
101
102	ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
103	if (ret < 0) {
104		kfree(tt);
105		return NULL;
106	}
107
108	return tt;
109}
110
111static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
112				struct ttm_tt *ttm,
113				struct ttm_operation_ctx *ctx)
114{
115	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
116
117	if (slave && ttm->sg) {
118		drm_prime_sg_to_dma_addr_array(ttm->sg,
119					       ttm->dma_address,
120					       ttm->num_pages);
121
122		return 0;
123	}
124
125	return ttm_pool_alloc(&bdev->pool, ttm, ctx);
126}
127
128static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
129				   struct ttm_tt *ttm)
130{
131	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
132
133	if (slave)
134		return;
135
136	return ttm_pool_free(&bdev->pool, ttm);
137}
138
139static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
140				struct ttm_placement *tplacement)
141{
142	struct ttm_resource *resource = tbo->resource;
143	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
144
145	switch (resource->mem_type) {
146	case TTM_PL_VRAM:
147		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
148		break;
149	case TTM_PL_TT:
150	default:
151		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
152		break;
153	}
154
155	*tplacement = lbo->placement;
156}
157
158static int lsdc_bo_move(struct ttm_buffer_object *tbo,
159			bool evict,
160			struct ttm_operation_ctx *ctx,
161			struct ttm_resource *new_mem,
162			struct ttm_place *hop)
163{
164	struct drm_device *ddev = tbo->base.dev;
165	struct ttm_resource *old_mem = tbo->resource;
166	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
167	int ret;
168
169	if (unlikely(tbo->pin_count > 0)) {
170		drm_warn(ddev, "Can't move a pinned BO\n");
171		return -EINVAL;
172	}
173
174	ret = ttm_bo_wait_ctx(tbo, ctx);
175	if (ret)
176		return ret;
177
178	if (!old_mem) {
179		drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
180			lbo, lsdc_mem_type_to_str(new_mem->mem_type),
181			lsdc_bo_size(lbo));
182		ttm_bo_move_null(tbo, new_mem);
183		return 0;
184	}
185
186	if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
187		ttm_bo_move_null(tbo, new_mem);
188		drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
189			lbo, lsdc_bo_size(lbo));
190		return 0;
191	}
192
193	if (old_mem->mem_type == TTM_PL_SYSTEM &&
194	    new_mem->mem_type == TTM_PL_TT) {
195		drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
196			lbo, lsdc_bo_size(lbo));
197		ttm_bo_move_null(tbo, new_mem);
198		return 0;
199	}
200
201	if (old_mem->mem_type == TTM_PL_TT &&
202	    new_mem->mem_type == TTM_PL_SYSTEM) {
203		drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
204			lbo, lsdc_bo_size(lbo));
205		ttm_resource_free(tbo, &tbo->resource);
206		ttm_bo_assign_mem(tbo, new_mem);
207		return 0;
208	}
209
210	drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
211		lbo,
212		lsdc_mem_type_to_str(old_mem->mem_type),
213		lsdc_mem_type_to_str(new_mem->mem_type),
214		lsdc_bo_size(lbo));
215
216	return ttm_bo_move_memcpy(tbo, ctx, new_mem);
217}
218
219static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
220				  struct ttm_resource *mem)
221{
222	struct lsdc_device *ldev = tdev_to_ldev(bdev);
223
224	switch (mem->mem_type) {
225	case TTM_PL_SYSTEM:
226		break;
227	case TTM_PL_TT:
228		break;
229	case TTM_PL_VRAM:
230		mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
231		mem->bus.is_iomem = true;
232		mem->bus.caching = ttm_write_combined;
233		break;
234	default:
235		return -EINVAL;
236	}
237
238	return 0;
239}
240
241static struct ttm_device_funcs lsdc_bo_driver = {
242	.ttm_tt_create = lsdc_ttm_tt_create,
243	.ttm_tt_populate = lsdc_ttm_tt_populate,
244	.ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
245	.ttm_tt_destroy = lsdc_ttm_tt_destroy,
246	.eviction_valuable = ttm_bo_eviction_valuable,
247	.evict_flags = lsdc_bo_evict_flags,
248	.move = lsdc_bo_move,
249	.io_mem_reserve = lsdc_bo_reserve_io_mem,
250};
251
252u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
253{
254	struct ttm_buffer_object *tbo = &lbo->tbo;
255	struct drm_device *ddev = tbo->base.dev;
256	struct ttm_resource *resource = tbo->resource;
257
258	if (unlikely(!tbo->pin_count)) {
259		drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
260		return 0;
261	}
262
263	if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
264		return 0;
265
266	return resource->start << PAGE_SHIFT;
267}
268
269size_t lsdc_bo_size(struct lsdc_bo *lbo)
270{
271	struct ttm_buffer_object *tbo = &lbo->tbo;
272
273	return tbo->base.size;
274}
275
276int lsdc_bo_reserve(struct lsdc_bo *lbo)
277{
278	return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
279}
280
281void lsdc_bo_unreserve(struct lsdc_bo *lbo)
282{
283	return ttm_bo_unreserve(&lbo->tbo);
284}
285
286int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
287{
288	struct ttm_operation_ctx ctx = { false, false };
289	struct ttm_buffer_object *tbo = &lbo->tbo;
290	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
291	int ret;
292
293	if (tbo->pin_count)
294		goto bo_pinned;
295
296	if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
297		return -EINVAL;
298
299	if (domain)
300		lsdc_bo_set_placement(lbo, domain);
301
302	ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
303	if (unlikely(ret)) {
304		drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
305		return ret;
306	}
307
308	if (domain == LSDC_GEM_DOMAIN_VRAM)
309		ldev->vram_pinned_size += lsdc_bo_size(lbo);
310	else if (domain == LSDC_GEM_DOMAIN_GTT)
311		ldev->gtt_pinned_size += lsdc_bo_size(lbo);
312
313bo_pinned:
314	ttm_bo_pin(tbo);
315
316	if (gpu_addr)
317		*gpu_addr = lsdc_bo_gpu_offset(lbo);
318
319	return 0;
320}
321
322void lsdc_bo_unpin(struct lsdc_bo *lbo)
323{
324	struct ttm_buffer_object *tbo = &lbo->tbo;
325	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
326
327	if (unlikely(!tbo->pin_count)) {
328		drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
329		return;
330	}
331
332	ttm_bo_unpin(tbo);
333
334	if (!tbo->pin_count) {
335		if (tbo->resource->mem_type == TTM_PL_VRAM)
336			ldev->vram_pinned_size -= lsdc_bo_size(lbo);
337		else if (tbo->resource->mem_type == TTM_PL_TT)
338			ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
339	}
340}
341
342void lsdc_bo_ref(struct lsdc_bo *lbo)
343{
344	struct ttm_buffer_object *tbo = &lbo->tbo;
345
346	ttm_bo_get(tbo);
347}
348
349void lsdc_bo_unref(struct lsdc_bo *lbo)
350{
351	struct ttm_buffer_object *tbo = &lbo->tbo;
352
353	ttm_bo_put(tbo);
354}
355
356int lsdc_bo_kmap(struct lsdc_bo *lbo)
357{
358	struct ttm_buffer_object *tbo = &lbo->tbo;
359	struct drm_gem_object *gem = &tbo->base;
360	struct drm_device *ddev = gem->dev;
361	long ret;
362	int err;
363
364	ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
365				    MAX_SCHEDULE_TIMEOUT);
366	if (ret < 0) {
367		drm_warn(ddev, "wait fence timeout\n");
368		return ret;
369	}
370
371	if (lbo->kptr)
372		return 0;
373
374	err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
375	if (err) {
376		drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
377		return err;
378	}
379
380	lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
381
382	return 0;
383}
384
385void lsdc_bo_kunmap(struct lsdc_bo *lbo)
386{
387	if (!lbo->kptr)
388		return;
389
390	lbo->kptr = NULL;
391	ttm_bo_kunmap(&lbo->kmap);
392}
393
394void lsdc_bo_clear(struct lsdc_bo *lbo)
395{
396	lsdc_bo_kmap(lbo);
397
398	if (lbo->is_iomem)
399		memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
400	else
401		memset(lbo->kptr, 0, lbo->size);
402
403	lsdc_bo_kunmap(lbo);
404}
405
406int lsdc_bo_evict_vram(struct drm_device *ddev)
407{
408	struct lsdc_device *ldev = to_lsdc(ddev);
409	struct ttm_device *bdev = &ldev->bdev;
410	struct ttm_resource_manager *man;
411
412	man = ttm_manager_type(bdev, TTM_PL_VRAM);
413	if (unlikely(!man))
414		return 0;
415
416	return ttm_resource_manager_evict_all(bdev, man);
417}
418
419static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
420{
421	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
422	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
423
424	mutex_lock(&ldev->gem.mutex);
425	list_del_init(&lbo->list);
426	mutex_unlock(&ldev->gem.mutex);
427
428	drm_gem_object_release(&tbo->base);
429
430	kfree(lbo);
431}
432
433struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
434			       u32 domain,
435			       size_t size,
436			       bool kernel,
437			       struct sg_table *sg,
438			       struct dma_resv *resv)
439{
440	struct lsdc_device *ldev = to_lsdc(ddev);
441	struct ttm_device *bdev = &ldev->bdev;
442	struct ttm_buffer_object *tbo;
443	struct lsdc_bo *lbo;
444	enum ttm_bo_type bo_type;
445	int ret;
446
447	lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
448	if (!lbo)
449		return ERR_PTR(-ENOMEM);
450
451	INIT_LIST_HEAD(&lbo->list);
452
453	lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
454					LSDC_GEM_DOMAIN_GTT |
455					LSDC_GEM_DOMAIN_SYSTEM);
456
457	tbo = &lbo->tbo;
458
459	size = ALIGN(size, PAGE_SIZE);
460
461	ret = drm_gem_object_init(ddev, &tbo->base, size);
462	if (ret) {
463		kfree(lbo);
464		return ERR_PTR(ret);
465	}
466
467	tbo->bdev = bdev;
468
469	if (kernel)
470		bo_type = ttm_bo_type_kernel;
471	else if (sg)
472		bo_type = ttm_bo_type_sg;
473	else
474		bo_type = ttm_bo_type_device;
475
476	lsdc_bo_set_placement(lbo, domain);
477	lbo->size = size;
478
479	ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
480				   false, sg, resv, lsdc_bo_destroy);
481	if (ret) {
482		kfree(lbo);
483		return ERR_PTR(ret);
484	}
485
486	return lbo;
487}
488
489struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
490					     u32 domain,
491					     size_t size)
492{
493	struct lsdc_bo *lbo;
494	int ret;
495
496	lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
497	if (IS_ERR(lbo))
498		return ERR_CAST(lbo);
499
500	ret = lsdc_bo_reserve(lbo);
501	if (unlikely(ret)) {
502		lsdc_bo_unref(lbo);
503		return ERR_PTR(ret);
504	}
505
506	ret = lsdc_bo_pin(lbo, domain, NULL);
507	lsdc_bo_unreserve(lbo);
508	if (unlikely(ret)) {
509		lsdc_bo_unref(lbo);
510		return ERR_PTR(ret);
511	}
512
513	return lbo;
514}
515
516void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
517{
518	int ret;
519
520	ret = lsdc_bo_reserve(lbo);
521	if (unlikely(ret))
522		return;
523
524	lsdc_bo_unpin(lbo);
525	lsdc_bo_unreserve(lbo);
526
527	lsdc_bo_unref(lbo);
528}
529
530static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
531{
532	struct lsdc_device *ldev = (struct lsdc_device *)data;
533
534	ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
535	ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
536
537	ttm_device_fini(&ldev->bdev);
538
539	drm_dbg(ddev, "ttm finished\n");
540}
541
542int lsdc_ttm_init(struct lsdc_device *ldev)
543{
544	struct drm_device *ddev = &ldev->base;
545	unsigned long num_vram_pages;
546	unsigned long num_gtt_pages;
547	int ret;
548
549	ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
550			      ddev->anon_inode->i_mapping,
551			      ddev->vma_offset_manager, false, true);
552	if (ret)
553		return ret;
554
555	num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
556
557	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
558	if (unlikely(ret))
559		return ret;
560
561	drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
562
563	/* 512M is far enough for us now */
564	ldev->gtt_size = 512 << 20;
565
566	num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
567
568	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
569	if (unlikely(ret))
570		return ret;
571
572	drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
573
574	return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
575}
576
577void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
578{
579	struct ttm_device *bdev = &ldev->bdev;
580	struct drm_device *ddev = &ldev->base;
581	struct drm_minor *minor = ddev->primary;
582	struct dentry *root = minor->debugfs_root;
583	struct ttm_resource_manager *vram_man;
584	struct ttm_resource_manager *gtt_man;
585
586	vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
587	gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
588
589	ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
590	ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
591}