1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3#include <drm/drm_gem_vram_helper.h>
  4#include <drm/drm_device.h>
  5#include <drm/drm_mode.h>
  6#include <drm/drm_prime.h>
  7#include <drm/drm_vram_mm_helper.h>
  8#include <drm/ttm/ttm_page_alloc.h>
  9
 10static const struct drm_gem_object_funcs drm_gem_vram_object_funcs;
 11
 12/**
 13 * DOC: overview
 14 *
 15 * This library provides a GEM buffer object that is backed by video RAM
 16 * (VRAM). It can be used for framebuffer devices with dedicated memory.
 17 */
 18
 19/*
 20 * Buffer-objects helpers
 21 */
 22
 23static void drm_gem_vram_cleanup(struct drm_gem_vram_object *gbo)
 24{
 25	/* We got here via ttm_bo_put(), which means that the
 26	 * TTM buffer object in 'bo' has already been cleaned
 27	 * up; only release the GEM object.
 28	 */
 29	drm_gem_object_release(&gbo->bo.base);
 30}
 31
 32static void drm_gem_vram_destroy(struct drm_gem_vram_object *gbo)
 33{
 34	drm_gem_vram_cleanup(gbo);
 35	kfree(gbo);
 36}
 37
 38static void ttm_buffer_object_destroy(struct ttm_buffer_object *bo)
 39{
 40	struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo);
 41
 42	drm_gem_vram_destroy(gbo);
 43}
 44
 45static void drm_gem_vram_placement(struct drm_gem_vram_object *gbo,
 46				   unsigned long pl_flag)
 47{
 48	unsigned int i;
 49	unsigned int c = 0;
 50
 51	gbo->placement.placement = gbo->placements;
 52	gbo->placement.busy_placement = gbo->placements;
 53
 54	if (pl_flag & TTM_PL_FLAG_VRAM)
 55		gbo->placements[c++].flags = TTM_PL_FLAG_WC |
 56					     TTM_PL_FLAG_UNCACHED |
 57					     TTM_PL_FLAG_VRAM;
 58
 59	if (pl_flag & TTM_PL_FLAG_SYSTEM)
 60		gbo->placements[c++].flags = TTM_PL_MASK_CACHING |
 61					     TTM_PL_FLAG_SYSTEM;
 62
 63	if (!c)
 64		gbo->placements[c++].flags = TTM_PL_MASK_CACHING |
 65					     TTM_PL_FLAG_SYSTEM;
 66
 67	gbo->placement.num_placement = c;
 68	gbo->placement.num_busy_placement = c;
 69
 70	for (i = 0; i < c; ++i) {
 71		gbo->placements[i].fpfn = 0;
 72		gbo->placements[i].lpfn = 0;
 73	}
 74}
 75
 76static int drm_gem_vram_init(struct drm_device *dev,
 77			     struct ttm_bo_device *bdev,
 78			     struct drm_gem_vram_object *gbo,
 79			     size_t size, unsigned long pg_align,
 80			     bool interruptible)
 81{
 82	int ret;
 83	size_t acc_size;
 84
 85	if (!gbo->bo.base.funcs)
 86		gbo->bo.base.funcs = &drm_gem_vram_object_funcs;
 87
 88	ret = drm_gem_object_init(dev, &gbo->bo.base, size);
 89	if (ret)
 90		return ret;
 91
 92	acc_size = ttm_bo_dma_acc_size(bdev, size, sizeof(*gbo));
 93
 94	gbo->bo.bdev = bdev;
 95	drm_gem_vram_placement(gbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
 96
 97	ret = ttm_bo_init(bdev, &gbo->bo, size, ttm_bo_type_device,
 98			  &gbo->placement, pg_align, interruptible, acc_size,
 99			  NULL, NULL, ttm_buffer_object_destroy);
100	if (ret)
101		goto err_drm_gem_object_release;
102
103	return 0;
104
105err_drm_gem_object_release:
106	drm_gem_object_release(&gbo->bo.base);
107	return ret;
108}
109
110/**
111 * drm_gem_vram_create() - Creates a VRAM-backed GEM object
112 * @dev:		the DRM device
113 * @bdev:		the TTM BO device backing the object
114 * @size:		the buffer size in bytes
115 * @pg_align:		the buffer's alignment in multiples of the page size
116 * @interruptible:	sleep interruptible if waiting for memory
117 *
118 * Returns:
119 * A new instance of &struct drm_gem_vram_object on success, or
120 * an ERR_PTR()-encoded error code otherwise.
121 */
122struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev,
123						struct ttm_bo_device *bdev,
124						size_t size,
125						unsigned long pg_align,
126						bool interruptible)
127{
128	struct drm_gem_vram_object *gbo;
129	int ret;
130
131	gbo = kzalloc(sizeof(*gbo), GFP_KERNEL);
132	if (!gbo)
133		return ERR_PTR(-ENOMEM);
134
135	ret = drm_gem_vram_init(dev, bdev, gbo, size, pg_align, interruptible);
136	if (ret < 0)
137		goto err_kfree;
138
139	return gbo;
140
141err_kfree:
142	kfree(gbo);
143	return ERR_PTR(ret);
144}
145EXPORT_SYMBOL(drm_gem_vram_create);
146
147/**
148 * drm_gem_vram_put() - Releases a reference to a VRAM-backed GEM object
149 * @gbo:	the GEM VRAM object
150 *
151 * See ttm_bo_put() for more information.
152 */
153void drm_gem_vram_put(struct drm_gem_vram_object *gbo)
154{
155	ttm_bo_put(&gbo->bo);
156}
157EXPORT_SYMBOL(drm_gem_vram_put);
158
159/**
160 * drm_gem_vram_mmap_offset() - Returns a GEM VRAM object's mmap offset
161 * @gbo:	the GEM VRAM object
162 *
163 * See drm_vma_node_offset_addr() for more information.
164 *
165 * Returns:
166 * The buffer object's offset for userspace mappings on success, or
167 * 0 if no offset is allocated.
168 */
169u64 drm_gem_vram_mmap_offset(struct drm_gem_vram_object *gbo)
170{
171	return drm_vma_node_offset_addr(&gbo->bo.base.vma_node);
172}
173EXPORT_SYMBOL(drm_gem_vram_mmap_offset);
174
175/**
176 * drm_gem_vram_offset() - \
177	Returns a GEM VRAM object's offset in video memory
178 * @gbo:	the GEM VRAM object
179 *
180 * This function returns the buffer object's offset in the device's video
181 * memory. The buffer object has to be pinned to %TTM_PL_VRAM.
182 *
183 * Returns:
184 * The buffer object's offset in video memory on success, or
185 * a negative errno code otherwise.
186 */
187s64 drm_gem_vram_offset(struct drm_gem_vram_object *gbo)
188{
189	if (WARN_ON_ONCE(!gbo->pin_count))
190		return (s64)-ENODEV;
191	return gbo->bo.offset;
192}
193EXPORT_SYMBOL(drm_gem_vram_offset);
194
195/**
196 * drm_gem_vram_pin() - Pins a GEM VRAM object in a region.
197 * @gbo:	the GEM VRAM object
198 * @pl_flag:	a bitmask of possible memory regions
199 *
200 * Pinning a buffer object ensures that it is not evicted from
201 * a memory region. A pinned buffer object has to be unpinned before
202 * it can be pinned to another region. If the pl_flag argument is 0,
203 * the buffer is pinned at its current location (video RAM or system
204 * memory).
205 *
206 * Returns:
207 * 0 on success, or
208 * a negative error code otherwise.
209 */
210int drm_gem_vram_pin(struct drm_gem_vram_object *gbo, unsigned long pl_flag)
211{
212	int i, ret;
213	struct ttm_operation_ctx ctx = { false, false };
214
215	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);
216	if (ret < 0)
217		return ret;
218
219	if (gbo->pin_count)
220		goto out;
221
222	if (pl_flag)
223		drm_gem_vram_placement(gbo, pl_flag);
224
225	for (i = 0; i < gbo->placement.num_placement; ++i)
226		gbo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
227
228	ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx);
229	if (ret < 0)
230		goto err_ttm_bo_unreserve;
231
232out:
233	++gbo->pin_count;
234	ttm_bo_unreserve(&gbo->bo);
235
236	return 0;
237
238err_ttm_bo_unreserve:
239	ttm_bo_unreserve(&gbo->bo);
240	return ret;
241}
242EXPORT_SYMBOL(drm_gem_vram_pin);
243
244/**
245 * drm_gem_vram_unpin() - Unpins a GEM VRAM object
246 * @gbo:	the GEM VRAM object
247 *
248 * Returns:
249 * 0 on success, or
250 * a negative error code otherwise.
251 */
252int drm_gem_vram_unpin(struct drm_gem_vram_object *gbo)
253{
254	int i, ret;
255	struct ttm_operation_ctx ctx = { false, false };
256
257	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);
258	if (ret < 0)
259		return ret;
260
261	if (WARN_ON_ONCE(!gbo->pin_count))
262		goto out;
263
264	--gbo->pin_count;
265	if (gbo->pin_count)
266		goto out;
267
268	for (i = 0; i < gbo->placement.num_placement ; ++i)
269		gbo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
270
271	ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx);
272	if (ret < 0)
273		goto err_ttm_bo_unreserve;
274
275out:
276	ttm_bo_unreserve(&gbo->bo);
277
278	return 0;
279
280err_ttm_bo_unreserve:
281	ttm_bo_unreserve(&gbo->bo);
282	return ret;
283}
284EXPORT_SYMBOL(drm_gem_vram_unpin);
285
286/**
287 * drm_gem_vram_kmap() - Maps a GEM VRAM object into kernel address space
288 * @gbo:	the GEM VRAM object
289 * @map:	establish a mapping if necessary
290 * @is_iomem:	returns true if the mapped memory is I/O memory, or false \
291	otherwise; can be NULL
292 *
293 * This function maps the buffer object into the kernel's address space
294 * or returns the current mapping. If the parameter map is false, the
295 * function only queries the current mapping, but does not establish a
296 * new one.
297 *
298 * Returns:
299 * The buffers virtual address if mapped, or
300 * NULL if not mapped, or
301 * an ERR_PTR()-encoded error code otherwise.
302 */
303void *drm_gem_vram_kmap(struct drm_gem_vram_object *gbo, bool map,
304			bool *is_iomem)
305{
306	int ret;
307	struct ttm_bo_kmap_obj *kmap = &gbo->kmap;
308
309	if (kmap->virtual || !map)
310		goto out;
311
312	ret = ttm_bo_kmap(&gbo->bo, 0, gbo->bo.num_pages, kmap);
313	if (ret)
314		return ERR_PTR(ret);
315
316out:
317	if (!is_iomem)
318		return kmap->virtual;
319	if (!kmap->virtual) {
320		*is_iomem = false;
321		return NULL;
322	}
323	return ttm_kmap_obj_virtual(kmap, is_iomem);
324}
325EXPORT_SYMBOL(drm_gem_vram_kmap);
326
327/**
328 * drm_gem_vram_kunmap() - Unmaps a GEM VRAM object
329 * @gbo:	the GEM VRAM object
330 */
331void drm_gem_vram_kunmap(struct drm_gem_vram_object *gbo)
332{
333	struct ttm_bo_kmap_obj *kmap = &gbo->kmap;
334
335	if (!kmap->virtual)
336		return;
337
338	ttm_bo_kunmap(kmap);
339	kmap->virtual = NULL;
340}
341EXPORT_SYMBOL(drm_gem_vram_kunmap);
342
343/**
344 * drm_gem_vram_fill_create_dumb() - \
345	Helper for implementing &struct drm_driver.dumb_create
346 * @file:		the DRM file
347 * @dev:		the DRM device
348 * @bdev:		the TTM BO device managing the buffer object
349 * @pg_align:		the buffer's alignment in multiples of the page size
350 * @interruptible:	sleep interruptible if waiting for memory
351 * @args:		the arguments as provided to \
352				&struct drm_driver.dumb_create
353 *
354 * This helper function fills &struct drm_mode_create_dumb, which is used
355 * by &struct drm_driver.dumb_create. Implementations of this interface
356 * should forwards their arguments to this helper, plus the driver-specific
357 * parameters.
358 *
359 * Returns:
360 * 0 on success, or
361 * a negative error code otherwise.
362 */
363int drm_gem_vram_fill_create_dumb(struct drm_file *file,
364				  struct drm_device *dev,
365				  struct ttm_bo_device *bdev,
366				  unsigned long pg_align,
367				  bool interruptible,
368				  struct drm_mode_create_dumb *args)
369{
370	size_t pitch, size;
371	struct drm_gem_vram_object *gbo;
372	int ret;
373	u32 handle;
374
375	pitch = args->width * ((args->bpp + 7) / 8);
376	size = pitch * args->height;
377
378	size = roundup(size, PAGE_SIZE);
379	if (!size)
380		return -EINVAL;
381
382	gbo = drm_gem_vram_create(dev, bdev, size, pg_align, interruptible);
383	if (IS_ERR(gbo))
384		return PTR_ERR(gbo);
385
386	ret = drm_gem_handle_create(file, &gbo->bo.base, &handle);
387	if (ret)
388		goto err_drm_gem_object_put_unlocked;
389
390	drm_gem_object_put_unlocked(&gbo->bo.base);
391
392	args->pitch = pitch;
393	args->size = size;
394	args->handle = handle;
395
396	return 0;
397
398err_drm_gem_object_put_unlocked:
399	drm_gem_object_put_unlocked(&gbo->bo.base);
400	return ret;
401}
402EXPORT_SYMBOL(drm_gem_vram_fill_create_dumb);
403
404/*
405 * Helpers for struct ttm_bo_driver
406 */
407
408static bool drm_is_gem_vram(struct ttm_buffer_object *bo)
409{
410	return (bo->destroy == ttm_buffer_object_destroy);
411}
412
413/**
414 * drm_gem_vram_bo_driver_evict_flags() - \
415	Implements &struct ttm_bo_driver.evict_flags
416 * @bo:	TTM buffer object. Refers to &struct drm_gem_vram_object.bo
417 * @pl:	TTM placement information.
418 */
419void drm_gem_vram_bo_driver_evict_flags(struct ttm_buffer_object *bo,
420					struct ttm_placement *pl)
421{
422	struct drm_gem_vram_object *gbo;
423
424	/* TTM may pass BOs that are not GEM VRAM BOs. */
425	if (!drm_is_gem_vram(bo))
426		return;
427
428	gbo = drm_gem_vram_of_bo(bo);
429	drm_gem_vram_placement(gbo, TTM_PL_FLAG_SYSTEM);
430	*pl = gbo->placement;
431}
432EXPORT_SYMBOL(drm_gem_vram_bo_driver_evict_flags);
433
434/**
435 * drm_gem_vram_bo_driver_verify_access() - \
436	Implements &struct ttm_bo_driver.verify_access
437 * @bo:		TTM buffer object. Refers to &struct drm_gem_vram_object.bo
438 * @filp:	File pointer.
439 *
440 * Returns:
441 * 0 on success, or
442 * a negative errno code otherwise.
443 */
444int drm_gem_vram_bo_driver_verify_access(struct ttm_buffer_object *bo,
445					 struct file *filp)
446{
447	struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo);
448
449	return drm_vma_node_verify_access(&gbo->bo.base.vma_node,
450					  filp->private_data);
451}
452EXPORT_SYMBOL(drm_gem_vram_bo_driver_verify_access);
453
454/*
455 * drm_gem_vram_mm_funcs - Functions for &struct drm_vram_mm
456 *
457 * Most users of @struct drm_gem_vram_object will also use
458 * @struct drm_vram_mm. This instance of &struct drm_vram_mm_funcs
459 * can be used to connect both.
460 */
461const struct drm_vram_mm_funcs drm_gem_vram_mm_funcs = {
462	.evict_flags = drm_gem_vram_bo_driver_evict_flags,
463	.verify_access = drm_gem_vram_bo_driver_verify_access
464};
465EXPORT_SYMBOL(drm_gem_vram_mm_funcs);
466
467/*
468 * Helpers for struct drm_gem_object_funcs
469 */
470
471/**
472 * drm_gem_vram_object_free() - \
473	Implements &struct drm_gem_object_funcs.free
474 * @gem:       GEM object. Refers to &struct drm_gem_vram_object.gem
475 */
476static void drm_gem_vram_object_free(struct drm_gem_object *gem)
477{
478	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
479
480	drm_gem_vram_put(gbo);
481}
482
483/*
484 * Helpers for dump buffers
485 */
486
487/**
488 * drm_gem_vram_driver_create_dumb() - \
489	Implements &struct drm_driver.dumb_create
490 * @file:		the DRM file
491 * @dev:		the DRM device
492 * @args:		the arguments as provided to \
493				&struct drm_driver.dumb_create
494 *
495 * This function requires the driver to use @drm_device.vram_mm for its
496 * instance of VRAM MM.
497 *
498 * Returns:
499 * 0 on success, or
500 * a negative error code otherwise.
501 */
502int drm_gem_vram_driver_dumb_create(struct drm_file *file,
503				    struct drm_device *dev,
504				    struct drm_mode_create_dumb *args)
505{
506	if (WARN_ONCE(!dev->vram_mm, "VRAM MM not initialized"))
507		return -EINVAL;
508
509	return drm_gem_vram_fill_create_dumb(file, dev, &dev->vram_mm->bdev, 0,
510					     false, args);
511}
512EXPORT_SYMBOL(drm_gem_vram_driver_dumb_create);
513
514/**
515 * drm_gem_vram_driver_dumb_mmap_offset() - \
516	Implements &struct drm_driver.dumb_mmap_offset
517 * @file:	DRM file pointer.
518 * @dev:	DRM device.
519 * @handle:	GEM handle
520 * @offset:	Returns the mapping's memory offset on success
521 *
522 * Returns:
523 * 0 on success, or
524 * a negative errno code otherwise.
525 */
526int drm_gem_vram_driver_dumb_mmap_offset(struct drm_file *file,
527					 struct drm_device *dev,
528					 uint32_t handle, uint64_t *offset)
529{
530	struct drm_gem_object *gem;
531	struct drm_gem_vram_object *gbo;
532
533	gem = drm_gem_object_lookup(file, handle);
534	if (!gem)
535		return -ENOENT;
536
537	gbo = drm_gem_vram_of_gem(gem);
538	*offset = drm_gem_vram_mmap_offset(gbo);
539
540	drm_gem_object_put_unlocked(gem);
541
542	return 0;
543}
544EXPORT_SYMBOL(drm_gem_vram_driver_dumb_mmap_offset);
545
546/*
547 * PRIME helpers
548 */
549
550/**
551 * drm_gem_vram_object_pin() - \
552	Implements &struct drm_gem_object_funcs.pin
553 * @gem:	The GEM object to pin
554 *
555 * Returns:
556 * 0 on success, or
557 * a negative errno code otherwise.
558 */
559static int drm_gem_vram_object_pin(struct drm_gem_object *gem)
560{
561	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
562
563	/* Fbdev console emulation is the use case of these PRIME
564	 * helpers. This may involve updating a hardware buffer from
565	 * a shadow FB. We pin the buffer to it's current location
566	 * (either video RAM or system memory) to prevent it from
567	 * being relocated during the update operation. If you require
568	 * the buffer to be pinned to VRAM, implement a callback that
569	 * sets the flags accordingly.
570	 */
571	return drm_gem_vram_pin(gbo, 0);
572}
573
574/**
575 * drm_gem_vram_object_unpin() - \
576	Implements &struct drm_gem_object_funcs.unpin
577 * @gem:	The GEM object to unpin
578 */
579static void drm_gem_vram_object_unpin(struct drm_gem_object *gem)
580{
581	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
582
583	drm_gem_vram_unpin(gbo);
584}
585
586/**
587 * drm_gem_vram_object_vmap() - \
588	Implements &struct drm_gem_object_funcs.vmap
589 * @gem:	The GEM object to map
590 *
591 * Returns:
592 * The buffers virtual address on success, or
593 * NULL otherwise.
594 */
595static void *drm_gem_vram_object_vmap(struct drm_gem_object *gem)
596{
597	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
598	int ret;
599	void *base;
600
601	ret = drm_gem_vram_pin(gbo, 0);
602	if (ret)
603		return NULL;
604	base = drm_gem_vram_kmap(gbo, true, NULL);
605	if (IS_ERR(base)) {
606		drm_gem_vram_unpin(gbo);
607		return NULL;
608	}
609	return base;
610}
611
612/**
613 * drm_gem_vram_object_vunmap() - \
614	Implements &struct drm_gem_object_funcs.vunmap
615 * @gem:	The GEM object to unmap
616 * @vaddr:	The mapping's base address
617 */
618static void drm_gem_vram_object_vunmap(struct drm_gem_object *gem,
619				       void *vaddr)
620{
621	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);
622
623	drm_gem_vram_kunmap(gbo);
624	drm_gem_vram_unpin(gbo);
625}
626
627/*
628 * GEM object funcs
629 */
630
631static const struct drm_gem_object_funcs drm_gem_vram_object_funcs = {
632	.free	= drm_gem_vram_object_free,
633	.pin	= drm_gem_vram_object_pin,
634	.unpin	= drm_gem_vram_object_unpin,
635	.vmap	= drm_gem_vram_object_vmap,
636	.vunmap	= drm_gem_vram_object_vunmap
637};