1/*
  2 * Copyright 2012 Red Hat Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the
  6 * "Software"), to deal in the Software without restriction, including
  7 * without limitation the rights to use, copy, modify, merge, publish,
  8 * distribute, sub license, and/or sell copies of the Software, and to
  9 * permit persons to whom the Software is furnished to do so, subject to
 10 * the following conditions:
 11 *
 12 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 13 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 14 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 15 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 16 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 17 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 18 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 19 *
 20 * The above copyright notice and this permission notice (including the
 21 * next paragraph) shall be included in all copies or substantial portions
 22 * of the Software.
 23 *
 24 */
 25/*
 26 * Authors: Dave Airlie <airlied@redhat.com>
 27 */
 28#include "drmP.h"
 29#include "cirrus_drv.h"
 30#include <ttm/ttm_page_alloc.h>
 31
 32static inline struct cirrus_device *
 33cirrus_bdev(struct ttm_bo_device *bd)
 34{
 35	return container_of(bd, struct cirrus_device, ttm.bdev);
 36}
 37
 38static int
 39cirrus_ttm_mem_global_init(struct drm_global_reference *ref)
 40{
 41	return ttm_mem_global_init(ref->object);
 42}
 43
 44static void
 45cirrus_ttm_mem_global_release(struct drm_global_reference *ref)
 46{
 47	ttm_mem_global_release(ref->object);
 48}
 49
 50static int cirrus_ttm_global_init(struct cirrus_device *cirrus)
 51{
 52	struct drm_global_reference *global_ref;
 53	int r;
 54
 55	global_ref = &cirrus->ttm.mem_global_ref;
 56	global_ref->global_type = DRM_GLOBAL_TTM_MEM;
 57	global_ref->size = sizeof(struct ttm_mem_global);
 58	global_ref->init = &cirrus_ttm_mem_global_init;
 59	global_ref->release = &cirrus_ttm_mem_global_release;
 60	r = drm_global_item_ref(global_ref);
 61	if (r != 0) {
 62		DRM_ERROR("Failed setting up TTM memory accounting "
 63			  "subsystem.\n");
 64		return r;
 65	}
 66
 67	cirrus->ttm.bo_global_ref.mem_glob =
 68		cirrus->ttm.mem_global_ref.object;
 69	global_ref = &cirrus->ttm.bo_global_ref.ref;
 70	global_ref->global_type = DRM_GLOBAL_TTM_BO;
 71	global_ref->size = sizeof(struct ttm_bo_global);
 72	global_ref->init = &ttm_bo_global_init;
 73	global_ref->release = &ttm_bo_global_release;
 74	r = drm_global_item_ref(global_ref);
 75	if (r != 0) {
 76		DRM_ERROR("Failed setting up TTM BO subsystem.\n");
 77		drm_global_item_unref(&cirrus->ttm.mem_global_ref);
 78		return r;
 79	}
 80	return 0;
 81}
 82
 83void
 84cirrus_ttm_global_release(struct cirrus_device *cirrus)
 85{
 86	if (cirrus->ttm.mem_global_ref.release == NULL)
 87		return;
 88
 89	drm_global_item_unref(&cirrus->ttm.bo_global_ref.ref);
 90	drm_global_item_unref(&cirrus->ttm.mem_global_ref);
 91	cirrus->ttm.mem_global_ref.release = NULL;
 92}
 93
 94
 95static void cirrus_bo_ttm_destroy(struct ttm_buffer_object *tbo)
 96{
 97	struct cirrus_bo *bo;
 98
 99	bo = container_of(tbo, struct cirrus_bo, bo);
100
101	drm_gem_object_release(&bo->gem);
102	kfree(bo);
103}
104
105bool cirrus_ttm_bo_is_cirrus_bo(struct ttm_buffer_object *bo)
106{
107	if (bo->destroy == &cirrus_bo_ttm_destroy)
108		return true;
109	return false;
110}
111
112static int
113cirrus_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
114		     struct ttm_mem_type_manager *man)
115{
116	switch (type) {
117	case TTM_PL_SYSTEM:
118		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
119		man->available_caching = TTM_PL_MASK_CACHING;
120		man->default_caching = TTM_PL_FLAG_CACHED;
121		break;
122	case TTM_PL_VRAM:
123		man->func = &ttm_bo_manager_func;
124		man->flags = TTM_MEMTYPE_FLAG_FIXED |
125			TTM_MEMTYPE_FLAG_MAPPABLE;
126		man->available_caching = TTM_PL_FLAG_UNCACHED |
127			TTM_PL_FLAG_WC;
128		man->default_caching = TTM_PL_FLAG_WC;
129		break;
130	default:
131		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
132		return -EINVAL;
133	}
134	return 0;
135}
136
137static void
138cirrus_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
139{
140	struct cirrus_bo *cirrusbo = cirrus_bo(bo);
141
142	if (!cirrus_ttm_bo_is_cirrus_bo(bo))
143		return;
144
145	cirrus_ttm_placement(cirrusbo, TTM_PL_FLAG_SYSTEM);
146	*pl = cirrusbo->placement;
147}
148
149static int cirrus_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
150{
151	return 0;
152}
153
154static int cirrus_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
155				  struct ttm_mem_reg *mem)
156{
157	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
158	struct cirrus_device *cirrus = cirrus_bdev(bdev);
159
160	mem->bus.addr = NULL;
161	mem->bus.offset = 0;
162	mem->bus.size = mem->num_pages << PAGE_SHIFT;
163	mem->bus.base = 0;
164	mem->bus.is_iomem = false;
165	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
166		return -EINVAL;
167	switch (mem->mem_type) {
168	case TTM_PL_SYSTEM:
169		/* system memory */
170		return 0;
171	case TTM_PL_VRAM:
172		mem->bus.offset = mem->start << PAGE_SHIFT;
173		mem->bus.base = pci_resource_start(cirrus->dev->pdev, 0);
174		mem->bus.is_iomem = true;
175		break;
176	default:
177		return -EINVAL;
178		break;
179	}
180	return 0;
181}
182
183static void cirrus_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
184{
185}
186
187static int cirrus_bo_move(struct ttm_buffer_object *bo,
188		       bool evict, bool interruptible,
189		       bool no_wait_reserve, bool no_wait_gpu,
190		       struct ttm_mem_reg *new_mem)
191{
192	int r;
193	r = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
194	return r;
195}
196
197
198static void cirrus_ttm_backend_destroy(struct ttm_tt *tt)
199{
200	ttm_tt_fini(tt);
201	kfree(tt);
202}
203
204static struct ttm_backend_func cirrus_tt_backend_func = {
205	.destroy = &cirrus_ttm_backend_destroy,
206};
207
208
209struct ttm_tt *cirrus_ttm_tt_create(struct ttm_bo_device *bdev,
210				 unsigned long size, uint32_t page_flags,
211				 struct page *dummy_read_page)
212{
213	struct ttm_tt *tt;
214
215	tt = kzalloc(sizeof(struct ttm_tt), GFP_KERNEL);
216	if (tt == NULL)
217		return NULL;
218	tt->func = &cirrus_tt_backend_func;
219	if (ttm_tt_init(tt, bdev, size, page_flags, dummy_read_page)) {
220		kfree(tt);
221		return NULL;
222	}
223	return tt;
224}
225
226static int cirrus_ttm_tt_populate(struct ttm_tt *ttm)
227{
228	return ttm_pool_populate(ttm);
229}
230
231static void cirrus_ttm_tt_unpopulate(struct ttm_tt *ttm)
232{
233	ttm_pool_unpopulate(ttm);
234}
235
236struct ttm_bo_driver cirrus_bo_driver = {
237	.ttm_tt_create = cirrus_ttm_tt_create,
238	.ttm_tt_populate = cirrus_ttm_tt_populate,
239	.ttm_tt_unpopulate = cirrus_ttm_tt_unpopulate,
240	.init_mem_type = cirrus_bo_init_mem_type,
241	.evict_flags = cirrus_bo_evict_flags,
242	.move = cirrus_bo_move,
243	.verify_access = cirrus_bo_verify_access,
244	.io_mem_reserve = &cirrus_ttm_io_mem_reserve,
245	.io_mem_free = &cirrus_ttm_io_mem_free,
246};
247
248int cirrus_mm_init(struct cirrus_device *cirrus)
249{
250	int ret;
251	struct drm_device *dev = cirrus->dev;
252	struct ttm_bo_device *bdev = &cirrus->ttm.bdev;
253
254	ret = cirrus_ttm_global_init(cirrus);
255	if (ret)
256		return ret;
257
258	ret = ttm_bo_device_init(&cirrus->ttm.bdev,
259				 cirrus->ttm.bo_global_ref.ref.object,
260				 &cirrus_bo_driver, DRM_FILE_PAGE_OFFSET,
261				 true);
262	if (ret) {
263		DRM_ERROR("Error initialising bo driver; %d\n", ret);
264		return ret;
265	}
266
267	ret = ttm_bo_init_mm(bdev, TTM_PL_VRAM,
268			     cirrus->mc.vram_size >> PAGE_SHIFT);
269	if (ret) {
270		DRM_ERROR("Failed ttm VRAM init: %d\n", ret);
271		return ret;
272	}
273
274	cirrus->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 0),
275				    pci_resource_len(dev->pdev, 0),
276				    DRM_MTRR_WC);
277
278	cirrus->mm_inited = true;
279	return 0;
280}
281
282void cirrus_mm_fini(struct cirrus_device *cirrus)
283{
284	struct drm_device *dev = cirrus->dev;
285
286	if (!cirrus->mm_inited)
287		return;
288
289	ttm_bo_device_release(&cirrus->ttm.bdev);
290
291	cirrus_ttm_global_release(cirrus);
292
293	if (cirrus->fb_mtrr >= 0) {
294		drm_mtrr_del(cirrus->fb_mtrr,
295			     pci_resource_start(dev->pdev, 0),
296			     pci_resource_len(dev->pdev, 0), DRM_MTRR_WC);
297		cirrus->fb_mtrr = -1;
298	}
299}
300
301void cirrus_ttm_placement(struct cirrus_bo *bo, int domain)
302{
303	u32 c = 0;
304	bo->placement.fpfn = 0;
305	bo->placement.lpfn = 0;
306	bo->placement.placement = bo->placements;
307	bo->placement.busy_placement = bo->placements;
308	if (domain & TTM_PL_FLAG_VRAM)
309		bo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM;
310	if (domain & TTM_PL_FLAG_SYSTEM)
311		bo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
312	if (!c)
313		bo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
314	bo->placement.num_placement = c;
315	bo->placement.num_busy_placement = c;
316}
317
318int cirrus_bo_reserve(struct cirrus_bo *bo, bool no_wait)
319{
320	int ret;
321
322	ret = ttm_bo_reserve(&bo->bo, true, no_wait, false, 0);
323	if (ret) {
324		if (ret != -ERESTARTSYS)
325			DRM_ERROR("reserve failed %p\n", bo);
326		return ret;
327	}
328	return 0;
329}
330
331void cirrus_bo_unreserve(struct cirrus_bo *bo)
332{
333	ttm_bo_unreserve(&bo->bo);
334}
335
336int cirrus_bo_create(struct drm_device *dev, int size, int align,
337		  uint32_t flags, struct cirrus_bo **pcirrusbo)
338{
339	struct cirrus_device *cirrus = dev->dev_private;
340	struct cirrus_bo *cirrusbo;
341	size_t acc_size;
342	int ret;
343
344	cirrusbo = kzalloc(sizeof(struct cirrus_bo), GFP_KERNEL);
345	if (!cirrusbo)
346		return -ENOMEM;
347
348	ret = drm_gem_object_init(dev, &cirrusbo->gem, size);
349	if (ret) {
350		kfree(cirrusbo);
351		return ret;
352	}
353
354	cirrusbo->gem.driver_private = NULL;
355	cirrusbo->bo.bdev = &cirrus->ttm.bdev;
356
357	cirrus_ttm_placement(cirrusbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
358
359	acc_size = ttm_bo_dma_acc_size(&cirrus->ttm.bdev, size,
360				       sizeof(struct cirrus_bo));
361
362	ret = ttm_bo_init(&cirrus->ttm.bdev, &cirrusbo->bo, size,
363			  ttm_bo_type_device, &cirrusbo->placement,
364			  align >> PAGE_SHIFT, 0, false, NULL, acc_size,
365			  NULL, cirrus_bo_ttm_destroy);
366	if (ret)
367		return ret;
368
369	*pcirrusbo = cirrusbo;
370	return 0;
371}
372
373static inline u64 cirrus_bo_gpu_offset(struct cirrus_bo *bo)
374{
375	return bo->bo.offset;
376}
377
378int cirrus_bo_pin(struct cirrus_bo *bo, u32 pl_flag, u64 *gpu_addr)
379{
380	int i, ret;
381
382	if (bo->pin_count) {
383		bo->pin_count++;
384		if (gpu_addr)
385			*gpu_addr = cirrus_bo_gpu_offset(bo);
386	}
387
388	cirrus_ttm_placement(bo, pl_flag);
389	for (i = 0; i < bo->placement.num_placement; i++)
390		bo->placements[i] |= TTM_PL_FLAG_NO_EVICT;
391	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false, false);
392	if (ret)
393		return ret;
394
395	bo->pin_count = 1;
396	if (gpu_addr)
397		*gpu_addr = cirrus_bo_gpu_offset(bo);
398	return 0;
399}
400
401int cirrus_bo_unpin(struct cirrus_bo *bo)
402{
403	int i, ret;
404	if (!bo->pin_count) {
405		DRM_ERROR("unpin bad %p\n", bo);
406		return 0;
407	}
408	bo->pin_count--;
409	if (bo->pin_count)
410		return 0;
411
412	for (i = 0; i < bo->placement.num_placement ; i++)
413		bo->placements[i] &= ~TTM_PL_FLAG_NO_EVICT;
414	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false, false);
415	if (ret)
416		return ret;
417
418	return 0;
419}
420
421int cirrus_bo_push_sysram(struct cirrus_bo *bo)
422{
423	int i, ret;
424	if (!bo->pin_count) {
425		DRM_ERROR("unpin bad %p\n", bo);
426		return 0;
427	}
428	bo->pin_count--;
429	if (bo->pin_count)
430		return 0;
431
432	if (bo->kmap.virtual)
433		ttm_bo_kunmap(&bo->kmap);
434
435	cirrus_ttm_placement(bo, TTM_PL_FLAG_SYSTEM);
436	for (i = 0; i < bo->placement.num_placement ; i++)
437		bo->placements[i] |= TTM_PL_FLAG_NO_EVICT;
438
439	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false, false);
440	if (ret) {
441		DRM_ERROR("pushing to VRAM failed\n");
442		return ret;
443	}
444	return 0;
445}
446
447int cirrus_mmap(struct file *filp, struct vm_area_struct *vma)
448{
449	struct drm_file *file_priv;
450	struct cirrus_device *cirrus;
451
452	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
453		return drm_mmap(filp, vma);
454
455	file_priv = filp->private_data;
456	cirrus = file_priv->minor->dev->dev_private;
457	return ttm_bo_mmap(filp, vma, &cirrus->ttm.bdev);
458}