1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
  3
  4#include <linux/slab.h>
  5#include <linux/dma-mapping.h>
  6
  7#include "lima_device.h"
  8#include "lima_vm.h"
  9#include "lima_object.h"
 10#include "lima_regs.h"
 11
 12struct lima_bo_va {
 13	struct list_head list;
 14	unsigned int ref_count;
 15
 16	struct drm_mm_node node;
 17
 18	struct lima_vm *vm;
 19};
 20
 21#define LIMA_VM_PD_SHIFT 22
 22#define LIMA_VM_PT_SHIFT 12
 23#define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT)
 24#define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT
 25
 26#define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)
 27#define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)
 28
 29#define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT)
 30#define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT)
 31#define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT)
 32#define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT)
 33
 34
 35static void lima_vm_unmap_page_table(struct lima_vm *vm, u32 start, u32 end)
 36{
 37	u32 addr;
 38
 39	for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
 40		u32 pbe = LIMA_PBE(addr);
 41		u32 bte = LIMA_BTE(addr);
 42
 43		vm->bts[pbe].cpu[bte] = 0;
 44	}
 45}
 46
 47static int lima_vm_map_page_table(struct lima_vm *vm, dma_addr_t *dma,
 48				  u32 start, u32 end)
 49{
 50	u64 addr;
 51	int i = 0;
 52
 53	for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
 54		u32 pbe = LIMA_PBE(addr);
 55		u32 bte = LIMA_BTE(addr);
 56
 57		if (!vm->bts[pbe].cpu) {
 58			dma_addr_t pts;
 59			u32 *pd;
 60			int j;
 61
 62			vm->bts[pbe].cpu = dma_alloc_wc(
 63				vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
 64				&vm->bts[pbe].dma, GFP_KERNEL | __GFP_ZERO);
 65			if (!vm->bts[pbe].cpu) {
 66				if (addr != start)
 67					lima_vm_unmap_page_table(vm, start, addr - 1);
 68				return -ENOMEM;
 69			}
 70
 71			pts = vm->bts[pbe].dma;
 72			pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);
 73			for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
 74				pd[j] = pts | LIMA_VM_FLAG_PRESENT;
 75				pts += LIMA_PAGE_SIZE;
 76			}
 77		}
 78
 79		vm->bts[pbe].cpu[bte] = dma[i++] | LIMA_VM_FLAGS_CACHE;
 80	}
 81
 82	return 0;
 83}
 84
 85static struct lima_bo_va *
 86lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo)
 87{
 88	struct lima_bo_va *bo_va, *ret = NULL;
 89
 90	list_for_each_entry(bo_va, &bo->va, list) {
 91		if (bo_va->vm == vm) {
 92			ret = bo_va;
 93			break;
 94		}
 95	}
 96
 97	return ret;
 98}
 99
100int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create)
101{
102	struct lima_bo_va *bo_va;
103	int err;
104
105	mutex_lock(&bo->lock);
106
107	bo_va = lima_vm_bo_find(vm, bo);
108	if (bo_va) {
109		bo_va->ref_count++;
110		mutex_unlock(&bo->lock);
111		return 0;
112	}
113
114	/* should not create new bo_va if not asked by caller */
115	if (!create) {
116		mutex_unlock(&bo->lock);
117		return -ENOENT;
118	}
119
120	bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL);
121	if (!bo_va) {
122		err = -ENOMEM;
123		goto err_out0;
124	}
125
126	bo_va->vm = vm;
127	bo_va->ref_count = 1;
128
129	mutex_lock(&vm->lock);
130
131	err = drm_mm_insert_node(&vm->mm, &bo_va->node, bo->gem.size);
132	if (err)
133		goto err_out1;
134
135	err = lima_vm_map_page_table(vm, bo->pages_dma_addr, bo_va->node.start,
136				     bo_va->node.start + bo_va->node.size - 1);
137	if (err)
138		goto err_out2;
139
140	mutex_unlock(&vm->lock);
141
142	list_add_tail(&bo_va->list, &bo->va);
143
144	mutex_unlock(&bo->lock);
145	return 0;
146
147err_out2:
148	drm_mm_remove_node(&bo_va->node);
149err_out1:
150	mutex_unlock(&vm->lock);
151	kfree(bo_va);
152err_out0:
153	mutex_unlock(&bo->lock);
154	return err;
155}
156
157void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo)
158{
159	struct lima_bo_va *bo_va;
160
161	mutex_lock(&bo->lock);
162
163	bo_va = lima_vm_bo_find(vm, bo);
164	if (--bo_va->ref_count > 0) {
165		mutex_unlock(&bo->lock);
166		return;
167	}
168
169	mutex_lock(&vm->lock);
170
171	lima_vm_unmap_page_table(vm, bo_va->node.start,
172				 bo_va->node.start + bo_va->node.size - 1);
173
174	drm_mm_remove_node(&bo_va->node);
175
176	mutex_unlock(&vm->lock);
177
178	list_del(&bo_va->list);
179
180	mutex_unlock(&bo->lock);
181
182	kfree(bo_va);
183}
184
185u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo)
186{
187	struct lima_bo_va *bo_va;
188	u32 ret;
189
190	mutex_lock(&bo->lock);
191
192	bo_va = lima_vm_bo_find(vm, bo);
193	ret = bo_va->node.start;
194
195	mutex_unlock(&bo->lock);
196
197	return ret;
198}
199
200struct lima_vm *lima_vm_create(struct lima_device *dev)
201{
202	struct lima_vm *vm;
203
204	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
205	if (!vm)
206		return NULL;
207
208	vm->dev = dev;
209	mutex_init(&vm->lock);
210	kref_init(&vm->refcount);
211
212	vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma,
213				  GFP_KERNEL | __GFP_ZERO);
214	if (!vm->pd.cpu)
215		goto err_out0;
216
217	if (dev->dlbu_cpu) {
218		int err = lima_vm_map_page_table(
219			vm, &dev->dlbu_dma, LIMA_VA_RESERVE_DLBU,
220			LIMA_VA_RESERVE_DLBU + LIMA_PAGE_SIZE - 1);
221		if (err)
222			goto err_out1;
223	}
224
225	drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start);
226
227	return vm;
228
229err_out1:
230	dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
231err_out0:
232	kfree(vm);
233	return NULL;
234}
235
236void lima_vm_release(struct kref *kref)
237{
238	struct lima_vm *vm = container_of(kref, struct lima_vm, refcount);
239	int i;
240
241	drm_mm_takedown(&vm->mm);
242
243	for (i = 0; i < LIMA_VM_NUM_BT; i++) {
244		if (vm->bts[i].cpu)
245			dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
246				    vm->bts[i].cpu, vm->bts[i].dma);
247	}
248
249	if (vm->pd.cpu)
250		dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
251
252	kfree(vm);
253}
254
255void lima_vm_print(struct lima_vm *vm)
256{
257	int i, j, k;
258	u32 *pd, *pt;
259
260	if (!vm->pd.cpu)
261		return;
262
263	pd = vm->pd.cpu;
264	for (i = 0; i < LIMA_VM_NUM_BT; i++) {
265		if (!vm->bts[i].cpu)
266			continue;
267
268		pt = vm->bts[i].cpu;
269		for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
270			int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j;
271
272			printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]);
273
274			for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) {
275				u32 pte = *pt++;
276
277				if (pte)
278					printk(KERN_INFO "  pt %03x:%08x\n", k, pte);
279			}
280		}
281	}
282}