1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * DMABUF System heap exporter
  4 *
  5 * Copyright (C) 2011 Google, Inc.
  6 * Copyright (C) 2019, 2020 Linaro Ltd.
  7 *
  8 * Portions based off of Andrew Davis' SRAM heap:
  9 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
 10 *	Andrew F. Davis <afd@ti.com>
 11 */
 12
 13#include <linux/dma-buf.h>
 14#include <linux/dma-mapping.h>
 15#include <linux/dma-heap.h>
 16#include <linux/dma-resv.h>
 17#include <linux/err.h>
 18#include <linux/highmem.h>
 19#include <linux/mm.h>
 20#include <linux/module.h>
 21#include <linux/scatterlist.h>
 22#include <linux/slab.h>
 23#include <linux/vmalloc.h>
 24
 25static struct dma_heap *sys_heap;
 26
 27struct system_heap_buffer {
 28	struct dma_heap *heap;
 29	struct list_head attachments;
 30	struct mutex lock;
 31	unsigned long len;
 32	struct sg_table sg_table;
 33	int vmap_cnt;
 34	void *vaddr;
 35};
 36
 37struct dma_heap_attachment {
 38	struct device *dev;
 39	struct sg_table *table;
 40	struct list_head list;
 41	bool mapped;
 42};
 43
 44#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
 45#define MID_ORDER_GFP (LOW_ORDER_GFP | __GFP_NOWARN)
 46#define HIGH_ORDER_GFP  (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
 47				| __GFP_NORETRY) & ~__GFP_RECLAIM) \
 48				| __GFP_COMP)
 49static gfp_t order_flags[] = {HIGH_ORDER_GFP, MID_ORDER_GFP, LOW_ORDER_GFP};
 50/*
 51 * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
 52 * to match with the sizes often found in IOMMUs. Using order 4 pages instead
 53 * of order 0 pages can significantly improve the performance of many IOMMUs
 54 * by reducing TLB pressure and time spent updating page tables.
 55 */
 56static const unsigned int orders[] = {8, 4, 0};
 57#define NUM_ORDERS ARRAY_SIZE(orders)
 58
 59static struct sg_table *dup_sg_table(struct sg_table *table)
 60{
 61	struct sg_table *new_table;
 62	int ret, i;
 63	struct scatterlist *sg, *new_sg;
 64
 65	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
 66	if (!new_table)
 67		return ERR_PTR(-ENOMEM);
 68
 69	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
 70	if (ret) {
 71		kfree(new_table);
 72		return ERR_PTR(-ENOMEM);
 73	}
 74
 75	new_sg = new_table->sgl;
 76	for_each_sgtable_sg(table, sg, i) {
 77		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
 78		new_sg = sg_next(new_sg);
 79	}
 80
 81	return new_table;
 82}
 83
 84static int system_heap_attach(struct dma_buf *dmabuf,
 85			      struct dma_buf_attachment *attachment)
 86{
 87	struct system_heap_buffer *buffer = dmabuf->priv;
 88	struct dma_heap_attachment *a;
 89	struct sg_table *table;
 90
 91	a = kzalloc(sizeof(*a), GFP_KERNEL);
 92	if (!a)
 93		return -ENOMEM;
 94
 95	table = dup_sg_table(&buffer->sg_table);
 96	if (IS_ERR(table)) {
 97		kfree(a);
 98		return -ENOMEM;
 99	}
100
101	a->table = table;
102	a->dev = attachment->dev;
103	INIT_LIST_HEAD(&a->list);
104	a->mapped = false;
105
106	attachment->priv = a;
107
108	mutex_lock(&buffer->lock);
109	list_add(&a->list, &buffer->attachments);
110	mutex_unlock(&buffer->lock);
111
112	return 0;
113}
114
115static void system_heap_detach(struct dma_buf *dmabuf,
116			       struct dma_buf_attachment *attachment)
117{
118	struct system_heap_buffer *buffer = dmabuf->priv;
119	struct dma_heap_attachment *a = attachment->priv;
120
121	mutex_lock(&buffer->lock);
122	list_del(&a->list);
123	mutex_unlock(&buffer->lock);
124
125	sg_free_table(a->table);
126	kfree(a->table);
127	kfree(a);
128}
129
130static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
131						enum dma_data_direction direction)
132{
133	struct dma_heap_attachment *a = attachment->priv;
134	struct sg_table *table = a->table;
135	int ret;
136
137	ret = dma_map_sgtable(attachment->dev, table, direction, 0);
138	if (ret)
139		return ERR_PTR(ret);
140
141	a->mapped = true;
142	return table;
143}
144
145static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
146				      struct sg_table *table,
147				      enum dma_data_direction direction)
148{
149	struct dma_heap_attachment *a = attachment->priv;
150
151	a->mapped = false;
152	dma_unmap_sgtable(attachment->dev, table, direction, 0);
153}
154
155static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
156						enum dma_data_direction direction)
157{
158	struct system_heap_buffer *buffer = dmabuf->priv;
159	struct dma_heap_attachment *a;
160
161	mutex_lock(&buffer->lock);
162
163	if (buffer->vmap_cnt)
164		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
165
166	list_for_each_entry(a, &buffer->attachments, list) {
167		if (!a->mapped)
168			continue;
169		dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
170	}
171	mutex_unlock(&buffer->lock);
172
173	return 0;
174}
175
176static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
177					      enum dma_data_direction direction)
178{
179	struct system_heap_buffer *buffer = dmabuf->priv;
180	struct dma_heap_attachment *a;
181
182	mutex_lock(&buffer->lock);
183
184	if (buffer->vmap_cnt)
185		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
186
187	list_for_each_entry(a, &buffer->attachments, list) {
188		if (!a->mapped)
189			continue;
190		dma_sync_sgtable_for_device(a->dev, a->table, direction);
191	}
192	mutex_unlock(&buffer->lock);
193
194	return 0;
195}
196
197static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
198{
199	struct system_heap_buffer *buffer = dmabuf->priv;
200	struct sg_table *table = &buffer->sg_table;
201	unsigned long addr = vma->vm_start;
202	struct sg_page_iter piter;
203	int ret;
204
205	dma_resv_assert_held(dmabuf->resv);
206
207	for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
208		struct page *page = sg_page_iter_page(&piter);
209
210		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
211				      vma->vm_page_prot);
212		if (ret)
213			return ret;
214		addr += PAGE_SIZE;
215		if (addr >= vma->vm_end)
216			return 0;
217	}
218	return 0;
219}
220
221static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
222{
223	struct sg_table *table = &buffer->sg_table;
224	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
225	struct page **pages = vmalloc(sizeof(struct page *) * npages);
226	struct page **tmp = pages;
227	struct sg_page_iter piter;
228	void *vaddr;
229
230	if (!pages)
231		return ERR_PTR(-ENOMEM);
232
233	for_each_sgtable_page(table, &piter, 0) {
234		WARN_ON(tmp - pages >= npages);
235		*tmp++ = sg_page_iter_page(&piter);
236	}
237
238	vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
239	vfree(pages);
240
241	if (!vaddr)
242		return ERR_PTR(-ENOMEM);
243
244	return vaddr;
245}
246
247static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
248{
249	struct system_heap_buffer *buffer = dmabuf->priv;
250	void *vaddr;
251	int ret = 0;
252
253	mutex_lock(&buffer->lock);
254	if (buffer->vmap_cnt) {
255		buffer->vmap_cnt++;
256		iosys_map_set_vaddr(map, buffer->vaddr);
257		goto out;
258	}
259
260	vaddr = system_heap_do_vmap(buffer);
261	if (IS_ERR(vaddr)) {
262		ret = PTR_ERR(vaddr);
263		goto out;
264	}
265
266	buffer->vaddr = vaddr;
267	buffer->vmap_cnt++;
268	iosys_map_set_vaddr(map, buffer->vaddr);
269out:
270	mutex_unlock(&buffer->lock);
271
272	return ret;
273}
274
275static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
276{
277	struct system_heap_buffer *buffer = dmabuf->priv;
278
279	mutex_lock(&buffer->lock);
280	if (!--buffer->vmap_cnt) {
281		vunmap(buffer->vaddr);
282		buffer->vaddr = NULL;
283	}
284	mutex_unlock(&buffer->lock);
285	iosys_map_clear(map);
286}
287
288static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
289{
290	struct system_heap_buffer *buffer = dmabuf->priv;
291	struct sg_table *table;
292	struct scatterlist *sg;
293	int i;
294
295	table = &buffer->sg_table;
296	for_each_sgtable_sg(table, sg, i) {
297		struct page *page = sg_page(sg);
298
299		__free_pages(page, compound_order(page));
300	}
301	sg_free_table(table);
302	kfree(buffer);
303}
304
305static const struct dma_buf_ops system_heap_buf_ops = {
306	.attach = system_heap_attach,
307	.detach = system_heap_detach,
308	.map_dma_buf = system_heap_map_dma_buf,
309	.unmap_dma_buf = system_heap_unmap_dma_buf,
310	.begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
311	.end_cpu_access = system_heap_dma_buf_end_cpu_access,
312	.mmap = system_heap_mmap,
313	.vmap = system_heap_vmap,
314	.vunmap = system_heap_vunmap,
315	.release = system_heap_dma_buf_release,
316};
317
318static struct page *alloc_largest_available(unsigned long size,
319					    unsigned int max_order)
320{
321	struct page *page;
322	int i;
323
324	for (i = 0; i < NUM_ORDERS; i++) {
325		if (size <  (PAGE_SIZE << orders[i]))
326			continue;
327		if (max_order < orders[i])
328			continue;
329
330		page = alloc_pages(order_flags[i], orders[i]);
331		if (!page)
332			continue;
333		return page;
334	}
335	return NULL;
336}
337
338static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
339					    unsigned long len,
340					    unsigned long fd_flags,
341					    unsigned long heap_flags)
342{
343	struct system_heap_buffer *buffer;
344	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
345	unsigned long size_remaining = len;
346	unsigned int max_order = orders[0];
347	struct dma_buf *dmabuf;
348	struct sg_table *table;
349	struct scatterlist *sg;
350	struct list_head pages;
351	struct page *page, *tmp_page;
352	int i, ret = -ENOMEM;
353
354	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
355	if (!buffer)
356		return ERR_PTR(-ENOMEM);
357
358	INIT_LIST_HEAD(&buffer->attachments);
359	mutex_init(&buffer->lock);
360	buffer->heap = heap;
361	buffer->len = len;
362
363	INIT_LIST_HEAD(&pages);
364	i = 0;
365	while (size_remaining > 0) {
366		/*
367		 * Avoid trying to allocate memory if the process
368		 * has been killed by SIGKILL
369		 */
370		if (fatal_signal_pending(current)) {
371			ret = -EINTR;
372			goto free_buffer;
373		}
374
375		page = alloc_largest_available(size_remaining, max_order);
376		if (!page)
377			goto free_buffer;
378
379		list_add_tail(&page->lru, &pages);
380		size_remaining -= page_size(page);
381		max_order = compound_order(page);
382		i++;
383	}
384
385	table = &buffer->sg_table;
386	if (sg_alloc_table(table, i, GFP_KERNEL))
387		goto free_buffer;
388
389	sg = table->sgl;
390	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
391		sg_set_page(sg, page, page_size(page), 0);
392		sg = sg_next(sg);
393		list_del(&page->lru);
394	}
395
396	/* create the dmabuf */
397	exp_info.exp_name = dma_heap_get_name(heap);
398	exp_info.ops = &system_heap_buf_ops;
399	exp_info.size = buffer->len;
400	exp_info.flags = fd_flags;
401	exp_info.priv = buffer;
402	dmabuf = dma_buf_export(&exp_info);
403	if (IS_ERR(dmabuf)) {
404		ret = PTR_ERR(dmabuf);
405		goto free_pages;
406	}
407	return dmabuf;
408
409free_pages:
410	for_each_sgtable_sg(table, sg, i) {
411		struct page *p = sg_page(sg);
412
413		__free_pages(p, compound_order(p));
414	}
415	sg_free_table(table);
416free_buffer:
417	list_for_each_entry_safe(page, tmp_page, &pages, lru)
418		__free_pages(page, compound_order(page));
419	kfree(buffer);
420
421	return ERR_PTR(ret);
422}
423
424static const struct dma_heap_ops system_heap_ops = {
425	.allocate = system_heap_allocate,
426};
427
428static int system_heap_create(void)
429{
430	struct dma_heap_export_info exp_info;
431
432	exp_info.name = "system";
433	exp_info.ops = &system_heap_ops;
434	exp_info.priv = NULL;
435
436	sys_heap = dma_heap_add(&exp_info);
437	if (IS_ERR(sys_heap))
438		return PTR_ERR(sys_heap);
439
440	return 0;
441}
442module_init(system_heap_create);
443MODULE_LICENSE("GPL v2");