Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * DMABUF System heap exporter
4 *
5 * Copyright (C) 2011 Google, Inc.
6 * Copyright (C) 2019, 2020 Linaro Ltd.
7 *
8 * Portions based off of Andrew Davis' SRAM heap:
9 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
10 * Andrew F. Davis <afd@ti.com>
11 */
12
13#include <linux/dma-buf.h>
14#include <linux/dma-mapping.h>
15#include <linux/dma-heap.h>
16#include <linux/err.h>
17#include <linux/highmem.h>
18#include <linux/mm.h>
19#include <linux/module.h>
20#include <linux/scatterlist.h>
21#include <linux/slab.h>
22#include <linux/vmalloc.h>
23
24static struct dma_heap *sys_heap;
25
26struct system_heap_buffer {
27 struct dma_heap *heap;
28 struct list_head attachments;
29 struct mutex lock;
30 unsigned long len;
31 struct sg_table sg_table;
32 int vmap_cnt;
33 void *vaddr;
34};
35
36struct dma_heap_attachment {
37 struct device *dev;
38 struct sg_table *table;
39 struct list_head list;
40 bool mapped;
41};
42
43#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO)
44#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
45 | __GFP_NORETRY) & ~__GFP_RECLAIM) \
46 | __GFP_COMP)
47static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP};
48/*
49 * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
50 * to match with the sizes often found in IOMMUs. Using order 4 pages instead
51 * of order 0 pages can significantly improve the performance of many IOMMUs
52 * by reducing TLB pressure and time spent updating page tables.
53 */
54static const unsigned int orders[] = {8, 4, 0};
55#define NUM_ORDERS ARRAY_SIZE(orders)
56
57static struct sg_table *dup_sg_table(struct sg_table *table)
58{
59 struct sg_table *new_table;
60 int ret, i;
61 struct scatterlist *sg, *new_sg;
62
63 new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
64 if (!new_table)
65 return ERR_PTR(-ENOMEM);
66
67 ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
68 if (ret) {
69 kfree(new_table);
70 return ERR_PTR(-ENOMEM);
71 }
72
73 new_sg = new_table->sgl;
74 for_each_sgtable_sg(table, sg, i) {
75 sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
76 new_sg = sg_next(new_sg);
77 }
78
79 return new_table;
80}
81
82static int system_heap_attach(struct dma_buf *dmabuf,
83 struct dma_buf_attachment *attachment)
84{
85 struct system_heap_buffer *buffer = dmabuf->priv;
86 struct dma_heap_attachment *a;
87 struct sg_table *table;
88
89 a = kzalloc(sizeof(*a), GFP_KERNEL);
90 if (!a)
91 return -ENOMEM;
92
93 table = dup_sg_table(&buffer->sg_table);
94 if (IS_ERR(table)) {
95 kfree(a);
96 return -ENOMEM;
97 }
98
99 a->table = table;
100 a->dev = attachment->dev;
101 INIT_LIST_HEAD(&a->list);
102 a->mapped = false;
103
104 attachment->priv = a;
105
106 mutex_lock(&buffer->lock);
107 list_add(&a->list, &buffer->attachments);
108 mutex_unlock(&buffer->lock);
109
110 return 0;
111}
112
113static void system_heap_detach(struct dma_buf *dmabuf,
114 struct dma_buf_attachment *attachment)
115{
116 struct system_heap_buffer *buffer = dmabuf->priv;
117 struct dma_heap_attachment *a = attachment->priv;
118
119 mutex_lock(&buffer->lock);
120 list_del(&a->list);
121 mutex_unlock(&buffer->lock);
122
123 sg_free_table(a->table);
124 kfree(a->table);
125 kfree(a);
126}
127
128static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
129 enum dma_data_direction direction)
130{
131 struct dma_heap_attachment *a = attachment->priv;
132 struct sg_table *table = a->table;
133 int ret;
134
135 ret = dma_map_sgtable(attachment->dev, table, direction, 0);
136 if (ret)
137 return ERR_PTR(ret);
138
139 a->mapped = true;
140 return table;
141}
142
143static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
144 struct sg_table *table,
145 enum dma_data_direction direction)
146{
147 struct dma_heap_attachment *a = attachment->priv;
148
149 a->mapped = false;
150 dma_unmap_sgtable(attachment->dev, table, direction, 0);
151}
152
153static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
154 enum dma_data_direction direction)
155{
156 struct system_heap_buffer *buffer = dmabuf->priv;
157 struct dma_heap_attachment *a;
158
159 mutex_lock(&buffer->lock);
160
161 if (buffer->vmap_cnt)
162 invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
163
164 list_for_each_entry(a, &buffer->attachments, list) {
165 if (!a->mapped)
166 continue;
167 dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
168 }
169 mutex_unlock(&buffer->lock);
170
171 return 0;
172}
173
174static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
175 enum dma_data_direction direction)
176{
177 struct system_heap_buffer *buffer = dmabuf->priv;
178 struct dma_heap_attachment *a;
179
180 mutex_lock(&buffer->lock);
181
182 if (buffer->vmap_cnt)
183 flush_kernel_vmap_range(buffer->vaddr, buffer->len);
184
185 list_for_each_entry(a, &buffer->attachments, list) {
186 if (!a->mapped)
187 continue;
188 dma_sync_sgtable_for_device(a->dev, a->table, direction);
189 }
190 mutex_unlock(&buffer->lock);
191
192 return 0;
193}
194
195static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
196{
197 struct system_heap_buffer *buffer = dmabuf->priv;
198 struct sg_table *table = &buffer->sg_table;
199 unsigned long addr = vma->vm_start;
200 struct sg_page_iter piter;
201 int ret;
202
203 for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
204 struct page *page = sg_page_iter_page(&piter);
205
206 ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
207 vma->vm_page_prot);
208 if (ret)
209 return ret;
210 addr += PAGE_SIZE;
211 if (addr >= vma->vm_end)
212 return 0;
213 }
214 return 0;
215}
216
217static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
218{
219 struct sg_table *table = &buffer->sg_table;
220 int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
221 struct page **pages = vmalloc(sizeof(struct page *) * npages);
222 struct page **tmp = pages;
223 struct sg_page_iter piter;
224 void *vaddr;
225
226 if (!pages)
227 return ERR_PTR(-ENOMEM);
228
229 for_each_sgtable_page(table, &piter, 0) {
230 WARN_ON(tmp - pages >= npages);
231 *tmp++ = sg_page_iter_page(&piter);
232 }
233
234 vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
235 vfree(pages);
236
237 if (!vaddr)
238 return ERR_PTR(-ENOMEM);
239
240 return vaddr;
241}
242
243static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
244{
245 struct system_heap_buffer *buffer = dmabuf->priv;
246 void *vaddr;
247 int ret = 0;
248
249 mutex_lock(&buffer->lock);
250 if (buffer->vmap_cnt) {
251 buffer->vmap_cnt++;
252 iosys_map_set_vaddr(map, buffer->vaddr);
253 goto out;
254 }
255
256 vaddr = system_heap_do_vmap(buffer);
257 if (IS_ERR(vaddr)) {
258 ret = PTR_ERR(vaddr);
259 goto out;
260 }
261
262 buffer->vaddr = vaddr;
263 buffer->vmap_cnt++;
264 iosys_map_set_vaddr(map, buffer->vaddr);
265out:
266 mutex_unlock(&buffer->lock);
267
268 return ret;
269}
270
271static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
272{
273 struct system_heap_buffer *buffer = dmabuf->priv;
274
275 mutex_lock(&buffer->lock);
276 if (!--buffer->vmap_cnt) {
277 vunmap(buffer->vaddr);
278 buffer->vaddr = NULL;
279 }
280 mutex_unlock(&buffer->lock);
281 iosys_map_clear(map);
282}
283
284static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
285{
286 struct system_heap_buffer *buffer = dmabuf->priv;
287 struct sg_table *table;
288 struct scatterlist *sg;
289 int i;
290
291 table = &buffer->sg_table;
292 for_each_sgtable_sg(table, sg, i) {
293 struct page *page = sg_page(sg);
294
295 __free_pages(page, compound_order(page));
296 }
297 sg_free_table(table);
298 kfree(buffer);
299}
300
301static const struct dma_buf_ops system_heap_buf_ops = {
302 .attach = system_heap_attach,
303 .detach = system_heap_detach,
304 .map_dma_buf = system_heap_map_dma_buf,
305 .unmap_dma_buf = system_heap_unmap_dma_buf,
306 .begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
307 .end_cpu_access = system_heap_dma_buf_end_cpu_access,
308 .mmap = system_heap_mmap,
309 .vmap = system_heap_vmap,
310 .vunmap = system_heap_vunmap,
311 .release = system_heap_dma_buf_release,
312};
313
314static struct page *alloc_largest_available(unsigned long size,
315 unsigned int max_order)
316{
317 struct page *page;
318 int i;
319
320 for (i = 0; i < NUM_ORDERS; i++) {
321 if (size < (PAGE_SIZE << orders[i]))
322 continue;
323 if (max_order < orders[i])
324 continue;
325
326 page = alloc_pages(order_flags[i], orders[i]);
327 if (!page)
328 continue;
329 return page;
330 }
331 return NULL;
332}
333
334static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
335 unsigned long len,
336 unsigned long fd_flags,
337 unsigned long heap_flags)
338{
339 struct system_heap_buffer *buffer;
340 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
341 unsigned long size_remaining = len;
342 unsigned int max_order = orders[0];
343 struct dma_buf *dmabuf;
344 struct sg_table *table;
345 struct scatterlist *sg;
346 struct list_head pages;
347 struct page *page, *tmp_page;
348 int i, ret = -ENOMEM;
349
350 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
351 if (!buffer)
352 return ERR_PTR(-ENOMEM);
353
354 INIT_LIST_HEAD(&buffer->attachments);
355 mutex_init(&buffer->lock);
356 buffer->heap = heap;
357 buffer->len = len;
358
359 INIT_LIST_HEAD(&pages);
360 i = 0;
361 while (size_remaining > 0) {
362 /*
363 * Avoid trying to allocate memory if the process
364 * has been killed by SIGKILL
365 */
366 if (fatal_signal_pending(current)) {
367 ret = -EINTR;
368 goto free_buffer;
369 }
370
371 page = alloc_largest_available(size_remaining, max_order);
372 if (!page)
373 goto free_buffer;
374
375 list_add_tail(&page->lru, &pages);
376 size_remaining -= page_size(page);
377 max_order = compound_order(page);
378 i++;
379 }
380
381 table = &buffer->sg_table;
382 if (sg_alloc_table(table, i, GFP_KERNEL))
383 goto free_buffer;
384
385 sg = table->sgl;
386 list_for_each_entry_safe(page, tmp_page, &pages, lru) {
387 sg_set_page(sg, page, page_size(page), 0);
388 sg = sg_next(sg);
389 list_del(&page->lru);
390 }
391
392 /* create the dmabuf */
393 exp_info.exp_name = dma_heap_get_name(heap);
394 exp_info.ops = &system_heap_buf_ops;
395 exp_info.size = buffer->len;
396 exp_info.flags = fd_flags;
397 exp_info.priv = buffer;
398 dmabuf = dma_buf_export(&exp_info);
399 if (IS_ERR(dmabuf)) {
400 ret = PTR_ERR(dmabuf);
401 goto free_pages;
402 }
403 return dmabuf;
404
405free_pages:
406 for_each_sgtable_sg(table, sg, i) {
407 struct page *p = sg_page(sg);
408
409 __free_pages(p, compound_order(p));
410 }
411 sg_free_table(table);
412free_buffer:
413 list_for_each_entry_safe(page, tmp_page, &pages, lru)
414 __free_pages(page, compound_order(page));
415 kfree(buffer);
416
417 return ERR_PTR(ret);
418}
419
420static const struct dma_heap_ops system_heap_ops = {
421 .allocate = system_heap_allocate,
422};
423
424static int system_heap_create(void)
425{
426 struct dma_heap_export_info exp_info;
427
428 exp_info.name = "system";
429 exp_info.ops = &system_heap_ops;
430 exp_info.priv = NULL;
431
432 sys_heap = dma_heap_add(&exp_info);
433 if (IS_ERR(sys_heap))
434 return PTR_ERR(sys_heap);
435
436 return 0;
437}
438module_init(system_heap_create);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * DMABUF System heap exporter
4 *
5 * Copyright (C) 2011 Google, Inc.
6 * Copyright (C) 2019, 2020 Linaro Ltd.
7 *
8 * Portions based off of Andrew Davis' SRAM heap:
9 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
10 * Andrew F. Davis <afd@ti.com>
11 */
12
13#include <linux/dma-buf.h>
14#include <linux/dma-mapping.h>
15#include <linux/dma-heap.h>
16#include <linux/dma-resv.h>
17#include <linux/err.h>
18#include <linux/highmem.h>
19#include <linux/mm.h>
20#include <linux/module.h>
21#include <linux/scatterlist.h>
22#include <linux/slab.h>
23#include <linux/vmalloc.h>
24
25static struct dma_heap *sys_heap;
26
27struct system_heap_buffer {
28 struct dma_heap *heap;
29 struct list_head attachments;
30 struct mutex lock;
31 unsigned long len;
32 struct sg_table sg_table;
33 int vmap_cnt;
34 void *vaddr;
35};
36
37struct dma_heap_attachment {
38 struct device *dev;
39 struct sg_table *table;
40 struct list_head list;
41 bool mapped;
42};
43
44#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
45#define MID_ORDER_GFP (LOW_ORDER_GFP | __GFP_NOWARN)
46#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
47 | __GFP_NORETRY) & ~__GFP_RECLAIM) \
48 | __GFP_COMP)
49static gfp_t order_flags[] = {HIGH_ORDER_GFP, MID_ORDER_GFP, LOW_ORDER_GFP};
50/*
51 * The selection of the orders used for allocation (1MB, 64K, 4K) is designed
52 * to match with the sizes often found in IOMMUs. Using order 4 pages instead
53 * of order 0 pages can significantly improve the performance of many IOMMUs
54 * by reducing TLB pressure and time spent updating page tables.
55 */
56static const unsigned int orders[] = {8, 4, 0};
57#define NUM_ORDERS ARRAY_SIZE(orders)
58
59static struct sg_table *dup_sg_table(struct sg_table *table)
60{
61 struct sg_table *new_table;
62 int ret, i;
63 struct scatterlist *sg, *new_sg;
64
65 new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
66 if (!new_table)
67 return ERR_PTR(-ENOMEM);
68
69 ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
70 if (ret) {
71 kfree(new_table);
72 return ERR_PTR(-ENOMEM);
73 }
74
75 new_sg = new_table->sgl;
76 for_each_sgtable_sg(table, sg, i) {
77 sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
78 new_sg = sg_next(new_sg);
79 }
80
81 return new_table;
82}
83
84static int system_heap_attach(struct dma_buf *dmabuf,
85 struct dma_buf_attachment *attachment)
86{
87 struct system_heap_buffer *buffer = dmabuf->priv;
88 struct dma_heap_attachment *a;
89 struct sg_table *table;
90
91 a = kzalloc(sizeof(*a), GFP_KERNEL);
92 if (!a)
93 return -ENOMEM;
94
95 table = dup_sg_table(&buffer->sg_table);
96 if (IS_ERR(table)) {
97 kfree(a);
98 return -ENOMEM;
99 }
100
101 a->table = table;
102 a->dev = attachment->dev;
103 INIT_LIST_HEAD(&a->list);
104 a->mapped = false;
105
106 attachment->priv = a;
107
108 mutex_lock(&buffer->lock);
109 list_add(&a->list, &buffer->attachments);
110 mutex_unlock(&buffer->lock);
111
112 return 0;
113}
114
115static void system_heap_detach(struct dma_buf *dmabuf,
116 struct dma_buf_attachment *attachment)
117{
118 struct system_heap_buffer *buffer = dmabuf->priv;
119 struct dma_heap_attachment *a = attachment->priv;
120
121 mutex_lock(&buffer->lock);
122 list_del(&a->list);
123 mutex_unlock(&buffer->lock);
124
125 sg_free_table(a->table);
126 kfree(a->table);
127 kfree(a);
128}
129
130static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
131 enum dma_data_direction direction)
132{
133 struct dma_heap_attachment *a = attachment->priv;
134 struct sg_table *table = a->table;
135 int ret;
136
137 ret = dma_map_sgtable(attachment->dev, table, direction, 0);
138 if (ret)
139 return ERR_PTR(ret);
140
141 a->mapped = true;
142 return table;
143}
144
145static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
146 struct sg_table *table,
147 enum dma_data_direction direction)
148{
149 struct dma_heap_attachment *a = attachment->priv;
150
151 a->mapped = false;
152 dma_unmap_sgtable(attachment->dev, table, direction, 0);
153}
154
155static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
156 enum dma_data_direction direction)
157{
158 struct system_heap_buffer *buffer = dmabuf->priv;
159 struct dma_heap_attachment *a;
160
161 mutex_lock(&buffer->lock);
162
163 if (buffer->vmap_cnt)
164 invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
165
166 list_for_each_entry(a, &buffer->attachments, list) {
167 if (!a->mapped)
168 continue;
169 dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
170 }
171 mutex_unlock(&buffer->lock);
172
173 return 0;
174}
175
176static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
177 enum dma_data_direction direction)
178{
179 struct system_heap_buffer *buffer = dmabuf->priv;
180 struct dma_heap_attachment *a;
181
182 mutex_lock(&buffer->lock);
183
184 if (buffer->vmap_cnt)
185 flush_kernel_vmap_range(buffer->vaddr, buffer->len);
186
187 list_for_each_entry(a, &buffer->attachments, list) {
188 if (!a->mapped)
189 continue;
190 dma_sync_sgtable_for_device(a->dev, a->table, direction);
191 }
192 mutex_unlock(&buffer->lock);
193
194 return 0;
195}
196
197static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
198{
199 struct system_heap_buffer *buffer = dmabuf->priv;
200 struct sg_table *table = &buffer->sg_table;
201 unsigned long addr = vma->vm_start;
202 struct sg_page_iter piter;
203 int ret;
204
205 dma_resv_assert_held(dmabuf->resv);
206
207 for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
208 struct page *page = sg_page_iter_page(&piter);
209
210 ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
211 vma->vm_page_prot);
212 if (ret)
213 return ret;
214 addr += PAGE_SIZE;
215 if (addr >= vma->vm_end)
216 return 0;
217 }
218 return 0;
219}
220
221static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
222{
223 struct sg_table *table = &buffer->sg_table;
224 int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
225 struct page **pages = vmalloc(sizeof(struct page *) * npages);
226 struct page **tmp = pages;
227 struct sg_page_iter piter;
228 void *vaddr;
229
230 if (!pages)
231 return ERR_PTR(-ENOMEM);
232
233 for_each_sgtable_page(table, &piter, 0) {
234 WARN_ON(tmp - pages >= npages);
235 *tmp++ = sg_page_iter_page(&piter);
236 }
237
238 vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
239 vfree(pages);
240
241 if (!vaddr)
242 return ERR_PTR(-ENOMEM);
243
244 return vaddr;
245}
246
247static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
248{
249 struct system_heap_buffer *buffer = dmabuf->priv;
250 void *vaddr;
251 int ret = 0;
252
253 mutex_lock(&buffer->lock);
254 if (buffer->vmap_cnt) {
255 buffer->vmap_cnt++;
256 iosys_map_set_vaddr(map, buffer->vaddr);
257 goto out;
258 }
259
260 vaddr = system_heap_do_vmap(buffer);
261 if (IS_ERR(vaddr)) {
262 ret = PTR_ERR(vaddr);
263 goto out;
264 }
265
266 buffer->vaddr = vaddr;
267 buffer->vmap_cnt++;
268 iosys_map_set_vaddr(map, buffer->vaddr);
269out:
270 mutex_unlock(&buffer->lock);
271
272 return ret;
273}
274
275static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
276{
277 struct system_heap_buffer *buffer = dmabuf->priv;
278
279 mutex_lock(&buffer->lock);
280 if (!--buffer->vmap_cnt) {
281 vunmap(buffer->vaddr);
282 buffer->vaddr = NULL;
283 }
284 mutex_unlock(&buffer->lock);
285 iosys_map_clear(map);
286}
287
288static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
289{
290 struct system_heap_buffer *buffer = dmabuf->priv;
291 struct sg_table *table;
292 struct scatterlist *sg;
293 int i;
294
295 table = &buffer->sg_table;
296 for_each_sgtable_sg(table, sg, i) {
297 struct page *page = sg_page(sg);
298
299 __free_pages(page, compound_order(page));
300 }
301 sg_free_table(table);
302 kfree(buffer);
303}
304
305static const struct dma_buf_ops system_heap_buf_ops = {
306 .attach = system_heap_attach,
307 .detach = system_heap_detach,
308 .map_dma_buf = system_heap_map_dma_buf,
309 .unmap_dma_buf = system_heap_unmap_dma_buf,
310 .begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
311 .end_cpu_access = system_heap_dma_buf_end_cpu_access,
312 .mmap = system_heap_mmap,
313 .vmap = system_heap_vmap,
314 .vunmap = system_heap_vunmap,
315 .release = system_heap_dma_buf_release,
316};
317
318static struct page *alloc_largest_available(unsigned long size,
319 unsigned int max_order)
320{
321 struct page *page;
322 int i;
323
324 for (i = 0; i < NUM_ORDERS; i++) {
325 if (size < (PAGE_SIZE << orders[i]))
326 continue;
327 if (max_order < orders[i])
328 continue;
329
330 page = alloc_pages(order_flags[i], orders[i]);
331 if (!page)
332 continue;
333 return page;
334 }
335 return NULL;
336}
337
338static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
339 unsigned long len,
340 unsigned long fd_flags,
341 unsigned long heap_flags)
342{
343 struct system_heap_buffer *buffer;
344 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
345 unsigned long size_remaining = len;
346 unsigned int max_order = orders[0];
347 struct dma_buf *dmabuf;
348 struct sg_table *table;
349 struct scatterlist *sg;
350 struct list_head pages;
351 struct page *page, *tmp_page;
352 int i, ret = -ENOMEM;
353
354 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
355 if (!buffer)
356 return ERR_PTR(-ENOMEM);
357
358 INIT_LIST_HEAD(&buffer->attachments);
359 mutex_init(&buffer->lock);
360 buffer->heap = heap;
361 buffer->len = len;
362
363 INIT_LIST_HEAD(&pages);
364 i = 0;
365 while (size_remaining > 0) {
366 /*
367 * Avoid trying to allocate memory if the process
368 * has been killed by SIGKILL
369 */
370 if (fatal_signal_pending(current)) {
371 ret = -EINTR;
372 goto free_buffer;
373 }
374
375 page = alloc_largest_available(size_remaining, max_order);
376 if (!page)
377 goto free_buffer;
378
379 list_add_tail(&page->lru, &pages);
380 size_remaining -= page_size(page);
381 max_order = compound_order(page);
382 i++;
383 }
384
385 table = &buffer->sg_table;
386 if (sg_alloc_table(table, i, GFP_KERNEL))
387 goto free_buffer;
388
389 sg = table->sgl;
390 list_for_each_entry_safe(page, tmp_page, &pages, lru) {
391 sg_set_page(sg, page, page_size(page), 0);
392 sg = sg_next(sg);
393 list_del(&page->lru);
394 }
395
396 /* create the dmabuf */
397 exp_info.exp_name = dma_heap_get_name(heap);
398 exp_info.ops = &system_heap_buf_ops;
399 exp_info.size = buffer->len;
400 exp_info.flags = fd_flags;
401 exp_info.priv = buffer;
402 dmabuf = dma_buf_export(&exp_info);
403 if (IS_ERR(dmabuf)) {
404 ret = PTR_ERR(dmabuf);
405 goto free_pages;
406 }
407 return dmabuf;
408
409free_pages:
410 for_each_sgtable_sg(table, sg, i) {
411 struct page *p = sg_page(sg);
412
413 __free_pages(p, compound_order(p));
414 }
415 sg_free_table(table);
416free_buffer:
417 list_for_each_entry_safe(page, tmp_page, &pages, lru)
418 __free_pages(page, compound_order(page));
419 kfree(buffer);
420
421 return ERR_PTR(ret);
422}
423
424static const struct dma_heap_ops system_heap_ops = {
425 .allocate = system_heap_allocate,
426};
427
428static int system_heap_create(void)
429{
430 struct dma_heap_export_info exp_info;
431
432 exp_info.name = "system";
433 exp_info.ops = &system_heap_ops;
434 exp_info.priv = NULL;
435
436 sys_heap = dma_heap_add(&exp_info);
437 if (IS_ERR(sys_heap))
438 return PTR_ERR(sys_heap);
439
440 return 0;
441}
442module_init(system_heap_create);
443MODULE_LICENSE("GPL v2");