Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3// Copyright (c) 2018, Linaro Limited
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/dma-resv.h>
10#include <linux/idr.h>
11#include <linux/list.h>
12#include <linux/miscdevice.h>
13#include <linux/module.h>
14#include <linux/of_address.h>
15#include <linux/of.h>
16#include <linux/platform_device.h>
17#include <linux/sort.h>
18#include <linux/of_platform.h>
19#include <linux/rpmsg.h>
20#include <linux/scatterlist.h>
21#include <linux/slab.h>
22#include <linux/firmware/qcom/qcom_scm.h>
23#include <uapi/misc/fastrpc.h>
24#include <linux/of_reserved_mem.h>
25
26#define ADSP_DOMAIN_ID (0)
27#define MDSP_DOMAIN_ID (1)
28#define SDSP_DOMAIN_ID (2)
29#define CDSP_DOMAIN_ID (3)
30#define CDSP1_DOMAIN_ID (4)
31#define FASTRPC_DEV_MAX 5 /* adsp, mdsp, slpi, cdsp, cdsp1 */
32#define FASTRPC_MAX_SESSIONS 14
33#define FASTRPC_MAX_VMIDS 16
34#define FASTRPC_ALIGN 128
35#define FASTRPC_MAX_FDLIST 16
36#define FASTRPC_MAX_CRCLIST 64
37#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
38#define FASTRPC_CTX_MAX (256)
39#define FASTRPC_INIT_HANDLE 1
40#define FASTRPC_DSP_UTILITIES_HANDLE 2
41#define FASTRPC_CTXID_MASK (0xFF0)
42#define INIT_FILELEN_MAX (2 * 1024 * 1024)
43#define INIT_FILE_NAMELEN_MAX (128)
44#define FASTRPC_DEVICE_NAME "fastrpc"
45
46/* Add memory to static PD pool, protection thru XPU */
47#define ADSP_MMAP_HEAP_ADDR 4
48/* MAP static DMA buffer on DSP User PD */
49#define ADSP_MMAP_DMA_BUFFER 6
50/* Add memory to static PD pool protection thru hypervisor */
51#define ADSP_MMAP_REMOTE_HEAP_ADDR 8
52/* Add memory to userPD pool, for user heap */
53#define ADSP_MMAP_ADD_PAGES 0x1000
54/* Add memory to userPD pool, for LLC heap */
55#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
56
57#define DSP_UNSUPPORTED_API (0x80000414)
58/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
59#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
60#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
61
62/* Retrives number of input buffers from the scalars parameter */
63#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
64
65/* Retrives number of output buffers from the scalars parameter */
66#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
67
68/* Retrives number of input handles from the scalars parameter */
69#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
70
71/* Retrives number of output handles from the scalars parameter */
72#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
73
74#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
75 REMOTE_SCALARS_OUTBUFS(sc) + \
76 REMOTE_SCALARS_INHANDLES(sc)+ \
77 REMOTE_SCALARS_OUTHANDLES(sc))
78#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
79 (((attr & 0x07) << 29) | \
80 ((method & 0x1f) << 24) | \
81 ((in & 0xff) << 16) | \
82 ((out & 0xff) << 8) | \
83 ((oin & 0x0f) << 4) | \
84 (oout & 0x0f))
85
86#define FASTRPC_SCALARS(method, in, out) \
87 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
88
89#define FASTRPC_CREATE_PROCESS_NARGS 6
90#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3
91/* Remote Method id table */
92#define FASTRPC_RMID_INIT_ATTACH 0
93#define FASTRPC_RMID_INIT_RELEASE 1
94#define FASTRPC_RMID_INIT_MMAP 4
95#define FASTRPC_RMID_INIT_MUNMAP 5
96#define FASTRPC_RMID_INIT_CREATE 6
97#define FASTRPC_RMID_INIT_CREATE_ATTR 7
98#define FASTRPC_RMID_INIT_CREATE_STATIC 8
99#define FASTRPC_RMID_INIT_MEM_MAP 10
100#define FASTRPC_RMID_INIT_MEM_UNMAP 11
101
102/* Protection Domain(PD) ids */
103#define ROOT_PD (0)
104#define USER_PD (1)
105#define SENSORS_PD (2)
106
107#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
108
109static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
110 "sdsp", "cdsp", "cdsp1" };
111struct fastrpc_phy_page {
112 u64 addr; /* physical address */
113 u64 size; /* size of contiguous region */
114};
115
116struct fastrpc_invoke_buf {
117 u32 num; /* number of contiguous regions */
118 u32 pgidx; /* index to start of contiguous region */
119};
120
121struct fastrpc_remote_dmahandle {
122 s32 fd; /* dma handle fd */
123 u32 offset; /* dma handle offset */
124 u32 len; /* dma handle length */
125};
126
127struct fastrpc_remote_buf {
128 u64 pv; /* buffer pointer */
129 u64 len; /* length of buffer */
130};
131
132union fastrpc_remote_arg {
133 struct fastrpc_remote_buf buf;
134 struct fastrpc_remote_dmahandle dma;
135};
136
137struct fastrpc_mmap_rsp_msg {
138 u64 vaddr;
139};
140
141struct fastrpc_mmap_req_msg {
142 s32 pgid;
143 u32 flags;
144 u64 vaddr;
145 s32 num;
146};
147
148struct fastrpc_mem_map_req_msg {
149 s32 pgid;
150 s32 fd;
151 s32 offset;
152 u32 flags;
153 u64 vaddrin;
154 s32 num;
155 s32 data_len;
156};
157
158struct fastrpc_munmap_req_msg {
159 s32 pgid;
160 u64 vaddr;
161 u64 size;
162};
163
164struct fastrpc_mem_unmap_req_msg {
165 s32 pgid;
166 s32 fd;
167 u64 vaddrin;
168 u64 len;
169};
170
171struct fastrpc_msg {
172 int pid; /* process group id */
173 int tid; /* thread id */
174 u64 ctx; /* invoke caller context */
175 u32 handle; /* handle to invoke */
176 u32 sc; /* scalars structure describing the data */
177 u64 addr; /* physical address */
178 u64 size; /* size of contiguous region */
179};
180
181struct fastrpc_invoke_rsp {
182 u64 ctx; /* invoke caller context */
183 int retval; /* invoke return value */
184};
185
186struct fastrpc_buf_overlap {
187 u64 start;
188 u64 end;
189 int raix;
190 u64 mstart;
191 u64 mend;
192 u64 offset;
193};
194
195struct fastrpc_buf {
196 struct fastrpc_user *fl;
197 struct dma_buf *dmabuf;
198 struct device *dev;
199 void *virt;
200 u64 phys;
201 u64 size;
202 /* Lock for dma buf attachments */
203 struct mutex lock;
204 struct list_head attachments;
205 /* mmap support */
206 struct list_head node; /* list of user requested mmaps */
207 uintptr_t raddr;
208};
209
210struct fastrpc_dma_buf_attachment {
211 struct device *dev;
212 struct sg_table sgt;
213 struct list_head node;
214};
215
216struct fastrpc_map {
217 struct list_head node;
218 struct fastrpc_user *fl;
219 int fd;
220 struct dma_buf *buf;
221 struct sg_table *table;
222 struct dma_buf_attachment *attach;
223 u64 phys;
224 u64 size;
225 void *va;
226 u64 len;
227 u64 raddr;
228 u32 attr;
229 struct kref refcount;
230};
231
232struct fastrpc_invoke_ctx {
233 int nscalars;
234 int nbufs;
235 int retval;
236 int pid;
237 int tgid;
238 u32 sc;
239 u32 *crc;
240 u64 ctxid;
241 u64 msg_sz;
242 struct kref refcount;
243 struct list_head node; /* list of ctxs */
244 struct completion work;
245 struct work_struct put_work;
246 struct fastrpc_msg msg;
247 struct fastrpc_user *fl;
248 union fastrpc_remote_arg *rpra;
249 struct fastrpc_map **maps;
250 struct fastrpc_buf *buf;
251 struct fastrpc_invoke_args *args;
252 struct fastrpc_buf_overlap *olaps;
253 struct fastrpc_channel_ctx *cctx;
254};
255
256struct fastrpc_session_ctx {
257 struct device *dev;
258 int sid;
259 bool used;
260 bool valid;
261};
262
263struct fastrpc_channel_ctx {
264 int domain_id;
265 int sesscount;
266 int vmcount;
267 struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
268 struct rpmsg_device *rpdev;
269 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
270 spinlock_t lock;
271 struct idr ctx_idr;
272 struct list_head users;
273 struct kref refcount;
274 /* Flag if dsp attributes are cached */
275 bool valid_attributes;
276 u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
277 struct fastrpc_device *secure_fdevice;
278 struct fastrpc_device *fdevice;
279 struct fastrpc_buf *remote_heap;
280 struct list_head invoke_interrupted_mmaps;
281 bool secure;
282 bool unsigned_support;
283 u64 dma_mask;
284};
285
286struct fastrpc_device {
287 struct fastrpc_channel_ctx *cctx;
288 struct miscdevice miscdev;
289 bool secure;
290};
291
292struct fastrpc_user {
293 struct list_head user;
294 struct list_head maps;
295 struct list_head pending;
296 struct list_head mmaps;
297
298 struct fastrpc_channel_ctx *cctx;
299 struct fastrpc_session_ctx *sctx;
300 struct fastrpc_buf *init_mem;
301
302 int tgid;
303 int pd;
304 bool is_secure_dev;
305 /* Lock for lists */
306 spinlock_t lock;
307 /* lock for allocations */
308 struct mutex mutex;
309};
310
311static void fastrpc_free_map(struct kref *ref)
312{
313 struct fastrpc_map *map;
314
315 map = container_of(ref, struct fastrpc_map, refcount);
316
317 if (map->table) {
318 if (map->attr & FASTRPC_ATTR_SECUREMAP) {
319 struct qcom_scm_vmperm perm;
320 int vmid = map->fl->cctx->vmperms[0].vmid;
321 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
322 int err = 0;
323
324 perm.vmid = QCOM_SCM_VMID_HLOS;
325 perm.perm = QCOM_SCM_PERM_RWX;
326 err = qcom_scm_assign_mem(map->phys, map->size,
327 &src_perms, &perm, 1);
328 if (err) {
329 dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
330 map->phys, map->size, err);
331 return;
332 }
333 }
334 dma_buf_unmap_attachment_unlocked(map->attach, map->table,
335 DMA_BIDIRECTIONAL);
336 dma_buf_detach(map->buf, map->attach);
337 dma_buf_put(map->buf);
338 }
339
340 if (map->fl) {
341 spin_lock(&map->fl->lock);
342 list_del(&map->node);
343 spin_unlock(&map->fl->lock);
344 map->fl = NULL;
345 }
346
347 kfree(map);
348}
349
350static void fastrpc_map_put(struct fastrpc_map *map)
351{
352 if (map)
353 kref_put(&map->refcount, fastrpc_free_map);
354}
355
356static int fastrpc_map_get(struct fastrpc_map *map)
357{
358 if (!map)
359 return -ENOENT;
360
361 return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;
362}
363
364
365static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
366 struct fastrpc_map **ppmap, bool take_ref)
367{
368 struct fastrpc_session_ctx *sess = fl->sctx;
369 struct fastrpc_map *map = NULL;
370 int ret = -ENOENT;
371
372 spin_lock(&fl->lock);
373 list_for_each_entry(map, &fl->maps, node) {
374 if (map->fd != fd)
375 continue;
376
377 if (take_ref) {
378 ret = fastrpc_map_get(map);
379 if (ret) {
380 dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",
381 __func__, fd, ret);
382 break;
383 }
384 }
385
386 *ppmap = map;
387 ret = 0;
388 break;
389 }
390 spin_unlock(&fl->lock);
391
392 return ret;
393}
394
395static void fastrpc_buf_free(struct fastrpc_buf *buf)
396{
397 dma_free_coherent(buf->dev, buf->size, buf->virt,
398 FASTRPC_PHYS(buf->phys));
399 kfree(buf);
400}
401
402static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
403 u64 size, struct fastrpc_buf **obuf)
404{
405 struct fastrpc_buf *buf;
406
407 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
408 if (!buf)
409 return -ENOMEM;
410
411 INIT_LIST_HEAD(&buf->attachments);
412 INIT_LIST_HEAD(&buf->node);
413 mutex_init(&buf->lock);
414
415 buf->fl = fl;
416 buf->virt = NULL;
417 buf->phys = 0;
418 buf->size = size;
419 buf->dev = dev;
420 buf->raddr = 0;
421
422 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
423 GFP_KERNEL);
424 if (!buf->virt) {
425 mutex_destroy(&buf->lock);
426 kfree(buf);
427 return -ENOMEM;
428 }
429
430 *obuf = buf;
431
432 return 0;
433}
434
435static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
436 u64 size, struct fastrpc_buf **obuf)
437{
438 int ret;
439 struct fastrpc_buf *buf;
440
441 ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
442 if (ret)
443 return ret;
444
445 buf = *obuf;
446
447 if (fl->sctx && fl->sctx->sid)
448 buf->phys += ((u64)fl->sctx->sid << 32);
449
450 return 0;
451}
452
453static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
454 u64 size, struct fastrpc_buf **obuf)
455{
456 struct device *rdev = &fl->cctx->rpdev->dev;
457
458 return __fastrpc_buf_alloc(fl, rdev, size, obuf);
459}
460
461static void fastrpc_channel_ctx_free(struct kref *ref)
462{
463 struct fastrpc_channel_ctx *cctx;
464
465 cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
466
467 kfree(cctx);
468}
469
470static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
471{
472 kref_get(&cctx->refcount);
473}
474
475static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
476{
477 kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
478}
479
480static void fastrpc_context_free(struct kref *ref)
481{
482 struct fastrpc_invoke_ctx *ctx;
483 struct fastrpc_channel_ctx *cctx;
484 unsigned long flags;
485 int i;
486
487 ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
488 cctx = ctx->cctx;
489
490 for (i = 0; i < ctx->nbufs; i++)
491 fastrpc_map_put(ctx->maps[i]);
492
493 if (ctx->buf)
494 fastrpc_buf_free(ctx->buf);
495
496 spin_lock_irqsave(&cctx->lock, flags);
497 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
498 spin_unlock_irqrestore(&cctx->lock, flags);
499
500 kfree(ctx->maps);
501 kfree(ctx->olaps);
502 kfree(ctx);
503
504 fastrpc_channel_ctx_put(cctx);
505}
506
507static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
508{
509 kref_get(&ctx->refcount);
510}
511
512static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
513{
514 kref_put(&ctx->refcount, fastrpc_context_free);
515}
516
517static void fastrpc_context_put_wq(struct work_struct *work)
518{
519 struct fastrpc_invoke_ctx *ctx =
520 container_of(work, struct fastrpc_invoke_ctx, put_work);
521
522 fastrpc_context_put(ctx);
523}
524
525#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
526static int olaps_cmp(const void *a, const void *b)
527{
528 struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
529 struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
530 /* sort with lowest starting buffer first */
531 int st = CMP(pa->start, pb->start);
532 /* sort with highest ending buffer first */
533 int ed = CMP(pb->end, pa->end);
534
535 return st == 0 ? ed : st;
536}
537
538static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
539{
540 u64 max_end = 0;
541 int i;
542
543 for (i = 0; i < ctx->nbufs; ++i) {
544 ctx->olaps[i].start = ctx->args[i].ptr;
545 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
546 ctx->olaps[i].raix = i;
547 }
548
549 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
550
551 for (i = 0; i < ctx->nbufs; ++i) {
552 /* Falling inside previous range */
553 if (ctx->olaps[i].start < max_end) {
554 ctx->olaps[i].mstart = max_end;
555 ctx->olaps[i].mend = ctx->olaps[i].end;
556 ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
557
558 if (ctx->olaps[i].end > max_end) {
559 max_end = ctx->olaps[i].end;
560 } else {
561 ctx->olaps[i].mend = 0;
562 ctx->olaps[i].mstart = 0;
563 }
564
565 } else {
566 ctx->olaps[i].mend = ctx->olaps[i].end;
567 ctx->olaps[i].mstart = ctx->olaps[i].start;
568 ctx->olaps[i].offset = 0;
569 max_end = ctx->olaps[i].end;
570 }
571 }
572}
573
574static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
575 struct fastrpc_user *user, u32 kernel, u32 sc,
576 struct fastrpc_invoke_args *args)
577{
578 struct fastrpc_channel_ctx *cctx = user->cctx;
579 struct fastrpc_invoke_ctx *ctx = NULL;
580 unsigned long flags;
581 int ret;
582
583 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
584 if (!ctx)
585 return ERR_PTR(-ENOMEM);
586
587 INIT_LIST_HEAD(&ctx->node);
588 ctx->fl = user;
589 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
590 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
591 REMOTE_SCALARS_OUTBUFS(sc);
592
593 if (ctx->nscalars) {
594 ctx->maps = kcalloc(ctx->nscalars,
595 sizeof(*ctx->maps), GFP_KERNEL);
596 if (!ctx->maps) {
597 kfree(ctx);
598 return ERR_PTR(-ENOMEM);
599 }
600 ctx->olaps = kcalloc(ctx->nscalars,
601 sizeof(*ctx->olaps), GFP_KERNEL);
602 if (!ctx->olaps) {
603 kfree(ctx->maps);
604 kfree(ctx);
605 return ERR_PTR(-ENOMEM);
606 }
607 ctx->args = args;
608 fastrpc_get_buff_overlaps(ctx);
609 }
610
611 /* Released in fastrpc_context_put() */
612 fastrpc_channel_ctx_get(cctx);
613
614 ctx->sc = sc;
615 ctx->retval = -1;
616 ctx->pid = current->pid;
617 ctx->tgid = user->tgid;
618 ctx->cctx = cctx;
619 init_completion(&ctx->work);
620 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
621
622 spin_lock(&user->lock);
623 list_add_tail(&ctx->node, &user->pending);
624 spin_unlock(&user->lock);
625
626 spin_lock_irqsave(&cctx->lock, flags);
627 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
628 FASTRPC_CTX_MAX, GFP_ATOMIC);
629 if (ret < 0) {
630 spin_unlock_irqrestore(&cctx->lock, flags);
631 goto err_idr;
632 }
633 ctx->ctxid = ret << 4;
634 spin_unlock_irqrestore(&cctx->lock, flags);
635
636 kref_init(&ctx->refcount);
637
638 return ctx;
639err_idr:
640 spin_lock(&user->lock);
641 list_del(&ctx->node);
642 spin_unlock(&user->lock);
643 fastrpc_channel_ctx_put(cctx);
644 kfree(ctx->maps);
645 kfree(ctx->olaps);
646 kfree(ctx);
647
648 return ERR_PTR(ret);
649}
650
651static struct sg_table *
652fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
653 enum dma_data_direction dir)
654{
655 struct fastrpc_dma_buf_attachment *a = attachment->priv;
656 struct sg_table *table;
657 int ret;
658
659 table = &a->sgt;
660
661 ret = dma_map_sgtable(attachment->dev, table, dir, 0);
662 if (ret)
663 table = ERR_PTR(ret);
664 return table;
665}
666
667static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
668 struct sg_table *table,
669 enum dma_data_direction dir)
670{
671 dma_unmap_sgtable(attach->dev, table, dir, 0);
672}
673
674static void fastrpc_release(struct dma_buf *dmabuf)
675{
676 struct fastrpc_buf *buffer = dmabuf->priv;
677
678 fastrpc_buf_free(buffer);
679}
680
681static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
682 struct dma_buf_attachment *attachment)
683{
684 struct fastrpc_dma_buf_attachment *a;
685 struct fastrpc_buf *buffer = dmabuf->priv;
686 int ret;
687
688 a = kzalloc(sizeof(*a), GFP_KERNEL);
689 if (!a)
690 return -ENOMEM;
691
692 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
693 FASTRPC_PHYS(buffer->phys), buffer->size);
694 if (ret < 0) {
695 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
696 kfree(a);
697 return -EINVAL;
698 }
699
700 a->dev = attachment->dev;
701 INIT_LIST_HEAD(&a->node);
702 attachment->priv = a;
703
704 mutex_lock(&buffer->lock);
705 list_add(&a->node, &buffer->attachments);
706 mutex_unlock(&buffer->lock);
707
708 return 0;
709}
710
711static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
712 struct dma_buf_attachment *attachment)
713{
714 struct fastrpc_dma_buf_attachment *a = attachment->priv;
715 struct fastrpc_buf *buffer = dmabuf->priv;
716
717 mutex_lock(&buffer->lock);
718 list_del(&a->node);
719 mutex_unlock(&buffer->lock);
720 sg_free_table(&a->sgt);
721 kfree(a);
722}
723
724static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
725{
726 struct fastrpc_buf *buf = dmabuf->priv;
727
728 iosys_map_set_vaddr(map, buf->virt);
729
730 return 0;
731}
732
733static int fastrpc_mmap(struct dma_buf *dmabuf,
734 struct vm_area_struct *vma)
735{
736 struct fastrpc_buf *buf = dmabuf->priv;
737 size_t size = vma->vm_end - vma->vm_start;
738
739 dma_resv_assert_held(dmabuf->resv);
740
741 return dma_mmap_coherent(buf->dev, vma, buf->virt,
742 FASTRPC_PHYS(buf->phys), size);
743}
744
745static const struct dma_buf_ops fastrpc_dma_buf_ops = {
746 .attach = fastrpc_dma_buf_attach,
747 .detach = fastrpc_dma_buf_detatch,
748 .map_dma_buf = fastrpc_map_dma_buf,
749 .unmap_dma_buf = fastrpc_unmap_dma_buf,
750 .mmap = fastrpc_mmap,
751 .vmap = fastrpc_vmap,
752 .release = fastrpc_release,
753};
754
755static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
756 u64 len, u32 attr, struct fastrpc_map **ppmap)
757{
758 struct fastrpc_session_ctx *sess = fl->sctx;
759 struct fastrpc_map *map = NULL;
760 struct sg_table *table;
761 int err = 0;
762
763 if (!fastrpc_map_lookup(fl, fd, ppmap, true))
764 return 0;
765
766 map = kzalloc(sizeof(*map), GFP_KERNEL);
767 if (!map)
768 return -ENOMEM;
769
770 INIT_LIST_HEAD(&map->node);
771 kref_init(&map->refcount);
772
773 map->fl = fl;
774 map->fd = fd;
775 map->buf = dma_buf_get(fd);
776 if (IS_ERR(map->buf)) {
777 err = PTR_ERR(map->buf);
778 goto get_err;
779 }
780
781 map->attach = dma_buf_attach(map->buf, sess->dev);
782 if (IS_ERR(map->attach)) {
783 dev_err(sess->dev, "Failed to attach dmabuf\n");
784 err = PTR_ERR(map->attach);
785 goto attach_err;
786 }
787
788 table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL);
789 if (IS_ERR(table)) {
790 err = PTR_ERR(table);
791 goto map_err;
792 }
793 map->table = table;
794
795 if (attr & FASTRPC_ATTR_SECUREMAP) {
796 map->phys = sg_phys(map->table->sgl);
797 } else {
798 map->phys = sg_dma_address(map->table->sgl);
799 map->phys += ((u64)fl->sctx->sid << 32);
800 }
801 map->size = len;
802 map->va = sg_virt(map->table->sgl);
803 map->len = len;
804
805 if (attr & FASTRPC_ATTR_SECUREMAP) {
806 /*
807 * If subsystem VMIDs are defined in DTSI, then do
808 * hyp_assign from HLOS to those VM(s)
809 */
810 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
811 struct qcom_scm_vmperm dst_perms[2] = {0};
812
813 dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
814 dst_perms[0].perm = QCOM_SCM_PERM_RW;
815 dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
816 dst_perms[1].perm = QCOM_SCM_PERM_RWX;
817 map->attr = attr;
818 err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2);
819 if (err) {
820 dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
821 map->phys, map->size, err);
822 goto map_err;
823 }
824 }
825 spin_lock(&fl->lock);
826 list_add_tail(&map->node, &fl->maps);
827 spin_unlock(&fl->lock);
828 *ppmap = map;
829
830 return 0;
831
832map_err:
833 dma_buf_detach(map->buf, map->attach);
834attach_err:
835 dma_buf_put(map->buf);
836get_err:
837 fastrpc_map_put(map);
838
839 return err;
840}
841
842/*
843 * Fastrpc payload buffer with metadata looks like:
844 *
845 * >>>>>> START of METADATA <<<<<<<<<
846 * +---------------------------------+
847 * | Arguments |
848 * | type:(union fastrpc_remote_arg)|
849 * | (0 - N) |
850 * +---------------------------------+
851 * | Invoke Buffer list |
852 * | type:(struct fastrpc_invoke_buf)|
853 * | (0 - N) |
854 * +---------------------------------+
855 * | Page info list |
856 * | type:(struct fastrpc_phy_page) |
857 * | (0 - N) |
858 * +---------------------------------+
859 * | Optional info |
860 * |(can be specific to SoC/Firmware)|
861 * +---------------------------------+
862 * >>>>>>>> END of METADATA <<<<<<<<<
863 * +---------------------------------+
864 * | Inline ARGS |
865 * | (0-N) |
866 * +---------------------------------+
867 */
868
869static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
870{
871 int size = 0;
872
873 size = (sizeof(struct fastrpc_remote_buf) +
874 sizeof(struct fastrpc_invoke_buf) +
875 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
876 sizeof(u64) * FASTRPC_MAX_FDLIST +
877 sizeof(u32) * FASTRPC_MAX_CRCLIST;
878
879 return size;
880}
881
882static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
883{
884 u64 size = 0;
885 int oix;
886
887 size = ALIGN(metalen, FASTRPC_ALIGN);
888 for (oix = 0; oix < ctx->nbufs; oix++) {
889 int i = ctx->olaps[oix].raix;
890
891 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
892
893 if (ctx->olaps[oix].offset == 0)
894 size = ALIGN(size, FASTRPC_ALIGN);
895
896 size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
897 }
898 }
899
900 return size;
901}
902
903static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
904{
905 struct device *dev = ctx->fl->sctx->dev;
906 int i, err;
907
908 for (i = 0; i < ctx->nscalars; ++i) {
909
910 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
911 ctx->args[i].length == 0)
912 continue;
913
914 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
915 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
916 if (err) {
917 dev_err(dev, "Error Creating map %d\n", err);
918 return -EINVAL;
919 }
920
921 }
922 return 0;
923}
924
925static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
926{
927 return (struct fastrpc_invoke_buf *)(&pra[len]);
928}
929
930static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
931{
932 return (struct fastrpc_phy_page *)(&buf[len]);
933}
934
935static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
936{
937 struct device *dev = ctx->fl->sctx->dev;
938 union fastrpc_remote_arg *rpra;
939 struct fastrpc_invoke_buf *list;
940 struct fastrpc_phy_page *pages;
941 int inbufs, i, oix, err = 0;
942 u64 len, rlen, pkt_size;
943 u64 pg_start, pg_end;
944 uintptr_t args;
945 int metalen;
946
947 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
948 metalen = fastrpc_get_meta_size(ctx);
949 pkt_size = fastrpc_get_payload_size(ctx, metalen);
950
951 err = fastrpc_create_maps(ctx);
952 if (err)
953 return err;
954
955 ctx->msg_sz = pkt_size;
956
957 if (ctx->fl->sctx->sid)
958 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
959 else
960 err = fastrpc_remote_heap_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
961 if (err)
962 return err;
963
964 memset(ctx->buf->virt, 0, pkt_size);
965 rpra = ctx->buf->virt;
966 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
967 pages = fastrpc_phy_page_start(list, ctx->nscalars);
968 args = (uintptr_t)ctx->buf->virt + metalen;
969 rlen = pkt_size - metalen;
970 ctx->rpra = rpra;
971
972 for (oix = 0; oix < ctx->nbufs; ++oix) {
973 int mlen;
974
975 i = ctx->olaps[oix].raix;
976 len = ctx->args[i].length;
977
978 rpra[i].buf.pv = 0;
979 rpra[i].buf.len = len;
980 list[i].num = len ? 1 : 0;
981 list[i].pgidx = i;
982
983 if (!len)
984 continue;
985
986 if (ctx->maps[i]) {
987 struct vm_area_struct *vma = NULL;
988
989 rpra[i].buf.pv = (u64) ctx->args[i].ptr;
990 pages[i].addr = ctx->maps[i]->phys;
991
992 mmap_read_lock(current->mm);
993 vma = find_vma(current->mm, ctx->args[i].ptr);
994 if (vma)
995 pages[i].addr += (ctx->args[i].ptr & PAGE_MASK) -
996 vma->vm_start;
997 mmap_read_unlock(current->mm);
998
999 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
1000 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
1001 PAGE_SHIFT;
1002 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1003
1004 } else {
1005
1006 if (ctx->olaps[oix].offset == 0) {
1007 rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1008 args = ALIGN(args, FASTRPC_ALIGN);
1009 }
1010
1011 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1012
1013 if (rlen < mlen)
1014 goto bail;
1015
1016 rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1017 pages[i].addr = ctx->buf->phys -
1018 ctx->olaps[oix].offset +
1019 (pkt_size - rlen);
1020 pages[i].addr = pages[i].addr & PAGE_MASK;
1021
1022 pg_start = (rpra[i].buf.pv & PAGE_MASK) >> PAGE_SHIFT;
1023 pg_end = ((rpra[i].buf.pv + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1024 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1025 args = args + mlen;
1026 rlen -= mlen;
1027 }
1028
1029 if (i < inbufs && !ctx->maps[i]) {
1030 void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1031 void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1032
1033 if (!kernel) {
1034 if (copy_from_user(dst, (void __user *)src,
1035 len)) {
1036 err = -EFAULT;
1037 goto bail;
1038 }
1039 } else {
1040 memcpy(dst, src, len);
1041 }
1042 }
1043 }
1044
1045 for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1046 list[i].num = ctx->args[i].length ? 1 : 0;
1047 list[i].pgidx = i;
1048 if (ctx->maps[i]) {
1049 pages[i].addr = ctx->maps[i]->phys;
1050 pages[i].size = ctx->maps[i]->size;
1051 }
1052 rpra[i].dma.fd = ctx->args[i].fd;
1053 rpra[i].dma.len = ctx->args[i].length;
1054 rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1055 }
1056
1057bail:
1058 if (err)
1059 dev_err(dev, "Error: get invoke args failed:%d\n", err);
1060
1061 return err;
1062}
1063
1064static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1065 u32 kernel)
1066{
1067 union fastrpc_remote_arg *rpra = ctx->rpra;
1068 struct fastrpc_user *fl = ctx->fl;
1069 struct fastrpc_map *mmap = NULL;
1070 struct fastrpc_invoke_buf *list;
1071 struct fastrpc_phy_page *pages;
1072 u64 *fdlist;
1073 int i, inbufs, outbufs, handles;
1074
1075 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1076 outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1077 handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1078 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
1079 pages = fastrpc_phy_page_start(list, ctx->nscalars);
1080 fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1081
1082 for (i = inbufs; i < ctx->nbufs; ++i) {
1083 if (!ctx->maps[i]) {
1084 void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1085 void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1086 u64 len = rpra[i].buf.len;
1087
1088 if (!kernel) {
1089 if (copy_to_user((void __user *)dst, src, len))
1090 return -EFAULT;
1091 } else {
1092 memcpy(dst, src, len);
1093 }
1094 }
1095 }
1096
1097 /* Clean up fdlist which is updated by DSP */
1098 for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1099 if (!fdlist[i])
1100 break;
1101 if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false))
1102 fastrpc_map_put(mmap);
1103 }
1104
1105 return 0;
1106}
1107
1108static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1109 struct fastrpc_invoke_ctx *ctx,
1110 u32 kernel, uint32_t handle)
1111{
1112 struct fastrpc_channel_ctx *cctx;
1113 struct fastrpc_user *fl = ctx->fl;
1114 struct fastrpc_msg *msg = &ctx->msg;
1115 int ret;
1116
1117 cctx = fl->cctx;
1118 msg->pid = fl->tgid;
1119 msg->tid = current->pid;
1120
1121 if (kernel)
1122 msg->pid = 0;
1123
1124 msg->ctx = ctx->ctxid | fl->pd;
1125 msg->handle = handle;
1126 msg->sc = ctx->sc;
1127 msg->addr = ctx->buf ? ctx->buf->phys : 0;
1128 msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1129 fastrpc_context_get(ctx);
1130
1131 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
1132
1133 if (ret)
1134 fastrpc_context_put(ctx);
1135
1136 return ret;
1137
1138}
1139
1140static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
1141 u32 handle, u32 sc,
1142 struct fastrpc_invoke_args *args)
1143{
1144 struct fastrpc_invoke_ctx *ctx = NULL;
1145 struct fastrpc_buf *buf, *b;
1146
1147 int err = 0;
1148
1149 if (!fl->sctx)
1150 return -EINVAL;
1151
1152 if (!fl->cctx->rpdev)
1153 return -EPIPE;
1154
1155 if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1156 dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
1157 return -EPERM;
1158 }
1159
1160 ctx = fastrpc_context_alloc(fl, kernel, sc, args);
1161 if (IS_ERR(ctx))
1162 return PTR_ERR(ctx);
1163
1164 err = fastrpc_get_args(kernel, ctx);
1165 if (err)
1166 goto bail;
1167
1168 /* make sure that all CPU memory writes are seen by DSP */
1169 dma_wmb();
1170 /* Send invoke buffer to remote dsp */
1171 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
1172 if (err)
1173 goto bail;
1174
1175 if (kernel) {
1176 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1177 err = -ETIMEDOUT;
1178 } else {
1179 err = wait_for_completion_interruptible(&ctx->work);
1180 }
1181
1182 if (err)
1183 goto bail;
1184
1185 /* make sure that all memory writes by DSP are seen by CPU */
1186 dma_rmb();
1187 /* populate all the output buffers with results */
1188 err = fastrpc_put_args(ctx, kernel);
1189 if (err)
1190 goto bail;
1191
1192 /* Check the response from remote dsp */
1193 err = ctx->retval;
1194 if (err)
1195 goto bail;
1196
1197bail:
1198 if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1199 /* We are done with this compute context */
1200 spin_lock(&fl->lock);
1201 list_del(&ctx->node);
1202 spin_unlock(&fl->lock);
1203 fastrpc_context_put(ctx);
1204 }
1205
1206 if (err == -ERESTARTSYS) {
1207 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1208 list_del(&buf->node);
1209 list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);
1210 }
1211 }
1212
1213 if (err)
1214 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1215
1216 return err;
1217}
1218
1219static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1220{
1221 /* Check if the device node is non-secure and channel is secure*/
1222 if (!fl->is_secure_dev && fl->cctx->secure) {
1223 /*
1224 * Allow untrusted applications to offload only to Unsigned PD when
1225 * channel is configured as secure and block untrusted apps on channel
1226 * that does not support unsigned PD offload
1227 */
1228 if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1229 dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n");
1230 return true;
1231 }
1232 }
1233
1234 return false;
1235}
1236
1237static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1238 char __user *argp)
1239{
1240 struct fastrpc_init_create_static init;
1241 struct fastrpc_invoke_args *args;
1242 struct fastrpc_phy_page pages[1];
1243 char *name;
1244 int err;
1245 bool scm_done = false;
1246 struct {
1247 int pgid;
1248 u32 namelen;
1249 u32 pageslen;
1250 } inbuf;
1251 u32 sc;
1252
1253 args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1254 if (!args)
1255 return -ENOMEM;
1256
1257 if (copy_from_user(&init, argp, sizeof(init))) {
1258 err = -EFAULT;
1259 goto err;
1260 }
1261
1262 if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1263 err = -EINVAL;
1264 goto err;
1265 }
1266
1267 name = memdup_user(u64_to_user_ptr(init.name), init.namelen);
1268 if (IS_ERR(name)) {
1269 err = PTR_ERR(name);
1270 goto err;
1271 }
1272
1273 if (!fl->cctx->remote_heap) {
1274 err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
1275 &fl->cctx->remote_heap);
1276 if (err)
1277 goto err_name;
1278
1279 /* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1280 if (fl->cctx->vmcount) {
1281 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1282
1283 err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1284 (u64)fl->cctx->remote_heap->size,
1285 &src_perms,
1286 fl->cctx->vmperms, fl->cctx->vmcount);
1287 if (err) {
1288 dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
1289 fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1290 goto err_map;
1291 }
1292 scm_done = true;
1293 }
1294 }
1295
1296 inbuf.pgid = fl->tgid;
1297 inbuf.namelen = init.namelen;
1298 inbuf.pageslen = 0;
1299 fl->pd = USER_PD;
1300
1301 args[0].ptr = (u64)(uintptr_t)&inbuf;
1302 args[0].length = sizeof(inbuf);
1303 args[0].fd = -1;
1304
1305 args[1].ptr = (u64)(uintptr_t)name;
1306 args[1].length = inbuf.namelen;
1307 args[1].fd = -1;
1308
1309 pages[0].addr = fl->cctx->remote_heap->phys;
1310 pages[0].size = fl->cctx->remote_heap->size;
1311
1312 args[2].ptr = (u64)(uintptr_t) pages;
1313 args[2].length = sizeof(*pages);
1314 args[2].fd = -1;
1315
1316 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1317
1318 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1319 sc, args);
1320 if (err)
1321 goto err_invoke;
1322
1323 kfree(args);
1324 kfree(name);
1325
1326 return 0;
1327err_invoke:
1328 if (fl->cctx->vmcount && scm_done) {
1329 u64 src_perms = 0;
1330 struct qcom_scm_vmperm dst_perms;
1331 u32 i;
1332
1333 for (i = 0; i < fl->cctx->vmcount; i++)
1334 src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1335
1336 dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1337 dst_perms.perm = QCOM_SCM_PERM_RWX;
1338 err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1339 (u64)fl->cctx->remote_heap->size,
1340 &src_perms, &dst_perms, 1);
1341 if (err)
1342 dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
1343 fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1344 }
1345err_map:
1346 fastrpc_buf_free(fl->cctx->remote_heap);
1347err_name:
1348 kfree(name);
1349err:
1350 kfree(args);
1351
1352 return err;
1353}
1354
1355static int fastrpc_init_create_process(struct fastrpc_user *fl,
1356 char __user *argp)
1357{
1358 struct fastrpc_init_create init;
1359 struct fastrpc_invoke_args *args;
1360 struct fastrpc_phy_page pages[1];
1361 struct fastrpc_map *map = NULL;
1362 struct fastrpc_buf *imem = NULL;
1363 int memlen;
1364 int err;
1365 struct {
1366 int pgid;
1367 u32 namelen;
1368 u32 filelen;
1369 u32 pageslen;
1370 u32 attrs;
1371 u32 siglen;
1372 } inbuf;
1373 u32 sc;
1374 bool unsigned_module = false;
1375
1376 args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1377 if (!args)
1378 return -ENOMEM;
1379
1380 if (copy_from_user(&init, argp, sizeof(init))) {
1381 err = -EFAULT;
1382 goto err;
1383 }
1384
1385 if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1386 unsigned_module = true;
1387
1388 if (is_session_rejected(fl, unsigned_module)) {
1389 err = -ECONNREFUSED;
1390 goto err;
1391 }
1392
1393 if (init.filelen > INIT_FILELEN_MAX) {
1394 err = -EINVAL;
1395 goto err;
1396 }
1397
1398 inbuf.pgid = fl->tgid;
1399 inbuf.namelen = strlen(current->comm) + 1;
1400 inbuf.filelen = init.filelen;
1401 inbuf.pageslen = 1;
1402 inbuf.attrs = init.attrs;
1403 inbuf.siglen = init.siglen;
1404 fl->pd = USER_PD;
1405
1406 if (init.filelen && init.filefd) {
1407 err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);
1408 if (err)
1409 goto err;
1410 }
1411
1412 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1413 1024 * 1024);
1414 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1415 &imem);
1416 if (err)
1417 goto err_alloc;
1418
1419 fl->init_mem = imem;
1420 args[0].ptr = (u64)(uintptr_t)&inbuf;
1421 args[0].length = sizeof(inbuf);
1422 args[0].fd = -1;
1423
1424 args[1].ptr = (u64)(uintptr_t)current->comm;
1425 args[1].length = inbuf.namelen;
1426 args[1].fd = -1;
1427
1428 args[2].ptr = (u64) init.file;
1429 args[2].length = inbuf.filelen;
1430 args[2].fd = init.filefd;
1431
1432 pages[0].addr = imem->phys;
1433 pages[0].size = imem->size;
1434
1435 args[3].ptr = (u64)(uintptr_t) pages;
1436 args[3].length = 1 * sizeof(*pages);
1437 args[3].fd = -1;
1438
1439 args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1440 args[4].length = sizeof(inbuf.attrs);
1441 args[4].fd = -1;
1442
1443 args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1444 args[5].length = sizeof(inbuf.siglen);
1445 args[5].fd = -1;
1446
1447 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1448 if (init.attrs)
1449 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1450
1451 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1452 sc, args);
1453 if (err)
1454 goto err_invoke;
1455
1456 kfree(args);
1457
1458 return 0;
1459
1460err_invoke:
1461 fl->init_mem = NULL;
1462 fastrpc_buf_free(imem);
1463err_alloc:
1464 fastrpc_map_put(map);
1465err:
1466 kfree(args);
1467
1468 return err;
1469}
1470
1471static struct fastrpc_session_ctx *fastrpc_session_alloc(
1472 struct fastrpc_channel_ctx *cctx)
1473{
1474 struct fastrpc_session_ctx *session = NULL;
1475 unsigned long flags;
1476 int i;
1477
1478 spin_lock_irqsave(&cctx->lock, flags);
1479 for (i = 0; i < cctx->sesscount; i++) {
1480 if (!cctx->session[i].used && cctx->session[i].valid) {
1481 cctx->session[i].used = true;
1482 session = &cctx->session[i];
1483 break;
1484 }
1485 }
1486 spin_unlock_irqrestore(&cctx->lock, flags);
1487
1488 return session;
1489}
1490
1491static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1492 struct fastrpc_session_ctx *session)
1493{
1494 unsigned long flags;
1495
1496 spin_lock_irqsave(&cctx->lock, flags);
1497 session->used = false;
1498 spin_unlock_irqrestore(&cctx->lock, flags);
1499}
1500
1501static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1502{
1503 struct fastrpc_invoke_args args[1];
1504 int tgid = 0;
1505 u32 sc;
1506
1507 tgid = fl->tgid;
1508 args[0].ptr = (u64)(uintptr_t) &tgid;
1509 args[0].length = sizeof(tgid);
1510 args[0].fd = -1;
1511 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1512
1513 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1514 sc, &args[0]);
1515}
1516
1517static int fastrpc_device_release(struct inode *inode, struct file *file)
1518{
1519 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1520 struct fastrpc_channel_ctx *cctx = fl->cctx;
1521 struct fastrpc_invoke_ctx *ctx, *n;
1522 struct fastrpc_map *map, *m;
1523 struct fastrpc_buf *buf, *b;
1524 unsigned long flags;
1525
1526 fastrpc_release_current_dsp_process(fl);
1527
1528 spin_lock_irqsave(&cctx->lock, flags);
1529 list_del(&fl->user);
1530 spin_unlock_irqrestore(&cctx->lock, flags);
1531
1532 if (fl->init_mem)
1533 fastrpc_buf_free(fl->init_mem);
1534
1535 list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1536 list_del(&ctx->node);
1537 fastrpc_context_put(ctx);
1538 }
1539
1540 list_for_each_entry_safe(map, m, &fl->maps, node)
1541 fastrpc_map_put(map);
1542
1543 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1544 list_del(&buf->node);
1545 fastrpc_buf_free(buf);
1546 }
1547
1548 fastrpc_session_free(cctx, fl->sctx);
1549 fastrpc_channel_ctx_put(cctx);
1550
1551 mutex_destroy(&fl->mutex);
1552 kfree(fl);
1553 file->private_data = NULL;
1554
1555 return 0;
1556}
1557
1558static int fastrpc_device_open(struct inode *inode, struct file *filp)
1559{
1560 struct fastrpc_channel_ctx *cctx;
1561 struct fastrpc_device *fdevice;
1562 struct fastrpc_user *fl = NULL;
1563 unsigned long flags;
1564
1565 fdevice = miscdev_to_fdevice(filp->private_data);
1566 cctx = fdevice->cctx;
1567
1568 fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1569 if (!fl)
1570 return -ENOMEM;
1571
1572 /* Released in fastrpc_device_release() */
1573 fastrpc_channel_ctx_get(cctx);
1574
1575 filp->private_data = fl;
1576 spin_lock_init(&fl->lock);
1577 mutex_init(&fl->mutex);
1578 INIT_LIST_HEAD(&fl->pending);
1579 INIT_LIST_HEAD(&fl->maps);
1580 INIT_LIST_HEAD(&fl->mmaps);
1581 INIT_LIST_HEAD(&fl->user);
1582 fl->tgid = current->tgid;
1583 fl->cctx = cctx;
1584 fl->is_secure_dev = fdevice->secure;
1585
1586 fl->sctx = fastrpc_session_alloc(cctx);
1587 if (!fl->sctx) {
1588 dev_err(&cctx->rpdev->dev, "No session available\n");
1589 mutex_destroy(&fl->mutex);
1590 kfree(fl);
1591
1592 return -EBUSY;
1593 }
1594
1595 spin_lock_irqsave(&cctx->lock, flags);
1596 list_add_tail(&fl->user, &cctx->users);
1597 spin_unlock_irqrestore(&cctx->lock, flags);
1598
1599 return 0;
1600}
1601
1602static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1603{
1604 struct fastrpc_alloc_dma_buf bp;
1605 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1606 struct fastrpc_buf *buf = NULL;
1607 int err;
1608
1609 if (copy_from_user(&bp, argp, sizeof(bp)))
1610 return -EFAULT;
1611
1612 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1613 if (err)
1614 return err;
1615 exp_info.ops = &fastrpc_dma_buf_ops;
1616 exp_info.size = bp.size;
1617 exp_info.flags = O_RDWR;
1618 exp_info.priv = buf;
1619 buf->dmabuf = dma_buf_export(&exp_info);
1620 if (IS_ERR(buf->dmabuf)) {
1621 err = PTR_ERR(buf->dmabuf);
1622 fastrpc_buf_free(buf);
1623 return err;
1624 }
1625
1626 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1627 if (bp.fd < 0) {
1628 dma_buf_put(buf->dmabuf);
1629 return -EINVAL;
1630 }
1631
1632 if (copy_to_user(argp, &bp, sizeof(bp))) {
1633 /*
1634 * The usercopy failed, but we can't do much about it, as
1635 * dma_buf_fd() already called fd_install() and made the
1636 * file descriptor accessible for the current process. It
1637 * might already be closed and dmabuf no longer valid when
1638 * we reach this point. Therefore "leak" the fd and rely on
1639 * the process exit path to do any required cleanup.
1640 */
1641 return -EFAULT;
1642 }
1643
1644 return 0;
1645}
1646
1647static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1648{
1649 struct fastrpc_invoke_args args[1];
1650 int tgid = fl->tgid;
1651 u32 sc;
1652
1653 args[0].ptr = (u64)(uintptr_t) &tgid;
1654 args[0].length = sizeof(tgid);
1655 args[0].fd = -1;
1656 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1657 fl->pd = pd;
1658
1659 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1660 sc, &args[0]);
1661}
1662
1663static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1664{
1665 struct fastrpc_invoke_args *args = NULL;
1666 struct fastrpc_invoke inv;
1667 u32 nscalars;
1668 int err;
1669
1670 if (copy_from_user(&inv, argp, sizeof(inv)))
1671 return -EFAULT;
1672
1673 /* nscalars is truncated here to max supported value */
1674 nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1675 if (nscalars) {
1676 args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1677 if (!args)
1678 return -ENOMEM;
1679
1680 if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1681 nscalars * sizeof(*args))) {
1682 kfree(args);
1683 return -EFAULT;
1684 }
1685 }
1686
1687 err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1688 kfree(args);
1689
1690 return err;
1691}
1692
1693static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1694 uint32_t dsp_attr_buf_len)
1695{
1696 struct fastrpc_invoke_args args[2] = { 0 };
1697
1698 /*
1699 * Capability filled in userspace. This carries the information
1700 * about the remoteproc support which is fetched from the remoteproc
1701 * sysfs node by userspace.
1702 */
1703 dsp_attr_buf[0] = 0;
1704 dsp_attr_buf_len -= 1;
1705
1706 args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1707 args[0].length = sizeof(dsp_attr_buf_len);
1708 args[0].fd = -1;
1709 args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1710 args[1].length = dsp_attr_buf_len * sizeof(u32);
1711 args[1].fd = -1;
1712
1713 return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE,
1714 FASTRPC_SCALARS(0, 1, 1), args);
1715}
1716
1717static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1718 struct fastrpc_user *fl)
1719{
1720 struct fastrpc_channel_ctx *cctx = fl->cctx;
1721 uint32_t attribute_id = cap->attribute_id;
1722 uint32_t *dsp_attributes;
1723 unsigned long flags;
1724 uint32_t domain = cap->domain;
1725 int err;
1726
1727 spin_lock_irqsave(&cctx->lock, flags);
1728 /* check if we already have queried dsp for attributes */
1729 if (cctx->valid_attributes) {
1730 spin_unlock_irqrestore(&cctx->lock, flags);
1731 goto done;
1732 }
1733 spin_unlock_irqrestore(&cctx->lock, flags);
1734
1735 dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1736 if (!dsp_attributes)
1737 return -ENOMEM;
1738
1739 err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES);
1740 if (err == DSP_UNSUPPORTED_API) {
1741 dev_info(&cctx->rpdev->dev,
1742 "Warning: DSP capabilities not supported on domain: %d\n", domain);
1743 kfree(dsp_attributes);
1744 return -EOPNOTSUPP;
1745 } else if (err) {
1746 dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1747 kfree(dsp_attributes);
1748 return err;
1749 }
1750
1751 spin_lock_irqsave(&cctx->lock, flags);
1752 memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1753 cctx->valid_attributes = true;
1754 spin_unlock_irqrestore(&cctx->lock, flags);
1755 kfree(dsp_attributes);
1756done:
1757 cap->capability = cctx->dsp_attributes[attribute_id];
1758 return 0;
1759}
1760
1761static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1762{
1763 struct fastrpc_ioctl_capability cap = {0};
1764 int err = 0;
1765
1766 if (copy_from_user(&cap, argp, sizeof(cap)))
1767 return -EFAULT;
1768
1769 cap.capability = 0;
1770 if (cap.domain >= FASTRPC_DEV_MAX) {
1771 dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",
1772 cap.domain, err);
1773 return -ECHRNG;
1774 }
1775
1776 /* Fastrpc Capablities does not support modem domain */
1777 if (cap.domain == MDSP_DOMAIN_ID) {
1778 dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);
1779 return -ECHRNG;
1780 }
1781
1782 if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1783 dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1784 cap.attribute_id, err);
1785 return -EOVERFLOW;
1786 }
1787
1788 err = fastrpc_get_info_from_kernel(&cap, fl);
1789 if (err)
1790 return err;
1791
1792 if (copy_to_user(argp, &cap, sizeof(cap)))
1793 return -EFAULT;
1794
1795 return 0;
1796}
1797
1798static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1799{
1800 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1801 struct fastrpc_munmap_req_msg req_msg;
1802 struct device *dev = fl->sctx->dev;
1803 int err;
1804 u32 sc;
1805
1806 req_msg.pgid = fl->tgid;
1807 req_msg.size = buf->size;
1808 req_msg.vaddr = buf->raddr;
1809
1810 args[0].ptr = (u64) (uintptr_t) &req_msg;
1811 args[0].length = sizeof(req_msg);
1812
1813 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1814 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1815 &args[0]);
1816 if (!err) {
1817 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1818 spin_lock(&fl->lock);
1819 list_del(&buf->node);
1820 spin_unlock(&fl->lock);
1821 fastrpc_buf_free(buf);
1822 } else {
1823 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1824 }
1825
1826 return err;
1827}
1828
1829static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1830{
1831 struct fastrpc_buf *buf = NULL, *iter, *b;
1832 struct fastrpc_req_munmap req;
1833 struct device *dev = fl->sctx->dev;
1834
1835 if (copy_from_user(&req, argp, sizeof(req)))
1836 return -EFAULT;
1837
1838 spin_lock(&fl->lock);
1839 list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1840 if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1841 buf = iter;
1842 break;
1843 }
1844 }
1845 spin_unlock(&fl->lock);
1846
1847 if (!buf) {
1848 dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1849 req.vaddrout, req.size);
1850 return -EINVAL;
1851 }
1852
1853 return fastrpc_req_munmap_impl(fl, buf);
1854}
1855
1856static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1857{
1858 struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1859 struct fastrpc_buf *buf = NULL;
1860 struct fastrpc_mmap_req_msg req_msg;
1861 struct fastrpc_mmap_rsp_msg rsp_msg;
1862 struct fastrpc_phy_page pages;
1863 struct fastrpc_req_mmap req;
1864 struct device *dev = fl->sctx->dev;
1865 int err;
1866 u32 sc;
1867
1868 if (copy_from_user(&req, argp, sizeof(req)))
1869 return -EFAULT;
1870
1871 if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1872 dev_err(dev, "flag not supported 0x%x\n", req.flags);
1873
1874 return -EINVAL;
1875 }
1876
1877 if (req.vaddrin) {
1878 dev_err(dev, "adding user allocated pages is not supported\n");
1879 return -EINVAL;
1880 }
1881
1882 if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1883 err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);
1884 else
1885 err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
1886
1887 if (err) {
1888 dev_err(dev, "failed to allocate buffer\n");
1889 return err;
1890 }
1891
1892 req_msg.pgid = fl->tgid;
1893 req_msg.flags = req.flags;
1894 req_msg.vaddr = req.vaddrin;
1895 req_msg.num = sizeof(pages);
1896
1897 args[0].ptr = (u64) (uintptr_t) &req_msg;
1898 args[0].length = sizeof(req_msg);
1899
1900 pages.addr = buf->phys;
1901 pages.size = buf->size;
1902
1903 args[1].ptr = (u64) (uintptr_t) &pages;
1904 args[1].length = sizeof(pages);
1905
1906 args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1907 args[2].length = sizeof(rsp_msg);
1908
1909 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1910 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1911 &args[0]);
1912 if (err) {
1913 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1914 fastrpc_buf_free(buf);
1915 return err;
1916 }
1917
1918 /* update the buffer to be able to deallocate the memory on the DSP */
1919 buf->raddr = (uintptr_t) rsp_msg.vaddr;
1920
1921 /* let the client know the address to use */
1922 req.vaddrout = rsp_msg.vaddr;
1923
1924 /* Add memory to static PD pool, protection thru hypervisor */
1925 if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1926 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1927
1928 err = qcom_scm_assign_mem(buf->phys, (u64)buf->size,
1929 &src_perms, fl->cctx->vmperms, fl->cctx->vmcount);
1930 if (err) {
1931 dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1932 buf->phys, buf->size, err);
1933 goto err_assign;
1934 }
1935 }
1936
1937 spin_lock(&fl->lock);
1938 list_add_tail(&buf->node, &fl->mmaps);
1939 spin_unlock(&fl->lock);
1940
1941 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1942 err = -EFAULT;
1943 goto err_assign;
1944 }
1945
1946 dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1947 buf->raddr, buf->size);
1948
1949 return 0;
1950
1951err_assign:
1952 fastrpc_req_munmap_impl(fl, buf);
1953
1954 return err;
1955}
1956
1957static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1958{
1959 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1960 struct fastrpc_map *map = NULL, *iter, *m;
1961 struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1962 int err = 0;
1963 u32 sc;
1964 struct device *dev = fl->sctx->dev;
1965
1966 spin_lock(&fl->lock);
1967 list_for_each_entry_safe(iter, m, &fl->maps, node) {
1968 if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1969 map = iter;
1970 break;
1971 }
1972 }
1973
1974 spin_unlock(&fl->lock);
1975
1976 if (!map) {
1977 dev_err(dev, "map not in list\n");
1978 return -EINVAL;
1979 }
1980
1981 req_msg.pgid = fl->tgid;
1982 req_msg.len = map->len;
1983 req_msg.vaddrin = map->raddr;
1984 req_msg.fd = map->fd;
1985
1986 args[0].ptr = (u64) (uintptr_t) &req_msg;
1987 args[0].length = sizeof(req_msg);
1988
1989 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
1990 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1991 &args[0]);
1992 if (err) {
1993 dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr);
1994 return err;
1995 }
1996 fastrpc_map_put(map);
1997
1998 return 0;
1999}
2000
2001static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
2002{
2003 struct fastrpc_mem_unmap req;
2004
2005 if (copy_from_user(&req, argp, sizeof(req)))
2006 return -EFAULT;
2007
2008 return fastrpc_req_mem_unmap_impl(fl, &req);
2009}
2010
2011static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2012{
2013 struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2014 struct fastrpc_mem_map_req_msg req_msg = { 0 };
2015 struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2016 struct fastrpc_mem_unmap req_unmap = { 0 };
2017 struct fastrpc_phy_page pages = { 0 };
2018 struct fastrpc_mem_map req;
2019 struct device *dev = fl->sctx->dev;
2020 struct fastrpc_map *map = NULL;
2021 int err;
2022 u32 sc;
2023
2024 if (copy_from_user(&req, argp, sizeof(req)))
2025 return -EFAULT;
2026
2027 /* create SMMU mapping */
2028 err = fastrpc_map_create(fl, req.fd, req.length, 0, &map);
2029 if (err) {
2030 dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2031 return err;
2032 }
2033
2034 req_msg.pgid = fl->tgid;
2035 req_msg.fd = req.fd;
2036 req_msg.offset = req.offset;
2037 req_msg.vaddrin = req.vaddrin;
2038 map->va = (void *) (uintptr_t) req.vaddrin;
2039 req_msg.flags = req.flags;
2040 req_msg.num = sizeof(pages);
2041 req_msg.data_len = 0;
2042
2043 args[0].ptr = (u64) (uintptr_t) &req_msg;
2044 args[0].length = sizeof(req_msg);
2045
2046 pages.addr = map->phys;
2047 pages.size = map->size;
2048
2049 args[1].ptr = (u64) (uintptr_t) &pages;
2050 args[1].length = sizeof(pages);
2051
2052 args[2].ptr = (u64) (uintptr_t) &pages;
2053 args[2].length = 0;
2054
2055 args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2056 args[3].length = sizeof(rsp_msg);
2057
2058 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2059 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);
2060 if (err) {
2061 dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2062 req.fd, req.vaddrin, map->size);
2063 goto err_invoke;
2064 }
2065
2066 /* update the buffer to be able to deallocate the memory on the DSP */
2067 map->raddr = rsp_msg.vaddr;
2068
2069 /* let the client know the address to use */
2070 req.vaddrout = rsp_msg.vaddr;
2071
2072 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
2073 /* unmap the memory and release the buffer */
2074 req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2075 req_unmap.length = map->size;
2076 fastrpc_req_mem_unmap_impl(fl, &req_unmap);
2077 return -EFAULT;
2078 }
2079
2080 return 0;
2081
2082err_invoke:
2083 fastrpc_map_put(map);
2084
2085 return err;
2086}
2087
2088static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2089 unsigned long arg)
2090{
2091 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2092 char __user *argp = (char __user *)arg;
2093 int err;
2094
2095 switch (cmd) {
2096 case FASTRPC_IOCTL_INVOKE:
2097 err = fastrpc_invoke(fl, argp);
2098 break;
2099 case FASTRPC_IOCTL_INIT_ATTACH:
2100 err = fastrpc_init_attach(fl, ROOT_PD);
2101 break;
2102 case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2103 err = fastrpc_init_attach(fl, SENSORS_PD);
2104 break;
2105 case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2106 err = fastrpc_init_create_static_process(fl, argp);
2107 break;
2108 case FASTRPC_IOCTL_INIT_CREATE:
2109 err = fastrpc_init_create_process(fl, argp);
2110 break;
2111 case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2112 err = fastrpc_dmabuf_alloc(fl, argp);
2113 break;
2114 case FASTRPC_IOCTL_MMAP:
2115 err = fastrpc_req_mmap(fl, argp);
2116 break;
2117 case FASTRPC_IOCTL_MUNMAP:
2118 err = fastrpc_req_munmap(fl, argp);
2119 break;
2120 case FASTRPC_IOCTL_MEM_MAP:
2121 err = fastrpc_req_mem_map(fl, argp);
2122 break;
2123 case FASTRPC_IOCTL_MEM_UNMAP:
2124 err = fastrpc_req_mem_unmap(fl, argp);
2125 break;
2126 case FASTRPC_IOCTL_GET_DSP_INFO:
2127 err = fastrpc_get_dsp_info(fl, argp);
2128 break;
2129 default:
2130 err = -ENOTTY;
2131 break;
2132 }
2133
2134 return err;
2135}
2136
2137static const struct file_operations fastrpc_fops = {
2138 .open = fastrpc_device_open,
2139 .release = fastrpc_device_release,
2140 .unlocked_ioctl = fastrpc_device_ioctl,
2141 .compat_ioctl = fastrpc_device_ioctl,
2142};
2143
2144static int fastrpc_cb_probe(struct platform_device *pdev)
2145{
2146 struct fastrpc_channel_ctx *cctx;
2147 struct fastrpc_session_ctx *sess;
2148 struct device *dev = &pdev->dev;
2149 int i, sessions = 0;
2150 unsigned long flags;
2151 int rc;
2152
2153 cctx = dev_get_drvdata(dev->parent);
2154 if (!cctx)
2155 return -EINVAL;
2156
2157 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
2158
2159 spin_lock_irqsave(&cctx->lock, flags);
2160 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2161 dev_err(&pdev->dev, "too many sessions\n");
2162 spin_unlock_irqrestore(&cctx->lock, flags);
2163 return -ENOSPC;
2164 }
2165 sess = &cctx->session[cctx->sesscount++];
2166 sess->used = false;
2167 sess->valid = true;
2168 sess->dev = dev;
2169 dev_set_drvdata(dev, sess);
2170
2171 if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
2172 dev_info(dev, "FastRPC Session ID not specified in DT\n");
2173
2174 if (sessions > 0) {
2175 struct fastrpc_session_ctx *dup_sess;
2176
2177 for (i = 1; i < sessions; i++) {
2178 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2179 break;
2180 dup_sess = &cctx->session[cctx->sesscount++];
2181 memcpy(dup_sess, sess, sizeof(*dup_sess));
2182 }
2183 }
2184 spin_unlock_irqrestore(&cctx->lock, flags);
2185 rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2186 if (rc) {
2187 dev_err(dev, "32-bit DMA enable failed\n");
2188 return rc;
2189 }
2190
2191 return 0;
2192}
2193
2194static void fastrpc_cb_remove(struct platform_device *pdev)
2195{
2196 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
2197 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
2198 unsigned long flags;
2199 int i;
2200
2201 spin_lock_irqsave(&cctx->lock, flags);
2202 for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2203 if (cctx->session[i].sid == sess->sid) {
2204 cctx->session[i].valid = false;
2205 cctx->sesscount--;
2206 }
2207 }
2208 spin_unlock_irqrestore(&cctx->lock, flags);
2209}
2210
2211static const struct of_device_id fastrpc_match_table[] = {
2212 { .compatible = "qcom,fastrpc-compute-cb", },
2213 {}
2214};
2215
2216static struct platform_driver fastrpc_cb_driver = {
2217 .probe = fastrpc_cb_probe,
2218 .remove = fastrpc_cb_remove,
2219 .driver = {
2220 .name = "qcom,fastrpc-cb",
2221 .of_match_table = fastrpc_match_table,
2222 .suppress_bind_attrs = true,
2223 },
2224};
2225
2226static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2227 bool is_secured, const char *domain)
2228{
2229 struct fastrpc_device *fdev;
2230 int err;
2231
2232 fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL);
2233 if (!fdev)
2234 return -ENOMEM;
2235
2236 fdev->secure = is_secured;
2237 fdev->cctx = cctx;
2238 fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2239 fdev->miscdev.fops = &fastrpc_fops;
2240 fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s",
2241 domain, is_secured ? "-secure" : "");
2242 if (!fdev->miscdev.name)
2243 return -ENOMEM;
2244
2245 err = misc_register(&fdev->miscdev);
2246 if (!err) {
2247 if (is_secured)
2248 cctx->secure_fdevice = fdev;
2249 else
2250 cctx->fdevice = fdev;
2251 }
2252
2253 return err;
2254}
2255
2256static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2257{
2258 struct device *rdev = &rpdev->dev;
2259 struct fastrpc_channel_ctx *data;
2260 int i, err, domain_id = -1, vmcount;
2261 const char *domain;
2262 bool secure_dsp;
2263 struct device_node *rmem_node;
2264 struct reserved_mem *rmem;
2265 unsigned int vmids[FASTRPC_MAX_VMIDS];
2266
2267 err = of_property_read_string(rdev->of_node, "label", &domain);
2268 if (err) {
2269 dev_info(rdev, "FastRPC Domain not specified in DT\n");
2270 return err;
2271 }
2272
2273 for (i = 0; i < FASTRPC_DEV_MAX; i++) {
2274 if (!strcmp(domains[i], domain)) {
2275 domain_id = i;
2276 break;
2277 }
2278 }
2279
2280 if (domain_id < 0) {
2281 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2282 return -EINVAL;
2283 }
2284
2285 if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))
2286 dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2287
2288 vmcount = of_property_read_variable_u32_array(rdev->of_node,
2289 "qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
2290 if (vmcount < 0)
2291 vmcount = 0;
2292 else if (!qcom_scm_is_available())
2293 return -EPROBE_DEFER;
2294
2295 data = kzalloc(sizeof(*data), GFP_KERNEL);
2296 if (!data)
2297 return -ENOMEM;
2298
2299 if (vmcount) {
2300 data->vmcount = vmcount;
2301 for (i = 0; i < data->vmcount; i++) {
2302 data->vmperms[i].vmid = vmids[i];
2303 data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2304 }
2305 }
2306
2307 rmem_node = of_parse_phandle(rdev->of_node, "memory-region", 0);
2308 if (domain_id == SDSP_DOMAIN_ID && rmem_node) {
2309 u64 src_perms;
2310
2311 rmem = of_reserved_mem_lookup(rmem_node);
2312 if (!rmem) {
2313 err = -EINVAL;
2314 goto fdev_error;
2315 }
2316
2317 src_perms = BIT(QCOM_SCM_VMID_HLOS);
2318
2319 qcom_scm_assign_mem(rmem->base, rmem->size, &src_perms,
2320 data->vmperms, data->vmcount);
2321
2322 }
2323
2324 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2325 data->secure = secure_dsp;
2326
2327 switch (domain_id) {
2328 case ADSP_DOMAIN_ID:
2329 case MDSP_DOMAIN_ID:
2330 case SDSP_DOMAIN_ID:
2331 /* Unsigned PD offloading is only supported on CDSP and CDSP1 */
2332 data->unsigned_support = false;
2333 err = fastrpc_device_register(rdev, data, secure_dsp, domains[domain_id]);
2334 if (err)
2335 goto fdev_error;
2336 break;
2337 case CDSP_DOMAIN_ID:
2338 case CDSP1_DOMAIN_ID:
2339 data->unsigned_support = true;
2340 /* Create both device nodes so that we can allow both Signed and Unsigned PD */
2341 err = fastrpc_device_register(rdev, data, true, domains[domain_id]);
2342 if (err)
2343 goto fdev_error;
2344
2345 err = fastrpc_device_register(rdev, data, false, domains[domain_id]);
2346 if (err)
2347 goto populate_error;
2348 break;
2349 default:
2350 err = -EINVAL;
2351 goto fdev_error;
2352 }
2353
2354 kref_init(&data->refcount);
2355
2356 dev_set_drvdata(&rpdev->dev, data);
2357 rdev->dma_mask = &data->dma_mask;
2358 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
2359 INIT_LIST_HEAD(&data->users);
2360 INIT_LIST_HEAD(&data->invoke_interrupted_mmaps);
2361 spin_lock_init(&data->lock);
2362 idr_init(&data->ctx_idr);
2363 data->domain_id = domain_id;
2364 data->rpdev = rpdev;
2365
2366 err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);
2367 if (err)
2368 goto populate_error;
2369
2370 return 0;
2371
2372populate_error:
2373 if (data->fdevice)
2374 misc_deregister(&data->fdevice->miscdev);
2375 if (data->secure_fdevice)
2376 misc_deregister(&data->secure_fdevice->miscdev);
2377
2378fdev_error:
2379 kfree(data);
2380 return err;
2381}
2382
2383static void fastrpc_notify_users(struct fastrpc_user *user)
2384{
2385 struct fastrpc_invoke_ctx *ctx;
2386
2387 spin_lock(&user->lock);
2388 list_for_each_entry(ctx, &user->pending, node) {
2389 ctx->retval = -EPIPE;
2390 complete(&ctx->work);
2391 }
2392 spin_unlock(&user->lock);
2393}
2394
2395static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2396{
2397 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2398 struct fastrpc_buf *buf, *b;
2399 struct fastrpc_user *user;
2400 unsigned long flags;
2401
2402 /* No invocations past this point */
2403 spin_lock_irqsave(&cctx->lock, flags);
2404 cctx->rpdev = NULL;
2405 list_for_each_entry(user, &cctx->users, user)
2406 fastrpc_notify_users(user);
2407 spin_unlock_irqrestore(&cctx->lock, flags);
2408
2409 if (cctx->fdevice)
2410 misc_deregister(&cctx->fdevice->miscdev);
2411
2412 if (cctx->secure_fdevice)
2413 misc_deregister(&cctx->secure_fdevice->miscdev);
2414
2415 list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2416 list_del(&buf->node);
2417
2418 if (cctx->remote_heap)
2419 fastrpc_buf_free(cctx->remote_heap);
2420
2421 of_platform_depopulate(&rpdev->dev);
2422
2423 fastrpc_channel_ctx_put(cctx);
2424}
2425
2426static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2427 int len, void *priv, u32 addr)
2428{
2429 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2430 struct fastrpc_invoke_rsp *rsp = data;
2431 struct fastrpc_invoke_ctx *ctx;
2432 unsigned long flags;
2433 unsigned long ctxid;
2434
2435 if (len < sizeof(*rsp))
2436 return -EINVAL;
2437
2438 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2439
2440 spin_lock_irqsave(&cctx->lock, flags);
2441 ctx = idr_find(&cctx->ctx_idr, ctxid);
2442 spin_unlock_irqrestore(&cctx->lock, flags);
2443
2444 if (!ctx) {
2445 dev_err(&rpdev->dev, "No context ID matches response\n");
2446 return -ENOENT;
2447 }
2448
2449 ctx->retval = rsp->retval;
2450 complete(&ctx->work);
2451
2452 /*
2453 * The DMA buffer associated with the context cannot be freed in
2454 * interrupt context so schedule it through a worker thread to
2455 * avoid a kernel BUG.
2456 */
2457 schedule_work(&ctx->put_work);
2458
2459 return 0;
2460}
2461
2462static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2463 { .compatible = "qcom,fastrpc" },
2464 { },
2465};
2466MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2467
2468static struct rpmsg_driver fastrpc_driver = {
2469 .probe = fastrpc_rpmsg_probe,
2470 .remove = fastrpc_rpmsg_remove,
2471 .callback = fastrpc_rpmsg_callback,
2472 .drv = {
2473 .name = "qcom,fastrpc",
2474 .of_match_table = fastrpc_rpmsg_of_match,
2475 },
2476};
2477
2478static int fastrpc_init(void)
2479{
2480 int ret;
2481
2482 ret = platform_driver_register(&fastrpc_cb_driver);
2483 if (ret < 0) {
2484 pr_err("fastrpc: failed to register cb driver\n");
2485 return ret;
2486 }
2487
2488 ret = register_rpmsg_driver(&fastrpc_driver);
2489 if (ret < 0) {
2490 pr_err("fastrpc: failed to register rpmsg driver\n");
2491 platform_driver_unregister(&fastrpc_cb_driver);
2492 return ret;
2493 }
2494
2495 return 0;
2496}
2497module_init(fastrpc_init);
2498
2499static void fastrpc_exit(void)
2500{
2501 platform_driver_unregister(&fastrpc_cb_driver);
2502 unregister_rpmsg_driver(&fastrpc_driver);
2503}
2504module_exit(fastrpc_exit);
2505
2506MODULE_DESCRIPTION("Qualcomm FastRPC");
2507MODULE_LICENSE("GPL v2");
2508MODULE_IMPORT_NS("DMA_BUF");