Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3// Copyright (c) 2018, Linaro Limited
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/idr.h>
10#include <linux/list.h>
11#include <linux/miscdevice.h>
12#include <linux/module.h>
13#include <linux/of_address.h>
14#include <linux/of.h>
15#include <linux/sort.h>
16#include <linux/of_platform.h>
17#include <linux/rpmsg.h>
18#include <linux/scatterlist.h>
19#include <linux/slab.h>
20#include <uapi/misc/fastrpc.h>
21
22#define ADSP_DOMAIN_ID (0)
23#define MDSP_DOMAIN_ID (1)
24#define SDSP_DOMAIN_ID (2)
25#define CDSP_DOMAIN_ID (3)
26#define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
27#define FASTRPC_MAX_SESSIONS 9 /*8 compute, 1 cpz*/
28#define FASTRPC_ALIGN 128
29#define FASTRPC_MAX_FDLIST 16
30#define FASTRPC_MAX_CRCLIST 64
31#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
32#define FASTRPC_CTX_MAX (256)
33#define FASTRPC_INIT_HANDLE 1
34#define FASTRPC_CTXID_MASK (0xFF0)
35#define INIT_FILELEN_MAX (64 * 1024 * 1024)
36#define FASTRPC_DEVICE_NAME "fastrpc"
37
38/* Retrives number of input buffers from the scalars parameter */
39#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
40
41/* Retrives number of output buffers from the scalars parameter */
42#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
43
44/* Retrives number of input handles from the scalars parameter */
45#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
46
47/* Retrives number of output handles from the scalars parameter */
48#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
49
50#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
51 REMOTE_SCALARS_OUTBUFS(sc) + \
52 REMOTE_SCALARS_INHANDLES(sc)+ \
53 REMOTE_SCALARS_OUTHANDLES(sc))
54#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
55 (((attr & 0x07) << 29) | \
56 ((method & 0x1f) << 24) | \
57 ((in & 0xff) << 16) | \
58 ((out & 0xff) << 8) | \
59 ((oin & 0x0f) << 4) | \
60 (oout & 0x0f))
61
62#define FASTRPC_SCALARS(method, in, out) \
63 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
64
65#define FASTRPC_CREATE_PROCESS_NARGS 6
66/* Remote Method id table */
67#define FASTRPC_RMID_INIT_ATTACH 0
68#define FASTRPC_RMID_INIT_RELEASE 1
69#define FASTRPC_RMID_INIT_CREATE 6
70#define FASTRPC_RMID_INIT_CREATE_ATTR 7
71#define FASTRPC_RMID_INIT_CREATE_STATIC 8
72
73#define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev)
74
75static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
76 "sdsp", "cdsp"};
77struct fastrpc_phy_page {
78 u64 addr; /* physical address */
79 u64 size; /* size of contiguous region */
80};
81
82struct fastrpc_invoke_buf {
83 u32 num; /* number of contiguous regions */
84 u32 pgidx; /* index to start of contiguous region */
85};
86
87struct fastrpc_remote_arg {
88 u64 pv;
89 u64 len;
90};
91
92struct fastrpc_msg {
93 int pid; /* process group id */
94 int tid; /* thread id */
95 u64 ctx; /* invoke caller context */
96 u32 handle; /* handle to invoke */
97 u32 sc; /* scalars structure describing the data */
98 u64 addr; /* physical address */
99 u64 size; /* size of contiguous region */
100};
101
102struct fastrpc_invoke_rsp {
103 u64 ctx; /* invoke caller context */
104 int retval; /* invoke return value */
105};
106
107struct fastrpc_buf_overlap {
108 u64 start;
109 u64 end;
110 int raix;
111 u64 mstart;
112 u64 mend;
113 u64 offset;
114};
115
116struct fastrpc_buf {
117 struct fastrpc_user *fl;
118 struct dma_buf *dmabuf;
119 struct device *dev;
120 void *virt;
121 u64 phys;
122 u64 size;
123 /* Lock for dma buf attachments */
124 struct mutex lock;
125 struct list_head attachments;
126};
127
128struct fastrpc_dma_buf_attachment {
129 struct device *dev;
130 struct sg_table sgt;
131 struct list_head node;
132};
133
134struct fastrpc_map {
135 struct list_head node;
136 struct fastrpc_user *fl;
137 int fd;
138 struct dma_buf *buf;
139 struct sg_table *table;
140 struct dma_buf_attachment *attach;
141 u64 phys;
142 u64 size;
143 void *va;
144 u64 len;
145 struct kref refcount;
146};
147
148struct fastrpc_invoke_ctx {
149 int nscalars;
150 int nbufs;
151 int retval;
152 int pid;
153 int tgid;
154 u32 sc;
155 u32 *crc;
156 u64 ctxid;
157 u64 msg_sz;
158 struct kref refcount;
159 struct list_head node; /* list of ctxs */
160 struct completion work;
161 struct work_struct put_work;
162 struct fastrpc_msg msg;
163 struct fastrpc_user *fl;
164 struct fastrpc_remote_arg *rpra;
165 struct fastrpc_map **maps;
166 struct fastrpc_buf *buf;
167 struct fastrpc_invoke_args *args;
168 struct fastrpc_buf_overlap *olaps;
169 struct fastrpc_channel_ctx *cctx;
170};
171
172struct fastrpc_session_ctx {
173 struct device *dev;
174 int sid;
175 bool used;
176 bool valid;
177};
178
179struct fastrpc_channel_ctx {
180 int domain_id;
181 int sesscount;
182 struct rpmsg_device *rpdev;
183 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
184 spinlock_t lock;
185 struct idr ctx_idr;
186 struct list_head users;
187 struct miscdevice miscdev;
188 struct kref refcount;
189};
190
191struct fastrpc_user {
192 struct list_head user;
193 struct list_head maps;
194 struct list_head pending;
195
196 struct fastrpc_channel_ctx *cctx;
197 struct fastrpc_session_ctx *sctx;
198 struct fastrpc_buf *init_mem;
199
200 int tgid;
201 int pd;
202 /* Lock for lists */
203 spinlock_t lock;
204 /* lock for allocations */
205 struct mutex mutex;
206};
207
208static void fastrpc_free_map(struct kref *ref)
209{
210 struct fastrpc_map *map;
211
212 map = container_of(ref, struct fastrpc_map, refcount);
213
214 if (map->table) {
215 dma_buf_unmap_attachment(map->attach, map->table,
216 DMA_BIDIRECTIONAL);
217 dma_buf_detach(map->buf, map->attach);
218 dma_buf_put(map->buf);
219 }
220
221 kfree(map);
222}
223
224static void fastrpc_map_put(struct fastrpc_map *map)
225{
226 if (map)
227 kref_put(&map->refcount, fastrpc_free_map);
228}
229
230static void fastrpc_map_get(struct fastrpc_map *map)
231{
232 if (map)
233 kref_get(&map->refcount);
234}
235
236static int fastrpc_map_find(struct fastrpc_user *fl, int fd,
237 struct fastrpc_map **ppmap)
238{
239 struct fastrpc_map *map = NULL;
240
241 mutex_lock(&fl->mutex);
242 list_for_each_entry(map, &fl->maps, node) {
243 if (map->fd == fd) {
244 fastrpc_map_get(map);
245 *ppmap = map;
246 mutex_unlock(&fl->mutex);
247 return 0;
248 }
249 }
250 mutex_unlock(&fl->mutex);
251
252 return -ENOENT;
253}
254
255static void fastrpc_buf_free(struct fastrpc_buf *buf)
256{
257 dma_free_coherent(buf->dev, buf->size, buf->virt,
258 FASTRPC_PHYS(buf->phys));
259 kfree(buf);
260}
261
262static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
263 u64 size, struct fastrpc_buf **obuf)
264{
265 struct fastrpc_buf *buf;
266
267 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
268 if (!buf)
269 return -ENOMEM;
270
271 INIT_LIST_HEAD(&buf->attachments);
272 mutex_init(&buf->lock);
273
274 buf->fl = fl;
275 buf->virt = NULL;
276 buf->phys = 0;
277 buf->size = size;
278 buf->dev = dev;
279
280 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
281 GFP_KERNEL);
282 if (!buf->virt) {
283 mutex_destroy(&buf->lock);
284 kfree(buf);
285 return -ENOMEM;
286 }
287
288 if (fl->sctx && fl->sctx->sid)
289 buf->phys += ((u64)fl->sctx->sid << 32);
290
291 *obuf = buf;
292
293 return 0;
294}
295
296static void fastrpc_channel_ctx_free(struct kref *ref)
297{
298 struct fastrpc_channel_ctx *cctx;
299
300 cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
301
302 kfree(cctx);
303}
304
305static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
306{
307 kref_get(&cctx->refcount);
308}
309
310static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
311{
312 kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
313}
314
315static void fastrpc_context_free(struct kref *ref)
316{
317 struct fastrpc_invoke_ctx *ctx;
318 struct fastrpc_channel_ctx *cctx;
319 unsigned long flags;
320 int i;
321
322 ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
323 cctx = ctx->cctx;
324
325 for (i = 0; i < ctx->nscalars; i++)
326 fastrpc_map_put(ctx->maps[i]);
327
328 if (ctx->buf)
329 fastrpc_buf_free(ctx->buf);
330
331 spin_lock_irqsave(&cctx->lock, flags);
332 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
333 spin_unlock_irqrestore(&cctx->lock, flags);
334
335 kfree(ctx->maps);
336 kfree(ctx->olaps);
337 kfree(ctx);
338
339 fastrpc_channel_ctx_put(cctx);
340}
341
342static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
343{
344 kref_get(&ctx->refcount);
345}
346
347static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
348{
349 kref_put(&ctx->refcount, fastrpc_context_free);
350}
351
352static void fastrpc_context_put_wq(struct work_struct *work)
353{
354 struct fastrpc_invoke_ctx *ctx =
355 container_of(work, struct fastrpc_invoke_ctx, put_work);
356
357 fastrpc_context_put(ctx);
358}
359
360#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
361static int olaps_cmp(const void *a, const void *b)
362{
363 struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
364 struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
365 /* sort with lowest starting buffer first */
366 int st = CMP(pa->start, pb->start);
367 /* sort with highest ending buffer first */
368 int ed = CMP(pb->end, pa->end);
369
370 return st == 0 ? ed : st;
371}
372
373static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
374{
375 u64 max_end = 0;
376 int i;
377
378 for (i = 0; i < ctx->nbufs; ++i) {
379 ctx->olaps[i].start = ctx->args[i].ptr;
380 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
381 ctx->olaps[i].raix = i;
382 }
383
384 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
385
386 for (i = 0; i < ctx->nbufs; ++i) {
387 /* Falling inside previous range */
388 if (ctx->olaps[i].start < max_end) {
389 ctx->olaps[i].mstart = max_end;
390 ctx->olaps[i].mend = ctx->olaps[i].end;
391 ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
392
393 if (ctx->olaps[i].end > max_end) {
394 max_end = ctx->olaps[i].end;
395 } else {
396 ctx->olaps[i].mend = 0;
397 ctx->olaps[i].mstart = 0;
398 }
399
400 } else {
401 ctx->olaps[i].mend = ctx->olaps[i].end;
402 ctx->olaps[i].mstart = ctx->olaps[i].start;
403 ctx->olaps[i].offset = 0;
404 max_end = ctx->olaps[i].end;
405 }
406 }
407}
408
409static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
410 struct fastrpc_user *user, u32 kernel, u32 sc,
411 struct fastrpc_invoke_args *args)
412{
413 struct fastrpc_channel_ctx *cctx = user->cctx;
414 struct fastrpc_invoke_ctx *ctx = NULL;
415 unsigned long flags;
416 int ret;
417
418 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
419 if (!ctx)
420 return ERR_PTR(-ENOMEM);
421
422 INIT_LIST_HEAD(&ctx->node);
423 ctx->fl = user;
424 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
425 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
426 REMOTE_SCALARS_OUTBUFS(sc);
427
428 if (ctx->nscalars) {
429 ctx->maps = kcalloc(ctx->nscalars,
430 sizeof(*ctx->maps), GFP_KERNEL);
431 if (!ctx->maps) {
432 kfree(ctx);
433 return ERR_PTR(-ENOMEM);
434 }
435 ctx->olaps = kcalloc(ctx->nscalars,
436 sizeof(*ctx->olaps), GFP_KERNEL);
437 if (!ctx->olaps) {
438 kfree(ctx->maps);
439 kfree(ctx);
440 return ERR_PTR(-ENOMEM);
441 }
442 ctx->args = args;
443 fastrpc_get_buff_overlaps(ctx);
444 }
445
446 /* Released in fastrpc_context_put() */
447 fastrpc_channel_ctx_get(cctx);
448
449 ctx->sc = sc;
450 ctx->retval = -1;
451 ctx->pid = current->pid;
452 ctx->tgid = user->tgid;
453 ctx->cctx = cctx;
454 init_completion(&ctx->work);
455 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
456
457 spin_lock(&user->lock);
458 list_add_tail(&ctx->node, &user->pending);
459 spin_unlock(&user->lock);
460
461 spin_lock_irqsave(&cctx->lock, flags);
462 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
463 FASTRPC_CTX_MAX, GFP_ATOMIC);
464 if (ret < 0) {
465 spin_unlock_irqrestore(&cctx->lock, flags);
466 goto err_idr;
467 }
468 ctx->ctxid = ret << 4;
469 spin_unlock_irqrestore(&cctx->lock, flags);
470
471 kref_init(&ctx->refcount);
472
473 return ctx;
474err_idr:
475 spin_lock(&user->lock);
476 list_del(&ctx->node);
477 spin_unlock(&user->lock);
478 fastrpc_channel_ctx_put(cctx);
479 kfree(ctx->maps);
480 kfree(ctx->olaps);
481 kfree(ctx);
482
483 return ERR_PTR(ret);
484}
485
486static struct sg_table *
487fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
488 enum dma_data_direction dir)
489{
490 struct fastrpc_dma_buf_attachment *a = attachment->priv;
491 struct sg_table *table;
492
493 table = &a->sgt;
494
495 if (!dma_map_sg(attachment->dev, table->sgl, table->nents, dir))
496 return ERR_PTR(-ENOMEM);
497
498 return table;
499}
500
501static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
502 struct sg_table *table,
503 enum dma_data_direction dir)
504{
505 dma_unmap_sg(attach->dev, table->sgl, table->nents, dir);
506}
507
508static void fastrpc_release(struct dma_buf *dmabuf)
509{
510 struct fastrpc_buf *buffer = dmabuf->priv;
511
512 fastrpc_buf_free(buffer);
513}
514
515static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
516 struct dma_buf_attachment *attachment)
517{
518 struct fastrpc_dma_buf_attachment *a;
519 struct fastrpc_buf *buffer = dmabuf->priv;
520 int ret;
521
522 a = kzalloc(sizeof(*a), GFP_KERNEL);
523 if (!a)
524 return -ENOMEM;
525
526 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
527 FASTRPC_PHYS(buffer->phys), buffer->size);
528 if (ret < 0) {
529 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
530 kfree(a);
531 return -EINVAL;
532 }
533
534 a->dev = attachment->dev;
535 INIT_LIST_HEAD(&a->node);
536 attachment->priv = a;
537
538 mutex_lock(&buffer->lock);
539 list_add(&a->node, &buffer->attachments);
540 mutex_unlock(&buffer->lock);
541
542 return 0;
543}
544
545static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
546 struct dma_buf_attachment *attachment)
547{
548 struct fastrpc_dma_buf_attachment *a = attachment->priv;
549 struct fastrpc_buf *buffer = dmabuf->priv;
550
551 mutex_lock(&buffer->lock);
552 list_del(&a->node);
553 mutex_unlock(&buffer->lock);
554 sg_free_table(&a->sgt);
555 kfree(a);
556}
557
558static void *fastrpc_kmap(struct dma_buf *dmabuf, unsigned long pgnum)
559{
560 struct fastrpc_buf *buf = dmabuf->priv;
561
562 return buf->virt ? buf->virt + pgnum * PAGE_SIZE : NULL;
563}
564
565static void *fastrpc_vmap(struct dma_buf *dmabuf)
566{
567 struct fastrpc_buf *buf = dmabuf->priv;
568
569 return buf->virt;
570}
571
572static int fastrpc_mmap(struct dma_buf *dmabuf,
573 struct vm_area_struct *vma)
574{
575 struct fastrpc_buf *buf = dmabuf->priv;
576 size_t size = vma->vm_end - vma->vm_start;
577
578 return dma_mmap_coherent(buf->dev, vma, buf->virt,
579 FASTRPC_PHYS(buf->phys), size);
580}
581
582static const struct dma_buf_ops fastrpc_dma_buf_ops = {
583 .attach = fastrpc_dma_buf_attach,
584 .detach = fastrpc_dma_buf_detatch,
585 .map_dma_buf = fastrpc_map_dma_buf,
586 .unmap_dma_buf = fastrpc_unmap_dma_buf,
587 .mmap = fastrpc_mmap,
588 .map = fastrpc_kmap,
589 .vmap = fastrpc_vmap,
590 .release = fastrpc_release,
591};
592
593static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
594 u64 len, struct fastrpc_map **ppmap)
595{
596 struct fastrpc_session_ctx *sess = fl->sctx;
597 struct fastrpc_map *map = NULL;
598 int err = 0;
599
600 if (!fastrpc_map_find(fl, fd, ppmap))
601 return 0;
602
603 map = kzalloc(sizeof(*map), GFP_KERNEL);
604 if (!map)
605 return -ENOMEM;
606
607 INIT_LIST_HEAD(&map->node);
608 map->fl = fl;
609 map->fd = fd;
610 map->buf = dma_buf_get(fd);
611 if (IS_ERR(map->buf)) {
612 err = PTR_ERR(map->buf);
613 goto get_err;
614 }
615
616 map->attach = dma_buf_attach(map->buf, sess->dev);
617 if (IS_ERR(map->attach)) {
618 dev_err(sess->dev, "Failed to attach dmabuf\n");
619 err = PTR_ERR(map->attach);
620 goto attach_err;
621 }
622
623 map->table = dma_buf_map_attachment(map->attach, DMA_BIDIRECTIONAL);
624 if (IS_ERR(map->table)) {
625 err = PTR_ERR(map->table);
626 goto map_err;
627 }
628
629 map->phys = sg_dma_address(map->table->sgl);
630 map->phys += ((u64)fl->sctx->sid << 32);
631 map->size = len;
632 map->va = sg_virt(map->table->sgl);
633 map->len = len;
634 kref_init(&map->refcount);
635
636 spin_lock(&fl->lock);
637 list_add_tail(&map->node, &fl->maps);
638 spin_unlock(&fl->lock);
639 *ppmap = map;
640
641 return 0;
642
643map_err:
644 dma_buf_detach(map->buf, map->attach);
645attach_err:
646 dma_buf_put(map->buf);
647get_err:
648 kfree(map);
649
650 return err;
651}
652
653/*
654 * Fastrpc payload buffer with metadata looks like:
655 *
656 * >>>>>> START of METADATA <<<<<<<<<
657 * +---------------------------------+
658 * | Arguments |
659 * | type:(struct fastrpc_remote_arg)|
660 * | (0 - N) |
661 * +---------------------------------+
662 * | Invoke Buffer list |
663 * | type:(struct fastrpc_invoke_buf)|
664 * | (0 - N) |
665 * +---------------------------------+
666 * | Page info list |
667 * | type:(struct fastrpc_phy_page) |
668 * | (0 - N) |
669 * +---------------------------------+
670 * | Optional info |
671 * |(can be specific to SoC/Firmware)|
672 * +---------------------------------+
673 * >>>>>>>> END of METADATA <<<<<<<<<
674 * +---------------------------------+
675 * | Inline ARGS |
676 * | (0-N) |
677 * +---------------------------------+
678 */
679
680static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
681{
682 int size = 0;
683
684 size = (sizeof(struct fastrpc_remote_arg) +
685 sizeof(struct fastrpc_invoke_buf) +
686 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
687 sizeof(u64) * FASTRPC_MAX_FDLIST +
688 sizeof(u32) * FASTRPC_MAX_CRCLIST;
689
690 return size;
691}
692
693static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
694{
695 u64 size = 0;
696 int i;
697
698 size = ALIGN(metalen, FASTRPC_ALIGN);
699 for (i = 0; i < ctx->nscalars; i++) {
700 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
701
702 if (ctx->olaps[i].offset == 0)
703 size = ALIGN(size, FASTRPC_ALIGN);
704
705 size += (ctx->olaps[i].mend - ctx->olaps[i].mstart);
706 }
707 }
708
709 return size;
710}
711
712static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
713{
714 struct device *dev = ctx->fl->sctx->dev;
715 int i, err;
716
717 for (i = 0; i < ctx->nscalars; ++i) {
718 /* Make sure reserved field is set to 0 */
719 if (ctx->args[i].reserved)
720 return -EINVAL;
721
722 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
723 ctx->args[i].length == 0)
724 continue;
725
726 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
727 ctx->args[i].length, &ctx->maps[i]);
728 if (err) {
729 dev_err(dev, "Error Creating map %d\n", err);
730 return -EINVAL;
731 }
732
733 }
734 return 0;
735}
736
737static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
738{
739 struct device *dev = ctx->fl->sctx->dev;
740 struct fastrpc_remote_arg *rpra;
741 struct fastrpc_invoke_buf *list;
742 struct fastrpc_phy_page *pages;
743 int inbufs, i, oix, err = 0;
744 u64 len, rlen, pkt_size;
745 u64 pg_start, pg_end;
746 uintptr_t args;
747 int metalen;
748
749 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
750 metalen = fastrpc_get_meta_size(ctx);
751 pkt_size = fastrpc_get_payload_size(ctx, metalen);
752
753 err = fastrpc_create_maps(ctx);
754 if (err)
755 return err;
756
757 ctx->msg_sz = pkt_size;
758
759 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
760 if (err)
761 return err;
762
763 rpra = ctx->buf->virt;
764 list = ctx->buf->virt + ctx->nscalars * sizeof(*rpra);
765 pages = ctx->buf->virt + ctx->nscalars * (sizeof(*list) +
766 sizeof(*rpra));
767 args = (uintptr_t)ctx->buf->virt + metalen;
768 rlen = pkt_size - metalen;
769 ctx->rpra = rpra;
770
771 for (oix = 0; oix < ctx->nbufs; ++oix) {
772 int mlen;
773
774 i = ctx->olaps[oix].raix;
775 len = ctx->args[i].length;
776
777 rpra[i].pv = 0;
778 rpra[i].len = len;
779 list[i].num = len ? 1 : 0;
780 list[i].pgidx = i;
781
782 if (!len)
783 continue;
784
785 if (ctx->maps[i]) {
786 struct vm_area_struct *vma = NULL;
787
788 rpra[i].pv = (u64) ctx->args[i].ptr;
789 pages[i].addr = ctx->maps[i]->phys;
790
791 vma = find_vma(current->mm, ctx->args[i].ptr);
792 if (vma)
793 pages[i].addr += ctx->args[i].ptr -
794 vma->vm_start;
795
796 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
797 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
798 PAGE_SHIFT;
799 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
800
801 } else {
802
803 if (ctx->olaps[oix].offset == 0) {
804 rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
805 args = ALIGN(args, FASTRPC_ALIGN);
806 }
807
808 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
809
810 if (rlen < mlen)
811 goto bail;
812
813 rpra[i].pv = args - ctx->olaps[oix].offset;
814 pages[i].addr = ctx->buf->phys -
815 ctx->olaps[oix].offset +
816 (pkt_size - rlen);
817 pages[i].addr = pages[i].addr & PAGE_MASK;
818
819 pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
820 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
821 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
822 args = args + mlen;
823 rlen -= mlen;
824 }
825
826 if (i < inbufs && !ctx->maps[i]) {
827 void *dst = (void *)(uintptr_t)rpra[i].pv;
828 void *src = (void *)(uintptr_t)ctx->args[i].ptr;
829
830 if (!kernel) {
831 if (copy_from_user(dst, (void __user *)src,
832 len)) {
833 err = -EFAULT;
834 goto bail;
835 }
836 } else {
837 memcpy(dst, src, len);
838 }
839 }
840 }
841
842 for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
843 rpra[i].pv = (u64) ctx->args[i].ptr;
844 rpra[i].len = ctx->args[i].length;
845 list[i].num = ctx->args[i].length ? 1 : 0;
846 list[i].pgidx = i;
847 pages[i].addr = ctx->maps[i]->phys;
848 pages[i].size = ctx->maps[i]->size;
849 }
850
851bail:
852 if (err)
853 dev_err(dev, "Error: get invoke args failed:%d\n", err);
854
855 return err;
856}
857
858static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
859 u32 kernel)
860{
861 struct fastrpc_remote_arg *rpra = ctx->rpra;
862 int i, inbufs;
863
864 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
865
866 for (i = inbufs; i < ctx->nbufs; ++i) {
867 void *src = (void *)(uintptr_t)rpra[i].pv;
868 void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
869 u64 len = rpra[i].len;
870
871 if (!kernel) {
872 if (copy_to_user((void __user *)dst, src, len))
873 return -EFAULT;
874 } else {
875 memcpy(dst, src, len);
876 }
877 }
878
879 return 0;
880}
881
882static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
883 struct fastrpc_invoke_ctx *ctx,
884 u32 kernel, uint32_t handle)
885{
886 struct fastrpc_channel_ctx *cctx;
887 struct fastrpc_user *fl = ctx->fl;
888 struct fastrpc_msg *msg = &ctx->msg;
889
890 cctx = fl->cctx;
891 msg->pid = fl->tgid;
892 msg->tid = current->pid;
893
894 if (kernel)
895 msg->pid = 0;
896
897 msg->ctx = ctx->ctxid | fl->pd;
898 msg->handle = handle;
899 msg->sc = ctx->sc;
900 msg->addr = ctx->buf ? ctx->buf->phys : 0;
901 msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
902 fastrpc_context_get(ctx);
903
904 return rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
905}
906
907static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
908 u32 handle, u32 sc,
909 struct fastrpc_invoke_args *args)
910{
911 struct fastrpc_invoke_ctx *ctx = NULL;
912 int err = 0;
913
914 if (!fl->sctx)
915 return -EINVAL;
916
917 if (!fl->cctx->rpdev)
918 return -EPIPE;
919
920 ctx = fastrpc_context_alloc(fl, kernel, sc, args);
921 if (IS_ERR(ctx))
922 return PTR_ERR(ctx);
923
924 if (ctx->nscalars) {
925 err = fastrpc_get_args(kernel, ctx);
926 if (err)
927 goto bail;
928 }
929
930 /* make sure that all CPU memory writes are seen by DSP */
931 dma_wmb();
932 /* Send invoke buffer to remote dsp */
933 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
934 if (err)
935 goto bail;
936
937 /* Wait for remote dsp to respond or time out */
938 err = wait_for_completion_interruptible(&ctx->work);
939 if (err)
940 goto bail;
941
942 /* Check the response from remote dsp */
943 err = ctx->retval;
944 if (err)
945 goto bail;
946
947 if (ctx->nscalars) {
948 /* make sure that all memory writes by DSP are seen by CPU */
949 dma_rmb();
950 /* populate all the output buffers with results */
951 err = fastrpc_put_args(ctx, kernel);
952 if (err)
953 goto bail;
954 }
955
956bail:
957 /* We are done with this compute context, remove it from pending list */
958 spin_lock(&fl->lock);
959 list_del(&ctx->node);
960 spin_unlock(&fl->lock);
961 fastrpc_context_put(ctx);
962
963 if (err)
964 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
965
966 return err;
967}
968
969static int fastrpc_init_create_process(struct fastrpc_user *fl,
970 char __user *argp)
971{
972 struct fastrpc_init_create init;
973 struct fastrpc_invoke_args *args;
974 struct fastrpc_phy_page pages[1];
975 struct fastrpc_map *map = NULL;
976 struct fastrpc_buf *imem = NULL;
977 int memlen;
978 int err;
979 struct {
980 int pgid;
981 u32 namelen;
982 u32 filelen;
983 u32 pageslen;
984 u32 attrs;
985 u32 siglen;
986 } inbuf;
987 u32 sc;
988
989 args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
990 if (!args)
991 return -ENOMEM;
992
993 if (copy_from_user(&init, argp, sizeof(init))) {
994 err = -EFAULT;
995 goto err;
996 }
997
998 if (init.filelen > INIT_FILELEN_MAX) {
999 err = -EINVAL;
1000 goto err;
1001 }
1002
1003 inbuf.pgid = fl->tgid;
1004 inbuf.namelen = strlen(current->comm) + 1;
1005 inbuf.filelen = init.filelen;
1006 inbuf.pageslen = 1;
1007 inbuf.attrs = init.attrs;
1008 inbuf.siglen = init.siglen;
1009 fl->pd = 1;
1010
1011 if (init.filelen && init.filefd) {
1012 err = fastrpc_map_create(fl, init.filefd, init.filelen, &map);
1013 if (err)
1014 goto err;
1015 }
1016
1017 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1018 1024 * 1024);
1019 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1020 &imem);
1021 if (err)
1022 goto err_alloc;
1023
1024 fl->init_mem = imem;
1025 args[0].ptr = (u64)(uintptr_t)&inbuf;
1026 args[0].length = sizeof(inbuf);
1027 args[0].fd = -1;
1028
1029 args[1].ptr = (u64)(uintptr_t)current->comm;
1030 args[1].length = inbuf.namelen;
1031 args[1].fd = -1;
1032
1033 args[2].ptr = (u64) init.file;
1034 args[2].length = inbuf.filelen;
1035 args[2].fd = init.filefd;
1036
1037 pages[0].addr = imem->phys;
1038 pages[0].size = imem->size;
1039
1040 args[3].ptr = (u64)(uintptr_t) pages;
1041 args[3].length = 1 * sizeof(*pages);
1042 args[3].fd = -1;
1043
1044 args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1045 args[4].length = sizeof(inbuf.attrs);
1046 args[4].fd = -1;
1047
1048 args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1049 args[5].length = sizeof(inbuf.siglen);
1050 args[5].fd = -1;
1051
1052 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1053 if (init.attrs)
1054 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 6, 0);
1055
1056 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1057 sc, args);
1058 if (err)
1059 goto err_invoke;
1060
1061 kfree(args);
1062
1063 return 0;
1064
1065err_invoke:
1066 fl->init_mem = NULL;
1067 fastrpc_buf_free(imem);
1068err_alloc:
1069 if (map) {
1070 spin_lock(&fl->lock);
1071 list_del(&map->node);
1072 spin_unlock(&fl->lock);
1073 fastrpc_map_put(map);
1074 }
1075err:
1076 kfree(args);
1077
1078 return err;
1079}
1080
1081static struct fastrpc_session_ctx *fastrpc_session_alloc(
1082 struct fastrpc_channel_ctx *cctx)
1083{
1084 struct fastrpc_session_ctx *session = NULL;
1085 unsigned long flags;
1086 int i;
1087
1088 spin_lock_irqsave(&cctx->lock, flags);
1089 for (i = 0; i < cctx->sesscount; i++) {
1090 if (!cctx->session[i].used && cctx->session[i].valid) {
1091 cctx->session[i].used = true;
1092 session = &cctx->session[i];
1093 break;
1094 }
1095 }
1096 spin_unlock_irqrestore(&cctx->lock, flags);
1097
1098 return session;
1099}
1100
1101static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1102 struct fastrpc_session_ctx *session)
1103{
1104 unsigned long flags;
1105
1106 spin_lock_irqsave(&cctx->lock, flags);
1107 session->used = false;
1108 spin_unlock_irqrestore(&cctx->lock, flags);
1109}
1110
1111static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1112{
1113 struct fastrpc_invoke_args args[1];
1114 int tgid = 0;
1115 u32 sc;
1116
1117 tgid = fl->tgid;
1118 args[0].ptr = (u64)(uintptr_t) &tgid;
1119 args[0].length = sizeof(tgid);
1120 args[0].fd = -1;
1121 args[0].reserved = 0;
1122 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1123
1124 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1125 sc, &args[0]);
1126}
1127
1128static int fastrpc_device_release(struct inode *inode, struct file *file)
1129{
1130 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1131 struct fastrpc_channel_ctx *cctx = fl->cctx;
1132 struct fastrpc_invoke_ctx *ctx, *n;
1133 struct fastrpc_map *map, *m;
1134 unsigned long flags;
1135
1136 fastrpc_release_current_dsp_process(fl);
1137
1138 spin_lock_irqsave(&cctx->lock, flags);
1139 list_del(&fl->user);
1140 spin_unlock_irqrestore(&cctx->lock, flags);
1141
1142 if (fl->init_mem)
1143 fastrpc_buf_free(fl->init_mem);
1144
1145 list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1146 list_del(&ctx->node);
1147 fastrpc_context_put(ctx);
1148 }
1149
1150 list_for_each_entry_safe(map, m, &fl->maps, node) {
1151 list_del(&map->node);
1152 fastrpc_map_put(map);
1153 }
1154
1155 fastrpc_session_free(cctx, fl->sctx);
1156 fastrpc_channel_ctx_put(cctx);
1157
1158 mutex_destroy(&fl->mutex);
1159 kfree(fl);
1160 file->private_data = NULL;
1161
1162 return 0;
1163}
1164
1165static int fastrpc_device_open(struct inode *inode, struct file *filp)
1166{
1167 struct fastrpc_channel_ctx *cctx = miscdev_to_cctx(filp->private_data);
1168 struct fastrpc_user *fl = NULL;
1169 unsigned long flags;
1170
1171 fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1172 if (!fl)
1173 return -ENOMEM;
1174
1175 /* Released in fastrpc_device_release() */
1176 fastrpc_channel_ctx_get(cctx);
1177
1178 filp->private_data = fl;
1179 spin_lock_init(&fl->lock);
1180 mutex_init(&fl->mutex);
1181 INIT_LIST_HEAD(&fl->pending);
1182 INIT_LIST_HEAD(&fl->maps);
1183 INIT_LIST_HEAD(&fl->user);
1184 fl->tgid = current->tgid;
1185 fl->cctx = cctx;
1186
1187 fl->sctx = fastrpc_session_alloc(cctx);
1188 if (!fl->sctx) {
1189 dev_err(&cctx->rpdev->dev, "No session available\n");
1190 mutex_destroy(&fl->mutex);
1191 kfree(fl);
1192
1193 return -EBUSY;
1194 }
1195
1196 spin_lock_irqsave(&cctx->lock, flags);
1197 list_add_tail(&fl->user, &cctx->users);
1198 spin_unlock_irqrestore(&cctx->lock, flags);
1199
1200 return 0;
1201}
1202
1203static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1204{
1205 struct fastrpc_alloc_dma_buf bp;
1206 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1207 struct fastrpc_buf *buf = NULL;
1208 int err;
1209
1210 if (copy_from_user(&bp, argp, sizeof(bp)))
1211 return -EFAULT;
1212
1213 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1214 if (err)
1215 return err;
1216 exp_info.ops = &fastrpc_dma_buf_ops;
1217 exp_info.size = bp.size;
1218 exp_info.flags = O_RDWR;
1219 exp_info.priv = buf;
1220 buf->dmabuf = dma_buf_export(&exp_info);
1221 if (IS_ERR(buf->dmabuf)) {
1222 err = PTR_ERR(buf->dmabuf);
1223 fastrpc_buf_free(buf);
1224 return err;
1225 }
1226
1227 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1228 if (bp.fd < 0) {
1229 dma_buf_put(buf->dmabuf);
1230 return -EINVAL;
1231 }
1232
1233 if (copy_to_user(argp, &bp, sizeof(bp))) {
1234 dma_buf_put(buf->dmabuf);
1235 return -EFAULT;
1236 }
1237
1238 return 0;
1239}
1240
1241static int fastrpc_init_attach(struct fastrpc_user *fl)
1242{
1243 struct fastrpc_invoke_args args[1];
1244 int tgid = fl->tgid;
1245 u32 sc;
1246
1247 args[0].ptr = (u64)(uintptr_t) &tgid;
1248 args[0].length = sizeof(tgid);
1249 args[0].fd = -1;
1250 args[0].reserved = 0;
1251 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1252 fl->pd = 0;
1253
1254 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1255 sc, &args[0]);
1256}
1257
1258static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1259{
1260 struct fastrpc_invoke_args *args = NULL;
1261 struct fastrpc_invoke inv;
1262 u32 nscalars;
1263 int err;
1264
1265 if (copy_from_user(&inv, argp, sizeof(inv)))
1266 return -EFAULT;
1267
1268 /* nscalars is truncated here to max supported value */
1269 nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1270 if (nscalars) {
1271 args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1272 if (!args)
1273 return -ENOMEM;
1274
1275 if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1276 nscalars * sizeof(*args))) {
1277 kfree(args);
1278 return -EFAULT;
1279 }
1280 }
1281
1282 err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1283 kfree(args);
1284
1285 return err;
1286}
1287
1288static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
1289 unsigned long arg)
1290{
1291 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1292 char __user *argp = (char __user *)arg;
1293 int err;
1294
1295 switch (cmd) {
1296 case FASTRPC_IOCTL_INVOKE:
1297 err = fastrpc_invoke(fl, argp);
1298 break;
1299 case FASTRPC_IOCTL_INIT_ATTACH:
1300 err = fastrpc_init_attach(fl);
1301 break;
1302 case FASTRPC_IOCTL_INIT_CREATE:
1303 err = fastrpc_init_create_process(fl, argp);
1304 break;
1305 case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
1306 err = fastrpc_dmabuf_alloc(fl, argp);
1307 break;
1308 default:
1309 err = -ENOTTY;
1310 break;
1311 }
1312
1313 return err;
1314}
1315
1316static const struct file_operations fastrpc_fops = {
1317 .open = fastrpc_device_open,
1318 .release = fastrpc_device_release,
1319 .unlocked_ioctl = fastrpc_device_ioctl,
1320 .compat_ioctl = fastrpc_device_ioctl,
1321};
1322
1323static int fastrpc_cb_probe(struct platform_device *pdev)
1324{
1325 struct fastrpc_channel_ctx *cctx;
1326 struct fastrpc_session_ctx *sess;
1327 struct device *dev = &pdev->dev;
1328 int i, sessions = 0;
1329 unsigned long flags;
1330 int rc;
1331
1332 cctx = dev_get_drvdata(dev->parent);
1333 if (!cctx)
1334 return -EINVAL;
1335
1336 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
1337
1338 spin_lock_irqsave(&cctx->lock, flags);
1339 sess = &cctx->session[cctx->sesscount];
1340 sess->used = false;
1341 sess->valid = true;
1342 sess->dev = dev;
1343 dev_set_drvdata(dev, sess);
1344
1345 if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
1346 dev_info(dev, "FastRPC Session ID not specified in DT\n");
1347
1348 if (sessions > 0) {
1349 struct fastrpc_session_ctx *dup_sess;
1350
1351 for (i = 1; i < sessions; i++) {
1352 if (cctx->sesscount++ >= FASTRPC_MAX_SESSIONS)
1353 break;
1354 dup_sess = &cctx->session[cctx->sesscount];
1355 memcpy(dup_sess, sess, sizeof(*dup_sess));
1356 }
1357 }
1358 cctx->sesscount++;
1359 spin_unlock_irqrestore(&cctx->lock, flags);
1360 rc = dma_set_mask(dev, DMA_BIT_MASK(32));
1361 if (rc) {
1362 dev_err(dev, "32-bit DMA enable failed\n");
1363 return rc;
1364 }
1365
1366 return 0;
1367}
1368
1369static int fastrpc_cb_remove(struct platform_device *pdev)
1370{
1371 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
1372 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
1373 unsigned long flags;
1374 int i;
1375
1376 spin_lock_irqsave(&cctx->lock, flags);
1377 for (i = 1; i < FASTRPC_MAX_SESSIONS; i++) {
1378 if (cctx->session[i].sid == sess->sid) {
1379 cctx->session[i].valid = false;
1380 cctx->sesscount--;
1381 }
1382 }
1383 spin_unlock_irqrestore(&cctx->lock, flags);
1384
1385 return 0;
1386}
1387
1388static const struct of_device_id fastrpc_match_table[] = {
1389 { .compatible = "qcom,fastrpc-compute-cb", },
1390 {}
1391};
1392
1393static struct platform_driver fastrpc_cb_driver = {
1394 .probe = fastrpc_cb_probe,
1395 .remove = fastrpc_cb_remove,
1396 .driver = {
1397 .name = "qcom,fastrpc-cb",
1398 .of_match_table = fastrpc_match_table,
1399 .suppress_bind_attrs = true,
1400 },
1401};
1402
1403static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
1404{
1405 struct device *rdev = &rpdev->dev;
1406 struct fastrpc_channel_ctx *data;
1407 int i, err, domain_id = -1;
1408 const char *domain;
1409
1410 err = of_property_read_string(rdev->of_node, "label", &domain);
1411 if (err) {
1412 dev_info(rdev, "FastRPC Domain not specified in DT\n");
1413 return err;
1414 }
1415
1416 for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
1417 if (!strcmp(domains[i], domain)) {
1418 domain_id = i;
1419 break;
1420 }
1421 }
1422
1423 if (domain_id < 0) {
1424 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
1425 return -EINVAL;
1426 }
1427
1428 data = kzalloc(sizeof(*data), GFP_KERNEL);
1429 if (!data)
1430 return -ENOMEM;
1431
1432 data->miscdev.minor = MISC_DYNAMIC_MINOR;
1433 data->miscdev.name = kasprintf(GFP_KERNEL, "fastrpc-%s",
1434 domains[domain_id]);
1435 data->miscdev.fops = &fastrpc_fops;
1436 err = misc_register(&data->miscdev);
1437 if (err)
1438 return err;
1439
1440 kref_init(&data->refcount);
1441
1442 dev_set_drvdata(&rpdev->dev, data);
1443 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
1444 INIT_LIST_HEAD(&data->users);
1445 spin_lock_init(&data->lock);
1446 idr_init(&data->ctx_idr);
1447 data->domain_id = domain_id;
1448 data->rpdev = rpdev;
1449
1450 return of_platform_populate(rdev->of_node, NULL, NULL, rdev);
1451}
1452
1453static void fastrpc_notify_users(struct fastrpc_user *user)
1454{
1455 struct fastrpc_invoke_ctx *ctx;
1456
1457 spin_lock(&user->lock);
1458 list_for_each_entry(ctx, &user->pending, node)
1459 complete(&ctx->work);
1460 spin_unlock(&user->lock);
1461}
1462
1463static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
1464{
1465 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1466 struct fastrpc_user *user;
1467 unsigned long flags;
1468
1469 spin_lock_irqsave(&cctx->lock, flags);
1470 list_for_each_entry(user, &cctx->users, user)
1471 fastrpc_notify_users(user);
1472 spin_unlock_irqrestore(&cctx->lock, flags);
1473
1474 misc_deregister(&cctx->miscdev);
1475 of_platform_depopulate(&rpdev->dev);
1476
1477 cctx->rpdev = NULL;
1478 fastrpc_channel_ctx_put(cctx);
1479}
1480
1481static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
1482 int len, void *priv, u32 addr)
1483{
1484 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
1485 struct fastrpc_invoke_rsp *rsp = data;
1486 struct fastrpc_invoke_ctx *ctx;
1487 unsigned long flags;
1488 unsigned long ctxid;
1489
1490 if (len < sizeof(*rsp))
1491 return -EINVAL;
1492
1493 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
1494
1495 spin_lock_irqsave(&cctx->lock, flags);
1496 ctx = idr_find(&cctx->ctx_idr, ctxid);
1497 spin_unlock_irqrestore(&cctx->lock, flags);
1498
1499 if (!ctx) {
1500 dev_err(&rpdev->dev, "No context ID matches response\n");
1501 return -ENOENT;
1502 }
1503
1504 ctx->retval = rsp->retval;
1505 complete(&ctx->work);
1506
1507 /*
1508 * The DMA buffer associated with the context cannot be freed in
1509 * interrupt context so schedule it through a worker thread to
1510 * avoid a kernel BUG.
1511 */
1512 schedule_work(&ctx->put_work);
1513
1514 return 0;
1515}
1516
1517static const struct of_device_id fastrpc_rpmsg_of_match[] = {
1518 { .compatible = "qcom,fastrpc" },
1519 { },
1520};
1521MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
1522
1523static struct rpmsg_driver fastrpc_driver = {
1524 .probe = fastrpc_rpmsg_probe,
1525 .remove = fastrpc_rpmsg_remove,
1526 .callback = fastrpc_rpmsg_callback,
1527 .drv = {
1528 .name = "qcom,fastrpc",
1529 .of_match_table = fastrpc_rpmsg_of_match,
1530 },
1531};
1532
1533static int fastrpc_init(void)
1534{
1535 int ret;
1536
1537 ret = platform_driver_register(&fastrpc_cb_driver);
1538 if (ret < 0) {
1539 pr_err("fastrpc: failed to register cb driver\n");
1540 return ret;
1541 }
1542
1543 ret = register_rpmsg_driver(&fastrpc_driver);
1544 if (ret < 0) {
1545 pr_err("fastrpc: failed to register rpmsg driver\n");
1546 platform_driver_unregister(&fastrpc_cb_driver);
1547 return ret;
1548 }
1549
1550 return 0;
1551}
1552module_init(fastrpc_init);
1553
1554static void fastrpc_exit(void)
1555{
1556 platform_driver_unregister(&fastrpc_cb_driver);
1557 unregister_rpmsg_driver(&fastrpc_driver);
1558}
1559module_exit(fastrpc_exit);
1560
1561MODULE_LICENSE("GPL v2");