1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
   3// Copyright (c) 2018, Linaro Limited
   4
   5#include <linux/completion.h>
   6#include <linux/device.h>
   7#include <linux/dma-buf.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/dma-resv.h>
  10#include <linux/idr.h>
  11#include <linux/list.h>
  12#include <linux/miscdevice.h>
  13#include <linux/module.h>
  14#include <linux/of_address.h>
  15#include <linux/of.h>
  16#include <linux/platform_device.h>
  17#include <linux/sort.h>
  18#include <linux/of_platform.h>
  19#include <linux/rpmsg.h>
  20#include <linux/scatterlist.h>
  21#include <linux/slab.h>
  22#include <linux/firmware/qcom/qcom_scm.h>
  23#include <uapi/misc/fastrpc.h>
  24#include <linux/of_reserved_mem.h>
  25
  26#define ADSP_DOMAIN_ID (0)
  27#define MDSP_DOMAIN_ID (1)
  28#define SDSP_DOMAIN_ID (2)
  29#define CDSP_DOMAIN_ID (3)
  30#define FASTRPC_DEV_MAX		4 /* adsp, mdsp, slpi, cdsp*/
 
  31#define FASTRPC_MAX_SESSIONS	14
  32#define FASTRPC_MAX_VMIDS	16
  33#define FASTRPC_ALIGN		128
  34#define FASTRPC_MAX_FDLIST	16
  35#define FASTRPC_MAX_CRCLIST	64
  36#define FASTRPC_PHYS(p)	((p) & 0xffffffff)
  37#define FASTRPC_CTX_MAX (256)
  38#define FASTRPC_INIT_HANDLE	1
  39#define FASTRPC_DSP_UTILITIES_HANDLE	2
  40#define FASTRPC_CTXID_MASK (0xFF0)
  41#define INIT_FILELEN_MAX (2 * 1024 * 1024)
  42#define INIT_FILE_NAMELEN_MAX (128)
  43#define FASTRPC_DEVICE_NAME	"fastrpc"
  44
  45/* Add memory to static PD pool, protection thru XPU */
  46#define ADSP_MMAP_HEAP_ADDR  4
  47/* MAP static DMA buffer on DSP User PD */
  48#define ADSP_MMAP_DMA_BUFFER  6
  49/* Add memory to static PD pool protection thru hypervisor */
  50#define ADSP_MMAP_REMOTE_HEAP_ADDR  8
  51/* Add memory to userPD pool, for user heap */
  52#define ADSP_MMAP_ADD_PAGES 0x1000
  53/* Add memory to userPD pool, for LLC heap */
  54#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
  55
  56#define DSP_UNSUPPORTED_API (0x80000414)
  57/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
  58#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
  59#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
  60
  61/* Retrives number of input buffers from the scalars parameter */
  62#define REMOTE_SCALARS_INBUFS(sc)	(((sc) >> 16) & 0x0ff)
  63
  64/* Retrives number of output buffers from the scalars parameter */
  65#define REMOTE_SCALARS_OUTBUFS(sc)	(((sc) >> 8) & 0x0ff)
  66
  67/* Retrives number of input handles from the scalars parameter */
  68#define REMOTE_SCALARS_INHANDLES(sc)	(((sc) >> 4) & 0x0f)
  69
  70/* Retrives number of output handles from the scalars parameter */
  71#define REMOTE_SCALARS_OUTHANDLES(sc)	((sc) & 0x0f)
  72
  73#define REMOTE_SCALARS_LENGTH(sc)	(REMOTE_SCALARS_INBUFS(sc) +   \
  74					 REMOTE_SCALARS_OUTBUFS(sc) +  \
  75					 REMOTE_SCALARS_INHANDLES(sc)+ \
  76					 REMOTE_SCALARS_OUTHANDLES(sc))
  77#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout)  \
  78				(((attr & 0x07) << 29) |		\
  79				((method & 0x1f) << 24) |	\
  80				((in & 0xff) << 16) |		\
  81				((out & 0xff) <<  8) |		\
  82				((oin & 0x0f) <<  4) |		\
  83				(oout & 0x0f))
  84
  85#define FASTRPC_SCALARS(method, in, out) \
  86		FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
  87
  88#define FASTRPC_CREATE_PROCESS_NARGS	6
  89#define FASTRPC_CREATE_STATIC_PROCESS_NARGS	3
  90/* Remote Method id table */
  91#define FASTRPC_RMID_INIT_ATTACH	0
  92#define FASTRPC_RMID_INIT_RELEASE	1
  93#define FASTRPC_RMID_INIT_MMAP		4
  94#define FASTRPC_RMID_INIT_MUNMAP	5
  95#define FASTRPC_RMID_INIT_CREATE	6
  96#define FASTRPC_RMID_INIT_CREATE_ATTR	7
  97#define FASTRPC_RMID_INIT_CREATE_STATIC	8
  98#define FASTRPC_RMID_INIT_MEM_MAP      10
  99#define FASTRPC_RMID_INIT_MEM_UNMAP    11
 100
 101/* Protection Domain(PD) ids */
 102#define ROOT_PD		(0)
 103#define USER_PD		(1)
 104#define SENSORS_PD	(2)
 105
 106#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
 107
 108static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
 109						"sdsp", "cdsp"};
 110struct fastrpc_phy_page {
 111	u64 addr;		/* physical address */
 112	u64 size;		/* size of contiguous region */
 113};
 114
 115struct fastrpc_invoke_buf {
 116	u32 num;		/* number of contiguous regions */
 117	u32 pgidx;		/* index to start of contiguous region */
 118};
 119
 120struct fastrpc_remote_dmahandle {
 121	s32 fd;		/* dma handle fd */
 122	u32 offset;	/* dma handle offset */
 123	u32 len;	/* dma handle length */
 124};
 125
 126struct fastrpc_remote_buf {
 127	u64 pv;		/* buffer pointer */
 128	u64 len;	/* length of buffer */
 129};
 130
 131union fastrpc_remote_arg {
 132	struct fastrpc_remote_buf buf;
 133	struct fastrpc_remote_dmahandle dma;
 134};
 135
 136struct fastrpc_mmap_rsp_msg {
 137	u64 vaddr;
 138};
 139
 140struct fastrpc_mmap_req_msg {
 141	s32 pgid;
 142	u32 flags;
 143	u64 vaddr;
 144	s32 num;
 145};
 146
 147struct fastrpc_mem_map_req_msg {
 148	s32 pgid;
 149	s32 fd;
 150	s32 offset;
 151	u32 flags;
 152	u64 vaddrin;
 153	s32 num;
 154	s32 data_len;
 155};
 156
 157struct fastrpc_munmap_req_msg {
 158	s32 pgid;
 159	u64 vaddr;
 160	u64 size;
 161};
 162
 163struct fastrpc_mem_unmap_req_msg {
 164	s32 pgid;
 165	s32 fd;
 166	u64 vaddrin;
 167	u64 len;
 168};
 169
 170struct fastrpc_msg {
 171	int pid;		/* process group id */
 172	int tid;		/* thread id */
 173	u64 ctx;		/* invoke caller context */
 174	u32 handle;	/* handle to invoke */
 175	u32 sc;		/* scalars structure describing the data */
 176	u64 addr;		/* physical address */
 177	u64 size;		/* size of contiguous region */
 178};
 179
 180struct fastrpc_invoke_rsp {
 181	u64 ctx;		/* invoke caller context */
 182	int retval;		/* invoke return value */
 183};
 184
 185struct fastrpc_buf_overlap {
 186	u64 start;
 187	u64 end;
 188	int raix;
 189	u64 mstart;
 190	u64 mend;
 191	u64 offset;
 192};
 193
 194struct fastrpc_buf {
 195	struct fastrpc_user *fl;
 196	struct dma_buf *dmabuf;
 197	struct device *dev;
 198	void *virt;
 199	u64 phys;
 200	u64 size;
 201	/* Lock for dma buf attachments */
 202	struct mutex lock;
 203	struct list_head attachments;
 204	/* mmap support */
 205	struct list_head node; /* list of user requested mmaps */
 206	uintptr_t raddr;
 207};
 208
 209struct fastrpc_dma_buf_attachment {
 210	struct device *dev;
 211	struct sg_table sgt;
 212	struct list_head node;
 213};
 214
 215struct fastrpc_map {
 216	struct list_head node;
 217	struct fastrpc_user *fl;
 218	int fd;
 219	struct dma_buf *buf;
 220	struct sg_table *table;
 221	struct dma_buf_attachment *attach;
 222	u64 phys;
 223	u64 size;
 224	void *va;
 225	u64 len;
 226	u64 raddr;
 227	u32 attr;
 228	struct kref refcount;
 229};
 230
 231struct fastrpc_invoke_ctx {
 232	int nscalars;
 233	int nbufs;
 234	int retval;
 235	int pid;
 236	int tgid;
 237	u32 sc;
 238	u32 *crc;
 239	u64 ctxid;
 240	u64 msg_sz;
 241	struct kref refcount;
 242	struct list_head node; /* list of ctxs */
 243	struct completion work;
 244	struct work_struct put_work;
 245	struct fastrpc_msg msg;
 246	struct fastrpc_user *fl;
 247	union fastrpc_remote_arg *rpra;
 248	struct fastrpc_map **maps;
 249	struct fastrpc_buf *buf;
 250	struct fastrpc_invoke_args *args;
 251	struct fastrpc_buf_overlap *olaps;
 252	struct fastrpc_channel_ctx *cctx;
 253};
 254
 255struct fastrpc_session_ctx {
 256	struct device *dev;
 257	int sid;
 258	bool used;
 259	bool valid;
 260};
 261
 262struct fastrpc_channel_ctx {
 263	int domain_id;
 264	int sesscount;
 265	int vmcount;
 266	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
 267	struct rpmsg_device *rpdev;
 268	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
 269	spinlock_t lock;
 270	struct idr ctx_idr;
 271	struct list_head users;
 272	struct kref refcount;
 273	/* Flag if dsp attributes are cached */
 274	bool valid_attributes;
 275	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
 276	struct fastrpc_device *secure_fdevice;
 277	struct fastrpc_device *fdevice;
 278	struct fastrpc_buf *remote_heap;
 279	struct list_head invoke_interrupted_mmaps;
 280	bool secure;
 281	bool unsigned_support;
 282	u64 dma_mask;
 283};
 284
 285struct fastrpc_device {
 286	struct fastrpc_channel_ctx *cctx;
 287	struct miscdevice miscdev;
 288	bool secure;
 289};
 290
 291struct fastrpc_user {
 292	struct list_head user;
 293	struct list_head maps;
 294	struct list_head pending;
 295	struct list_head mmaps;
 296
 297	struct fastrpc_channel_ctx *cctx;
 298	struct fastrpc_session_ctx *sctx;
 299	struct fastrpc_buf *init_mem;
 300
 301	int tgid;
 302	int pd;
 303	bool is_secure_dev;
 304	/* Lock for lists */
 305	spinlock_t lock;
 306	/* lock for allocations */
 307	struct mutex mutex;
 308};
 309
 310static void fastrpc_free_map(struct kref *ref)
 311{
 312	struct fastrpc_map *map;
 313
 314	map = container_of(ref, struct fastrpc_map, refcount);
 315
 316	if (map->table) {
 317		if (map->attr & FASTRPC_ATTR_SECUREMAP) {
 318			struct qcom_scm_vmperm perm;
 319			int vmid = map->fl->cctx->vmperms[0].vmid;
 320			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
 321			int err = 0;
 322
 323			perm.vmid = QCOM_SCM_VMID_HLOS;
 324			perm.perm = QCOM_SCM_PERM_RWX;
 325			err = qcom_scm_assign_mem(map->phys, map->size,
 326				&src_perms, &perm, 1);
 327			if (err) {
 328				dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
 329						map->phys, map->size, err);
 330				return;
 331			}
 332		}
 333		dma_buf_unmap_attachment_unlocked(map->attach, map->table,
 334						  DMA_BIDIRECTIONAL);
 335		dma_buf_detach(map->buf, map->attach);
 336		dma_buf_put(map->buf);
 337	}
 338
 339	if (map->fl) {
 340		spin_lock(&map->fl->lock);
 341		list_del(&map->node);
 342		spin_unlock(&map->fl->lock);
 343		map->fl = NULL;
 344	}
 345
 346	kfree(map);
 347}
 348
 349static void fastrpc_map_put(struct fastrpc_map *map)
 350{
 351	if (map)
 352		kref_put(&map->refcount, fastrpc_free_map);
 353}
 354
 355static int fastrpc_map_get(struct fastrpc_map *map)
 356{
 357	if (!map)
 358		return -ENOENT;
 359
 360	return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;
 361}
 362
 363
 364static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
 365			    struct fastrpc_map **ppmap, bool take_ref)
 366{
 367	struct fastrpc_session_ctx *sess = fl->sctx;
 368	struct fastrpc_map *map = NULL;
 369	int ret = -ENOENT;
 370
 371	spin_lock(&fl->lock);
 372	list_for_each_entry(map, &fl->maps, node) {
 373		if (map->fd != fd)
 374			continue;
 375
 376		if (take_ref) {
 377			ret = fastrpc_map_get(map);
 378			if (ret) {
 379				dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",
 380					__func__, fd, ret);
 381				break;
 382			}
 383		}
 384
 385		*ppmap = map;
 386		ret = 0;
 387		break;
 388	}
 389	spin_unlock(&fl->lock);
 390
 391	return ret;
 392}
 393
 394static void fastrpc_buf_free(struct fastrpc_buf *buf)
 395{
 396	dma_free_coherent(buf->dev, buf->size, buf->virt,
 397			  FASTRPC_PHYS(buf->phys));
 398	kfree(buf);
 399}
 400
 401static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
 402			     u64 size, struct fastrpc_buf **obuf)
 403{
 404	struct fastrpc_buf *buf;
 405
 406	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
 407	if (!buf)
 408		return -ENOMEM;
 409
 410	INIT_LIST_HEAD(&buf->attachments);
 411	INIT_LIST_HEAD(&buf->node);
 412	mutex_init(&buf->lock);
 413
 414	buf->fl = fl;
 415	buf->virt = NULL;
 416	buf->phys = 0;
 417	buf->size = size;
 418	buf->dev = dev;
 419	buf->raddr = 0;
 420
 421	buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
 422				       GFP_KERNEL);
 423	if (!buf->virt) {
 424		mutex_destroy(&buf->lock);
 425		kfree(buf);
 426		return -ENOMEM;
 427	}
 428
 429	*obuf = buf;
 430
 431	return 0;
 432}
 433
 434static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
 435			     u64 size, struct fastrpc_buf **obuf)
 436{
 437	int ret;
 438	struct fastrpc_buf *buf;
 439
 440	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
 441	if (ret)
 442		return ret;
 443
 444	buf = *obuf;
 445
 446	if (fl->sctx && fl->sctx->sid)
 447		buf->phys += ((u64)fl->sctx->sid << 32);
 448
 449	return 0;
 450}
 451
 452static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
 453				     u64 size, struct fastrpc_buf **obuf)
 454{
 455	struct device *rdev = &fl->cctx->rpdev->dev;
 456
 457	return  __fastrpc_buf_alloc(fl, rdev, size, obuf);
 458}
 459
 460static void fastrpc_channel_ctx_free(struct kref *ref)
 461{
 462	struct fastrpc_channel_ctx *cctx;
 463
 464	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
 465
 466	kfree(cctx);
 467}
 468
 469static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
 470{
 471	kref_get(&cctx->refcount);
 472}
 473
 474static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
 475{
 476	kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
 477}
 478
 479static void fastrpc_context_free(struct kref *ref)
 480{
 481	struct fastrpc_invoke_ctx *ctx;
 482	struct fastrpc_channel_ctx *cctx;
 483	unsigned long flags;
 484	int i;
 485
 486	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
 487	cctx = ctx->cctx;
 488
 489	for (i = 0; i < ctx->nbufs; i++)
 490		fastrpc_map_put(ctx->maps[i]);
 491
 492	if (ctx->buf)
 493		fastrpc_buf_free(ctx->buf);
 494
 495	spin_lock_irqsave(&cctx->lock, flags);
 496	idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
 497	spin_unlock_irqrestore(&cctx->lock, flags);
 498
 499	kfree(ctx->maps);
 500	kfree(ctx->olaps);
 501	kfree(ctx);
 502
 503	fastrpc_channel_ctx_put(cctx);
 504}
 505
 506static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
 507{
 508	kref_get(&ctx->refcount);
 509}
 510
 511static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
 512{
 513	kref_put(&ctx->refcount, fastrpc_context_free);
 514}
 515
 516static void fastrpc_context_put_wq(struct work_struct *work)
 517{
 518	struct fastrpc_invoke_ctx *ctx =
 519			container_of(work, struct fastrpc_invoke_ctx, put_work);
 520
 521	fastrpc_context_put(ctx);
 522}
 523
 524#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
 525static int olaps_cmp(const void *a, const void *b)
 526{
 527	struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
 528	struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
 529	/* sort with lowest starting buffer first */
 530	int st = CMP(pa->start, pb->start);
 531	/* sort with highest ending buffer first */
 532	int ed = CMP(pb->end, pa->end);
 533
 534	return st == 0 ? ed : st;
 535}
 536
 537static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
 538{
 539	u64 max_end = 0;
 540	int i;
 541
 542	for (i = 0; i < ctx->nbufs; ++i) {
 543		ctx->olaps[i].start = ctx->args[i].ptr;
 544		ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
 545		ctx->olaps[i].raix = i;
 546	}
 547
 548	sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
 549
 550	for (i = 0; i < ctx->nbufs; ++i) {
 551		/* Falling inside previous range */
 552		if (ctx->olaps[i].start < max_end) {
 553			ctx->olaps[i].mstart = max_end;
 554			ctx->olaps[i].mend = ctx->olaps[i].end;
 555			ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
 556
 557			if (ctx->olaps[i].end > max_end) {
 558				max_end = ctx->olaps[i].end;
 559			} else {
 560				ctx->olaps[i].mend = 0;
 561				ctx->olaps[i].mstart = 0;
 562			}
 563
 564		} else  {
 565			ctx->olaps[i].mend = ctx->olaps[i].end;
 566			ctx->olaps[i].mstart = ctx->olaps[i].start;
 567			ctx->olaps[i].offset = 0;
 568			max_end = ctx->olaps[i].end;
 569		}
 570	}
 571}
 572
 573static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
 574			struct fastrpc_user *user, u32 kernel, u32 sc,
 575			struct fastrpc_invoke_args *args)
 576{
 577	struct fastrpc_channel_ctx *cctx = user->cctx;
 578	struct fastrpc_invoke_ctx *ctx = NULL;
 579	unsigned long flags;
 580	int ret;
 581
 582	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 583	if (!ctx)
 584		return ERR_PTR(-ENOMEM);
 585
 586	INIT_LIST_HEAD(&ctx->node);
 587	ctx->fl = user;
 588	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
 589	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
 590		     REMOTE_SCALARS_OUTBUFS(sc);
 591
 592	if (ctx->nscalars) {
 593		ctx->maps = kcalloc(ctx->nscalars,
 594				    sizeof(*ctx->maps), GFP_KERNEL);
 595		if (!ctx->maps) {
 596			kfree(ctx);
 597			return ERR_PTR(-ENOMEM);
 598		}
 599		ctx->olaps = kcalloc(ctx->nscalars,
 600				    sizeof(*ctx->olaps), GFP_KERNEL);
 601		if (!ctx->olaps) {
 602			kfree(ctx->maps);
 603			kfree(ctx);
 604			return ERR_PTR(-ENOMEM);
 605		}
 606		ctx->args = args;
 607		fastrpc_get_buff_overlaps(ctx);
 608	}
 609
 610	/* Released in fastrpc_context_put() */
 611	fastrpc_channel_ctx_get(cctx);
 612
 613	ctx->sc = sc;
 614	ctx->retval = -1;
 615	ctx->pid = current->pid;
 616	ctx->tgid = user->tgid;
 617	ctx->cctx = cctx;
 618	init_completion(&ctx->work);
 619	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
 620
 621	spin_lock(&user->lock);
 622	list_add_tail(&ctx->node, &user->pending);
 623	spin_unlock(&user->lock);
 624
 625	spin_lock_irqsave(&cctx->lock, flags);
 626	ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
 627			       FASTRPC_CTX_MAX, GFP_ATOMIC);
 628	if (ret < 0) {
 629		spin_unlock_irqrestore(&cctx->lock, flags);
 630		goto err_idr;
 631	}
 632	ctx->ctxid = ret << 4;
 633	spin_unlock_irqrestore(&cctx->lock, flags);
 634
 635	kref_init(&ctx->refcount);
 636
 637	return ctx;
 638err_idr:
 639	spin_lock(&user->lock);
 640	list_del(&ctx->node);
 641	spin_unlock(&user->lock);
 642	fastrpc_channel_ctx_put(cctx);
 643	kfree(ctx->maps);
 644	kfree(ctx->olaps);
 645	kfree(ctx);
 646
 647	return ERR_PTR(ret);
 648}
 649
 650static struct sg_table *
 651fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
 652		    enum dma_data_direction dir)
 653{
 654	struct fastrpc_dma_buf_attachment *a = attachment->priv;
 655	struct sg_table *table;
 656	int ret;
 657
 658	table = &a->sgt;
 659
 660	ret = dma_map_sgtable(attachment->dev, table, dir, 0);
 661	if (ret)
 662		table = ERR_PTR(ret);
 663	return table;
 664}
 665
 666static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
 667				  struct sg_table *table,
 668				  enum dma_data_direction dir)
 669{
 670	dma_unmap_sgtable(attach->dev, table, dir, 0);
 671}
 672
 673static void fastrpc_release(struct dma_buf *dmabuf)
 674{
 675	struct fastrpc_buf *buffer = dmabuf->priv;
 676
 677	fastrpc_buf_free(buffer);
 678}
 679
 680static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
 681				  struct dma_buf_attachment *attachment)
 682{
 683	struct fastrpc_dma_buf_attachment *a;
 684	struct fastrpc_buf *buffer = dmabuf->priv;
 685	int ret;
 686
 687	a = kzalloc(sizeof(*a), GFP_KERNEL);
 688	if (!a)
 689		return -ENOMEM;
 690
 691	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
 692			      FASTRPC_PHYS(buffer->phys), buffer->size);
 693	if (ret < 0) {
 694		dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
 695		kfree(a);
 696		return -EINVAL;
 697	}
 698
 699	a->dev = attachment->dev;
 700	INIT_LIST_HEAD(&a->node);
 701	attachment->priv = a;
 702
 703	mutex_lock(&buffer->lock);
 704	list_add(&a->node, &buffer->attachments);
 705	mutex_unlock(&buffer->lock);
 706
 707	return 0;
 708}
 709
 710static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
 711				    struct dma_buf_attachment *attachment)
 712{
 713	struct fastrpc_dma_buf_attachment *a = attachment->priv;
 714	struct fastrpc_buf *buffer = dmabuf->priv;
 715
 716	mutex_lock(&buffer->lock);
 717	list_del(&a->node);
 718	mutex_unlock(&buffer->lock);
 719	sg_free_table(&a->sgt);
 720	kfree(a);
 721}
 722
 723static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
 724{
 725	struct fastrpc_buf *buf = dmabuf->priv;
 726
 727	iosys_map_set_vaddr(map, buf->virt);
 728
 729	return 0;
 730}
 731
 732static int fastrpc_mmap(struct dma_buf *dmabuf,
 733			struct vm_area_struct *vma)
 734{
 735	struct fastrpc_buf *buf = dmabuf->priv;
 736	size_t size = vma->vm_end - vma->vm_start;
 737
 738	dma_resv_assert_held(dmabuf->resv);
 739
 740	return dma_mmap_coherent(buf->dev, vma, buf->virt,
 741				 FASTRPC_PHYS(buf->phys), size);
 742}
 743
 744static const struct dma_buf_ops fastrpc_dma_buf_ops = {
 745	.attach = fastrpc_dma_buf_attach,
 746	.detach = fastrpc_dma_buf_detatch,
 747	.map_dma_buf = fastrpc_map_dma_buf,
 748	.unmap_dma_buf = fastrpc_unmap_dma_buf,
 749	.mmap = fastrpc_mmap,
 750	.vmap = fastrpc_vmap,
 751	.release = fastrpc_release,
 752};
 753
 754static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
 755			      u64 len, u32 attr, struct fastrpc_map **ppmap)
 756{
 757	struct fastrpc_session_ctx *sess = fl->sctx;
 758	struct fastrpc_map *map = NULL;
 759	struct sg_table *table;
 760	int err = 0;
 761
 762	if (!fastrpc_map_lookup(fl, fd, ppmap, true))
 763		return 0;
 764
 765	map = kzalloc(sizeof(*map), GFP_KERNEL);
 766	if (!map)
 767		return -ENOMEM;
 768
 769	INIT_LIST_HEAD(&map->node);
 770	kref_init(&map->refcount);
 771
 772	map->fl = fl;
 773	map->fd = fd;
 774	map->buf = dma_buf_get(fd);
 775	if (IS_ERR(map->buf)) {
 776		err = PTR_ERR(map->buf);
 777		goto get_err;
 778	}
 779
 780	map->attach = dma_buf_attach(map->buf, sess->dev);
 781	if (IS_ERR(map->attach)) {
 782		dev_err(sess->dev, "Failed to attach dmabuf\n");
 783		err = PTR_ERR(map->attach);
 784		goto attach_err;
 785	}
 786
 787	table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL);
 788	if (IS_ERR(table)) {
 789		err = PTR_ERR(table);
 790		goto map_err;
 791	}
 792	map->table = table;
 793
 794	if (attr & FASTRPC_ATTR_SECUREMAP) {
 795		map->phys = sg_phys(map->table->sgl);
 796	} else {
 797		map->phys = sg_dma_address(map->table->sgl);
 798		map->phys += ((u64)fl->sctx->sid << 32);
 799	}
 800	map->size = len;
 801	map->va = sg_virt(map->table->sgl);
 802	map->len = len;
 803
 804	if (attr & FASTRPC_ATTR_SECUREMAP) {
 805		/*
 806		 * If subsystem VMIDs are defined in DTSI, then do
 807		 * hyp_assign from HLOS to those VM(s)
 808		 */
 809		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
 810		struct qcom_scm_vmperm dst_perms[2] = {0};
 811
 812		dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
 813		dst_perms[0].perm = QCOM_SCM_PERM_RW;
 814		dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
 815		dst_perms[1].perm = QCOM_SCM_PERM_RWX;
 816		map->attr = attr;
 817		err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2);
 818		if (err) {
 819			dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
 820					map->phys, map->size, err);
 821			goto map_err;
 822		}
 823	}
 824	spin_lock(&fl->lock);
 825	list_add_tail(&map->node, &fl->maps);
 826	spin_unlock(&fl->lock);
 827	*ppmap = map;
 828
 829	return 0;
 830
 831map_err:
 832	dma_buf_detach(map->buf, map->attach);
 833attach_err:
 834	dma_buf_put(map->buf);
 835get_err:
 836	fastrpc_map_put(map);
 837
 838	return err;
 839}
 840
 841/*
 842 * Fastrpc payload buffer with metadata looks like:
 843 *
 844 * >>>>>>  START of METADATA <<<<<<<<<
 845 * +---------------------------------+
 846 * |           Arguments             |
 847 * | type:(union fastrpc_remote_arg)|
 848 * |             (0 - N)             |
 849 * +---------------------------------+
 850 * |         Invoke Buffer list      |
 851 * | type:(struct fastrpc_invoke_buf)|
 852 * |           (0 - N)               |
 853 * +---------------------------------+
 854 * |         Page info list          |
 855 * | type:(struct fastrpc_phy_page)  |
 856 * |             (0 - N)             |
 857 * +---------------------------------+
 858 * |         Optional info           |
 859 * |(can be specific to SoC/Firmware)|
 860 * +---------------------------------+
 861 * >>>>>>>>  END of METADATA <<<<<<<<<
 862 * +---------------------------------+
 863 * |         Inline ARGS             |
 864 * |            (0-N)                |
 865 * +---------------------------------+
 866 */
 867
 868static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
 869{
 870	int size = 0;
 871
 872	size = (sizeof(struct fastrpc_remote_buf) +
 873		sizeof(struct fastrpc_invoke_buf) +
 874		sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
 875		sizeof(u64) * FASTRPC_MAX_FDLIST +
 876		sizeof(u32) * FASTRPC_MAX_CRCLIST;
 877
 878	return size;
 879}
 880
 881static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
 882{
 883	u64 size = 0;
 884	int oix;
 885
 886	size = ALIGN(metalen, FASTRPC_ALIGN);
 887	for (oix = 0; oix < ctx->nbufs; oix++) {
 888		int i = ctx->olaps[oix].raix;
 889
 890		if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
 891
 892			if (ctx->olaps[oix].offset == 0)
 893				size = ALIGN(size, FASTRPC_ALIGN);
 894
 895			size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
 896		}
 897	}
 898
 899	return size;
 900}
 901
 902static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
 903{
 904	struct device *dev = ctx->fl->sctx->dev;
 905	int i, err;
 906
 907	for (i = 0; i < ctx->nscalars; ++i) {
 908
 909		if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
 910		    ctx->args[i].length == 0)
 911			continue;
 912
 913		err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
 914			 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
 915		if (err) {
 916			dev_err(dev, "Error Creating map %d\n", err);
 917			return -EINVAL;
 918		}
 919
 920	}
 921	return 0;
 922}
 923
 924static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
 925{
 926	return (struct fastrpc_invoke_buf *)(&pra[len]);
 927}
 928
 929static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
 930{
 931	return (struct fastrpc_phy_page *)(&buf[len]);
 932}
 933
 934static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
 935{
 936	struct device *dev = ctx->fl->sctx->dev;
 937	union fastrpc_remote_arg *rpra;
 938	struct fastrpc_invoke_buf *list;
 939	struct fastrpc_phy_page *pages;
 940	int inbufs, i, oix, err = 0;
 941	u64 len, rlen, pkt_size;
 942	u64 pg_start, pg_end;
 943	uintptr_t args;
 944	int metalen;
 945
 946	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
 947	metalen = fastrpc_get_meta_size(ctx);
 948	pkt_size = fastrpc_get_payload_size(ctx, metalen);
 949
 950	err = fastrpc_create_maps(ctx);
 951	if (err)
 952		return err;
 953
 954	ctx->msg_sz = pkt_size;
 955
 956	err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
 
 
 
 957	if (err)
 958		return err;
 959
 960	memset(ctx->buf->virt, 0, pkt_size);
 961	rpra = ctx->buf->virt;
 962	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
 963	pages = fastrpc_phy_page_start(list, ctx->nscalars);
 964	args = (uintptr_t)ctx->buf->virt + metalen;
 965	rlen = pkt_size - metalen;
 966	ctx->rpra = rpra;
 967
 968	for (oix = 0; oix < ctx->nbufs; ++oix) {
 969		int mlen;
 970
 971		i = ctx->olaps[oix].raix;
 972		len = ctx->args[i].length;
 973
 974		rpra[i].buf.pv = 0;
 975		rpra[i].buf.len = len;
 976		list[i].num = len ? 1 : 0;
 977		list[i].pgidx = i;
 978
 979		if (!len)
 980			continue;
 981
 982		if (ctx->maps[i]) {
 983			struct vm_area_struct *vma = NULL;
 984
 985			rpra[i].buf.pv = (u64) ctx->args[i].ptr;
 986			pages[i].addr = ctx->maps[i]->phys;
 987
 988			mmap_read_lock(current->mm);
 989			vma = find_vma(current->mm, ctx->args[i].ptr);
 990			if (vma)
 991				pages[i].addr += ctx->args[i].ptr -
 992						 vma->vm_start;
 993			mmap_read_unlock(current->mm);
 994
 995			pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
 996			pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
 997				  PAGE_SHIFT;
 998			pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
 999
1000		} else {
1001
1002			if (ctx->olaps[oix].offset == 0) {
1003				rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1004				args = ALIGN(args, FASTRPC_ALIGN);
1005			}
1006
1007			mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1008
1009			if (rlen < mlen)
1010				goto bail;
1011
1012			rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1013			pages[i].addr = ctx->buf->phys -
1014					ctx->olaps[oix].offset +
1015					(pkt_size - rlen);
1016			pages[i].addr = pages[i].addr &	PAGE_MASK;
1017
1018			pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
1019			pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1020			pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1021			args = args + mlen;
1022			rlen -= mlen;
1023		}
1024
1025		if (i < inbufs && !ctx->maps[i]) {
1026			void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1027			void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1028
1029			if (!kernel) {
1030				if (copy_from_user(dst, (void __user *)src,
1031						   len)) {
1032					err = -EFAULT;
1033					goto bail;
1034				}
1035			} else {
1036				memcpy(dst, src, len);
1037			}
1038		}
1039	}
1040
1041	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1042		list[i].num = ctx->args[i].length ? 1 : 0;
1043		list[i].pgidx = i;
1044		if (ctx->maps[i]) {
1045			pages[i].addr = ctx->maps[i]->phys;
1046			pages[i].size = ctx->maps[i]->size;
1047		}
1048		rpra[i].dma.fd = ctx->args[i].fd;
1049		rpra[i].dma.len = ctx->args[i].length;
1050		rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1051	}
1052
1053bail:
1054	if (err)
1055		dev_err(dev, "Error: get invoke args failed:%d\n", err);
1056
1057	return err;
1058}
1059
1060static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1061			    u32 kernel)
1062{
1063	union fastrpc_remote_arg *rpra = ctx->rpra;
1064	struct fastrpc_user *fl = ctx->fl;
1065	struct fastrpc_map *mmap = NULL;
1066	struct fastrpc_invoke_buf *list;
1067	struct fastrpc_phy_page *pages;
1068	u64 *fdlist;
1069	int i, inbufs, outbufs, handles;
1070
1071	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1072	outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1073	handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1074	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
1075	pages = fastrpc_phy_page_start(list, ctx->nscalars);
1076	fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1077
1078	for (i = inbufs; i < ctx->nbufs; ++i) {
1079		if (!ctx->maps[i]) {
1080			void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1081			void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1082			u64 len = rpra[i].buf.len;
1083
1084			if (!kernel) {
1085				if (copy_to_user((void __user *)dst, src, len))
1086					return -EFAULT;
1087			} else {
1088				memcpy(dst, src, len);
1089			}
1090		}
1091	}
1092
1093	/* Clean up fdlist which is updated by DSP */
1094	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1095		if (!fdlist[i])
1096			break;
1097		if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false))
1098			fastrpc_map_put(mmap);
1099	}
1100
1101	return 0;
1102}
1103
1104static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1105			       struct fastrpc_invoke_ctx *ctx,
1106			       u32 kernel, uint32_t handle)
1107{
1108	struct fastrpc_channel_ctx *cctx;
1109	struct fastrpc_user *fl = ctx->fl;
1110	struct fastrpc_msg *msg = &ctx->msg;
1111	int ret;
1112
1113	cctx = fl->cctx;
1114	msg->pid = fl->tgid;
1115	msg->tid = current->pid;
1116
1117	if (kernel)
1118		msg->pid = 0;
1119
1120	msg->ctx = ctx->ctxid | fl->pd;
1121	msg->handle = handle;
1122	msg->sc = ctx->sc;
1123	msg->addr = ctx->buf ? ctx->buf->phys : 0;
1124	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1125	fastrpc_context_get(ctx);
1126
1127	ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
1128
1129	if (ret)
1130		fastrpc_context_put(ctx);
1131
1132	return ret;
1133
1134}
1135
1136static int fastrpc_internal_invoke(struct fastrpc_user *fl,  u32 kernel,
1137				   u32 handle, u32 sc,
1138				   struct fastrpc_invoke_args *args)
1139{
1140	struct fastrpc_invoke_ctx *ctx = NULL;
1141	struct fastrpc_buf *buf, *b;
1142
1143	int err = 0;
1144
1145	if (!fl->sctx)
1146		return -EINVAL;
1147
1148	if (!fl->cctx->rpdev)
1149		return -EPIPE;
1150
1151	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1152		dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n",  handle);
1153		return -EPERM;
1154	}
1155
1156	ctx = fastrpc_context_alloc(fl, kernel, sc, args);
1157	if (IS_ERR(ctx))
1158		return PTR_ERR(ctx);
1159
1160	err = fastrpc_get_args(kernel, ctx);
1161	if (err)
1162		goto bail;
1163
1164	/* make sure that all CPU memory writes are seen by DSP */
1165	dma_wmb();
1166	/* Send invoke buffer to remote dsp */
1167	err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
1168	if (err)
1169		goto bail;
1170
1171	if (kernel) {
1172		if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1173			err = -ETIMEDOUT;
1174	} else {
1175		err = wait_for_completion_interruptible(&ctx->work);
1176	}
1177
1178	if (err)
1179		goto bail;
1180
1181	/* make sure that all memory writes by DSP are seen by CPU */
1182	dma_rmb();
1183	/* populate all the output buffers with results */
1184	err = fastrpc_put_args(ctx, kernel);
1185	if (err)
1186		goto bail;
1187
1188	/* Check the response from remote dsp */
1189	err = ctx->retval;
1190	if (err)
1191		goto bail;
1192
1193bail:
1194	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1195		/* We are done with this compute context */
1196		spin_lock(&fl->lock);
1197		list_del(&ctx->node);
1198		spin_unlock(&fl->lock);
1199		fastrpc_context_put(ctx);
1200	}
1201
1202	if (err == -ERESTARTSYS) {
1203		list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1204			list_del(&buf->node);
1205			list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);
1206		}
1207	}
1208
1209	if (err)
1210		dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1211
1212	return err;
1213}
1214
1215static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1216{
1217	/* Check if the device node is non-secure and channel is secure*/
1218	if (!fl->is_secure_dev && fl->cctx->secure) {
1219		/*
1220		 * Allow untrusted applications to offload only to Unsigned PD when
1221		 * channel is configured as secure and block untrusted apps on channel
1222		 * that does not support unsigned PD offload
1223		 */
1224		if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1225			dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD");
1226			return true;
1227		}
1228	}
1229
1230	return false;
1231}
1232
1233static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1234					      char __user *argp)
1235{
1236	struct fastrpc_init_create_static init;
1237	struct fastrpc_invoke_args *args;
1238	struct fastrpc_phy_page pages[1];
1239	char *name;
1240	int err;
 
1241	struct {
1242		int pgid;
1243		u32 namelen;
1244		u32 pageslen;
1245	} inbuf;
1246	u32 sc;
1247
1248	args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1249	if (!args)
1250		return -ENOMEM;
1251
1252	if (copy_from_user(&init, argp, sizeof(init))) {
1253		err = -EFAULT;
1254		goto err;
1255	}
1256
1257	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1258		err = -EINVAL;
1259		goto err;
1260	}
1261
1262	name = kzalloc(init.namelen, GFP_KERNEL);
1263	if (!name) {
1264		err = -ENOMEM;
1265		goto err;
1266	}
1267
1268	if (copy_from_user(name, (void __user *)(uintptr_t)init.name, init.namelen)) {
1269		err = -EFAULT;
1270		goto err_name;
1271	}
1272
1273	if (!fl->cctx->remote_heap) {
1274		err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
1275						&fl->cctx->remote_heap);
1276		if (err)
1277			goto err_name;
1278
1279		/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1280		if (fl->cctx->vmcount) {
1281			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1282
1283			err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1284							(u64)fl->cctx->remote_heap->size,
1285							&src_perms,
1286							fl->cctx->vmperms, fl->cctx->vmcount);
1287			if (err) {
1288				dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
1289					fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1290				goto err_map;
1291			}
 
1292		}
1293	}
1294
1295	inbuf.pgid = fl->tgid;
1296	inbuf.namelen = init.namelen;
1297	inbuf.pageslen = 0;
1298	fl->pd = USER_PD;
1299
1300	args[0].ptr = (u64)(uintptr_t)&inbuf;
1301	args[0].length = sizeof(inbuf);
1302	args[0].fd = -1;
1303
1304	args[1].ptr = (u64)(uintptr_t)name;
1305	args[1].length = inbuf.namelen;
1306	args[1].fd = -1;
1307
1308	pages[0].addr = fl->cctx->remote_heap->phys;
1309	pages[0].size = fl->cctx->remote_heap->size;
1310
1311	args[2].ptr = (u64)(uintptr_t) pages;
1312	args[2].length = sizeof(*pages);
1313	args[2].fd = -1;
1314
1315	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1316
1317	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1318				      sc, args);
1319	if (err)
1320		goto err_invoke;
1321
1322	kfree(args);
 
1323
1324	return 0;
1325err_invoke:
1326	if (fl->cctx->vmcount) {
1327		u64 src_perms = 0;
1328		struct qcom_scm_vmperm dst_perms;
1329		u32 i;
1330
1331		for (i = 0; i < fl->cctx->vmcount; i++)
1332			src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1333
1334		dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1335		dst_perms.perm = QCOM_SCM_PERM_RWX;
1336		err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1337						(u64)fl->cctx->remote_heap->size,
1338						&src_perms, &dst_perms, 1);
1339		if (err)
1340			dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1341				fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1342	}
1343err_map:
1344	fastrpc_buf_free(fl->cctx->remote_heap);
1345err_name:
1346	kfree(name);
1347err:
1348	kfree(args);
1349
1350	return err;
1351}
1352
1353static int fastrpc_init_create_process(struct fastrpc_user *fl,
1354					char __user *argp)
1355{
1356	struct fastrpc_init_create init;
1357	struct fastrpc_invoke_args *args;
1358	struct fastrpc_phy_page pages[1];
1359	struct fastrpc_map *map = NULL;
1360	struct fastrpc_buf *imem = NULL;
1361	int memlen;
1362	int err;
1363	struct {
1364		int pgid;
1365		u32 namelen;
1366		u32 filelen;
1367		u32 pageslen;
1368		u32 attrs;
1369		u32 siglen;
1370	} inbuf;
1371	u32 sc;
1372	bool unsigned_module = false;
1373
1374	args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1375	if (!args)
1376		return -ENOMEM;
1377
1378	if (copy_from_user(&init, argp, sizeof(init))) {
1379		err = -EFAULT;
1380		goto err;
1381	}
1382
1383	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1384		unsigned_module = true;
1385
1386	if (is_session_rejected(fl, unsigned_module)) {
1387		err = -ECONNREFUSED;
1388		goto err;
1389	}
1390
1391	if (init.filelen > INIT_FILELEN_MAX) {
1392		err = -EINVAL;
1393		goto err;
1394	}
1395
1396	inbuf.pgid = fl->tgid;
1397	inbuf.namelen = strlen(current->comm) + 1;
1398	inbuf.filelen = init.filelen;
1399	inbuf.pageslen = 1;
1400	inbuf.attrs = init.attrs;
1401	inbuf.siglen = init.siglen;
1402	fl->pd = USER_PD;
1403
1404	if (init.filelen && init.filefd) {
1405		err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);
1406		if (err)
1407			goto err;
1408	}
1409
1410	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1411		       1024 * 1024);
1412	err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1413				&imem);
1414	if (err)
1415		goto err_alloc;
1416
1417	fl->init_mem = imem;
1418	args[0].ptr = (u64)(uintptr_t)&inbuf;
1419	args[0].length = sizeof(inbuf);
1420	args[0].fd = -1;
1421
1422	args[1].ptr = (u64)(uintptr_t)current->comm;
1423	args[1].length = inbuf.namelen;
1424	args[1].fd = -1;
1425
1426	args[2].ptr = (u64) init.file;
1427	args[2].length = inbuf.filelen;
1428	args[2].fd = init.filefd;
1429
1430	pages[0].addr = imem->phys;
1431	pages[0].size = imem->size;
1432
1433	args[3].ptr = (u64)(uintptr_t) pages;
1434	args[3].length = 1 * sizeof(*pages);
1435	args[3].fd = -1;
1436
1437	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1438	args[4].length = sizeof(inbuf.attrs);
1439	args[4].fd = -1;
1440
1441	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1442	args[5].length = sizeof(inbuf.siglen);
1443	args[5].fd = -1;
1444
1445	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1446	if (init.attrs)
1447		sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1448
1449	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1450				      sc, args);
1451	if (err)
1452		goto err_invoke;
1453
1454	kfree(args);
1455
1456	return 0;
1457
1458err_invoke:
1459	fl->init_mem = NULL;
1460	fastrpc_buf_free(imem);
1461err_alloc:
1462	fastrpc_map_put(map);
1463err:
1464	kfree(args);
1465
1466	return err;
1467}
1468
1469static struct fastrpc_session_ctx *fastrpc_session_alloc(
1470					struct fastrpc_channel_ctx *cctx)
1471{
1472	struct fastrpc_session_ctx *session = NULL;
1473	unsigned long flags;
1474	int i;
1475
1476	spin_lock_irqsave(&cctx->lock, flags);
1477	for (i = 0; i < cctx->sesscount; i++) {
1478		if (!cctx->session[i].used && cctx->session[i].valid) {
1479			cctx->session[i].used = true;
1480			session = &cctx->session[i];
1481			break;
1482		}
1483	}
1484	spin_unlock_irqrestore(&cctx->lock, flags);
1485
1486	return session;
1487}
1488
1489static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1490				 struct fastrpc_session_ctx *session)
1491{
1492	unsigned long flags;
1493
1494	spin_lock_irqsave(&cctx->lock, flags);
1495	session->used = false;
1496	spin_unlock_irqrestore(&cctx->lock, flags);
1497}
1498
1499static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1500{
1501	struct fastrpc_invoke_args args[1];
1502	int tgid = 0;
1503	u32 sc;
1504
1505	tgid = fl->tgid;
1506	args[0].ptr = (u64)(uintptr_t) &tgid;
1507	args[0].length = sizeof(tgid);
1508	args[0].fd = -1;
1509	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1510
1511	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1512				       sc, &args[0]);
1513}
1514
1515static int fastrpc_device_release(struct inode *inode, struct file *file)
1516{
1517	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1518	struct fastrpc_channel_ctx *cctx = fl->cctx;
1519	struct fastrpc_invoke_ctx *ctx, *n;
1520	struct fastrpc_map *map, *m;
1521	struct fastrpc_buf *buf, *b;
1522	unsigned long flags;
1523
1524	fastrpc_release_current_dsp_process(fl);
1525
1526	spin_lock_irqsave(&cctx->lock, flags);
1527	list_del(&fl->user);
1528	spin_unlock_irqrestore(&cctx->lock, flags);
1529
1530	if (fl->init_mem)
1531		fastrpc_buf_free(fl->init_mem);
1532
1533	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1534		list_del(&ctx->node);
1535		fastrpc_context_put(ctx);
1536	}
1537
1538	list_for_each_entry_safe(map, m, &fl->maps, node)
1539		fastrpc_map_put(map);
1540
1541	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1542		list_del(&buf->node);
1543		fastrpc_buf_free(buf);
1544	}
1545
1546	fastrpc_session_free(cctx, fl->sctx);
1547	fastrpc_channel_ctx_put(cctx);
1548
1549	mutex_destroy(&fl->mutex);
1550	kfree(fl);
1551	file->private_data = NULL;
1552
1553	return 0;
1554}
1555
1556static int fastrpc_device_open(struct inode *inode, struct file *filp)
1557{
1558	struct fastrpc_channel_ctx *cctx;
1559	struct fastrpc_device *fdevice;
1560	struct fastrpc_user *fl = NULL;
1561	unsigned long flags;
1562
1563	fdevice = miscdev_to_fdevice(filp->private_data);
1564	cctx = fdevice->cctx;
1565
1566	fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1567	if (!fl)
1568		return -ENOMEM;
1569
1570	/* Released in fastrpc_device_release() */
1571	fastrpc_channel_ctx_get(cctx);
1572
1573	filp->private_data = fl;
1574	spin_lock_init(&fl->lock);
1575	mutex_init(&fl->mutex);
1576	INIT_LIST_HEAD(&fl->pending);
1577	INIT_LIST_HEAD(&fl->maps);
1578	INIT_LIST_HEAD(&fl->mmaps);
1579	INIT_LIST_HEAD(&fl->user);
1580	fl->tgid = current->tgid;
1581	fl->cctx = cctx;
1582	fl->is_secure_dev = fdevice->secure;
1583
1584	fl->sctx = fastrpc_session_alloc(cctx);
1585	if (!fl->sctx) {
1586		dev_err(&cctx->rpdev->dev, "No session available\n");
1587		mutex_destroy(&fl->mutex);
1588		kfree(fl);
1589
1590		return -EBUSY;
1591	}
1592
1593	spin_lock_irqsave(&cctx->lock, flags);
1594	list_add_tail(&fl->user, &cctx->users);
1595	spin_unlock_irqrestore(&cctx->lock, flags);
1596
1597	return 0;
1598}
1599
1600static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1601{
1602	struct fastrpc_alloc_dma_buf bp;
1603	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1604	struct fastrpc_buf *buf = NULL;
1605	int err;
1606
1607	if (copy_from_user(&bp, argp, sizeof(bp)))
1608		return -EFAULT;
1609
1610	err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1611	if (err)
1612		return err;
1613	exp_info.ops = &fastrpc_dma_buf_ops;
1614	exp_info.size = bp.size;
1615	exp_info.flags = O_RDWR;
1616	exp_info.priv = buf;
1617	buf->dmabuf = dma_buf_export(&exp_info);
1618	if (IS_ERR(buf->dmabuf)) {
1619		err = PTR_ERR(buf->dmabuf);
1620		fastrpc_buf_free(buf);
1621		return err;
1622	}
1623
1624	bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1625	if (bp.fd < 0) {
1626		dma_buf_put(buf->dmabuf);
1627		return -EINVAL;
1628	}
1629
1630	if (copy_to_user(argp, &bp, sizeof(bp))) {
1631		/*
1632		 * The usercopy failed, but we can't do much about it, as
1633		 * dma_buf_fd() already called fd_install() and made the
1634		 * file descriptor accessible for the current process. It
1635		 * might already be closed and dmabuf no longer valid when
1636		 * we reach this point. Therefore "leak" the fd and rely on
1637		 * the process exit path to do any required cleanup.
1638		 */
1639		return -EFAULT;
1640	}
1641
1642	return 0;
1643}
1644
1645static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1646{
1647	struct fastrpc_invoke_args args[1];
1648	int tgid = fl->tgid;
1649	u32 sc;
1650
1651	args[0].ptr = (u64)(uintptr_t) &tgid;
1652	args[0].length = sizeof(tgid);
1653	args[0].fd = -1;
1654	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1655	fl->pd = pd;
1656
1657	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1658				       sc, &args[0]);
1659}
1660
1661static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1662{
1663	struct fastrpc_invoke_args *args = NULL;
1664	struct fastrpc_invoke inv;
1665	u32 nscalars;
1666	int err;
1667
1668	if (copy_from_user(&inv, argp, sizeof(inv)))
1669		return -EFAULT;
1670
1671	/* nscalars is truncated here to max supported value */
1672	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1673	if (nscalars) {
1674		args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1675		if (!args)
1676			return -ENOMEM;
1677
1678		if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1679				   nscalars * sizeof(*args))) {
1680			kfree(args);
1681			return -EFAULT;
1682		}
1683	}
1684
1685	err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1686	kfree(args);
1687
1688	return err;
1689}
1690
1691static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1692				     uint32_t dsp_attr_buf_len)
1693{
1694	struct fastrpc_invoke_args args[2] = { 0 };
1695
1696	/* Capability filled in userspace */
 
 
 
 
1697	dsp_attr_buf[0] = 0;
 
1698
1699	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1700	args[0].length = sizeof(dsp_attr_buf_len);
1701	args[0].fd = -1;
1702	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1703	args[1].length = dsp_attr_buf_len;
1704	args[1].fd = -1;
1705	fl->pd = USER_PD;
1706
1707	return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE,
1708				       FASTRPC_SCALARS(0, 1, 1), args);
1709}
1710
1711static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1712					struct fastrpc_user *fl)
1713{
1714	struct fastrpc_channel_ctx *cctx = fl->cctx;
1715	uint32_t attribute_id = cap->attribute_id;
1716	uint32_t *dsp_attributes;
1717	unsigned long flags;
1718	uint32_t domain = cap->domain;
1719	int err;
1720
1721	spin_lock_irqsave(&cctx->lock, flags);
1722	/* check if we already have queried dsp for attributes */
1723	if (cctx->valid_attributes) {
1724		spin_unlock_irqrestore(&cctx->lock, flags);
1725		goto done;
1726	}
1727	spin_unlock_irqrestore(&cctx->lock, flags);
1728
1729	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1730	if (!dsp_attributes)
1731		return -ENOMEM;
1732
1733	err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1734	if (err == DSP_UNSUPPORTED_API) {
1735		dev_info(&cctx->rpdev->dev,
1736			 "Warning: DSP capabilities not supported on domain: %d\n", domain);
1737		kfree(dsp_attributes);
1738		return -EOPNOTSUPP;
1739	} else if (err) {
1740		dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1741		kfree(dsp_attributes);
1742		return err;
1743	}
1744
1745	spin_lock_irqsave(&cctx->lock, flags);
1746	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1747	cctx->valid_attributes = true;
1748	spin_unlock_irqrestore(&cctx->lock, flags);
1749	kfree(dsp_attributes);
1750done:
1751	cap->capability = cctx->dsp_attributes[attribute_id];
1752	return 0;
1753}
1754
1755static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1756{
1757	struct fastrpc_ioctl_capability cap = {0};
1758	int err = 0;
1759
1760	if (copy_from_user(&cap, argp, sizeof(cap)))
1761		return  -EFAULT;
1762
1763	cap.capability = 0;
1764	if (cap.domain >= FASTRPC_DEV_MAX) {
1765		dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",
1766			cap.domain, err);
1767		return -ECHRNG;
1768	}
1769
1770	/* Fastrpc Capablities does not support modem domain */
1771	if (cap.domain == MDSP_DOMAIN_ID) {
1772		dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);
1773		return -ECHRNG;
1774	}
1775
1776	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1777		dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1778			cap.attribute_id, err);
1779		return -EOVERFLOW;
1780	}
1781
1782	err = fastrpc_get_info_from_kernel(&cap, fl);
1783	if (err)
1784		return err;
1785
1786	if (copy_to_user(argp, &cap.capability, sizeof(cap.capability)))
1787		return -EFAULT;
1788
1789	return 0;
1790}
1791
1792static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1793{
1794	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1795	struct fastrpc_munmap_req_msg req_msg;
1796	struct device *dev = fl->sctx->dev;
1797	int err;
1798	u32 sc;
1799
1800	req_msg.pgid = fl->tgid;
1801	req_msg.size = buf->size;
1802	req_msg.vaddr = buf->raddr;
1803
1804	args[0].ptr = (u64) (uintptr_t) &req_msg;
1805	args[0].length = sizeof(req_msg);
1806
1807	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1808	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1809				      &args[0]);
1810	if (!err) {
1811		dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1812		spin_lock(&fl->lock);
1813		list_del(&buf->node);
1814		spin_unlock(&fl->lock);
1815		fastrpc_buf_free(buf);
1816	} else {
1817		dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1818	}
1819
1820	return err;
1821}
1822
1823static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1824{
1825	struct fastrpc_buf *buf = NULL, *iter, *b;
1826	struct fastrpc_req_munmap req;
1827	struct device *dev = fl->sctx->dev;
1828
1829	if (copy_from_user(&req, argp, sizeof(req)))
1830		return -EFAULT;
1831
1832	spin_lock(&fl->lock);
1833	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1834		if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1835			buf = iter;
1836			break;
1837		}
1838	}
1839	spin_unlock(&fl->lock);
1840
1841	if (!buf) {
1842		dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1843			req.vaddrout, req.size);
1844		return -EINVAL;
1845	}
1846
1847	return fastrpc_req_munmap_impl(fl, buf);
1848}
1849
1850static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1851{
1852	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1853	struct fastrpc_buf *buf = NULL;
1854	struct fastrpc_mmap_req_msg req_msg;
1855	struct fastrpc_mmap_rsp_msg rsp_msg;
1856	struct fastrpc_phy_page pages;
1857	struct fastrpc_req_mmap req;
1858	struct device *dev = fl->sctx->dev;
1859	int err;
1860	u32 sc;
1861
1862	if (copy_from_user(&req, argp, sizeof(req)))
1863		return -EFAULT;
1864
1865	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1866		dev_err(dev, "flag not supported 0x%x\n", req.flags);
1867
1868		return -EINVAL;
1869	}
1870
1871	if (req.vaddrin) {
1872		dev_err(dev, "adding user allocated pages is not supported\n");
1873		return -EINVAL;
1874	}
1875
1876	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1877		err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);
1878	else
1879		err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
1880
1881	if (err) {
1882		dev_err(dev, "failed to allocate buffer\n");
1883		return err;
1884	}
1885
1886	req_msg.pgid = fl->tgid;
1887	req_msg.flags = req.flags;
1888	req_msg.vaddr = req.vaddrin;
1889	req_msg.num = sizeof(pages);
1890
1891	args[0].ptr = (u64) (uintptr_t) &req_msg;
1892	args[0].length = sizeof(req_msg);
1893
1894	pages.addr = buf->phys;
1895	pages.size = buf->size;
1896
1897	args[1].ptr = (u64) (uintptr_t) &pages;
1898	args[1].length = sizeof(pages);
1899
1900	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1901	args[2].length = sizeof(rsp_msg);
1902
1903	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1904	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1905				      &args[0]);
1906	if (err) {
1907		dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1908		goto err_invoke;
 
1909	}
1910
1911	/* update the buffer to be able to deallocate the memory on the DSP */
1912	buf->raddr = (uintptr_t) rsp_msg.vaddr;
1913
1914	/* let the client know the address to use */
1915	req.vaddrout = rsp_msg.vaddr;
1916
1917	/* Add memory to static PD pool, protection thru hypervisor */
1918	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1919		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1920
1921		err = qcom_scm_assign_mem(buf->phys, (u64)buf->size,
1922			&src_perms, fl->cctx->vmperms, fl->cctx->vmcount);
1923		if (err) {
1924			dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1925					buf->phys, buf->size, err);
1926			goto err_assign;
1927		}
1928	}
1929
1930	spin_lock(&fl->lock);
1931	list_add_tail(&buf->node, &fl->mmaps);
1932	spin_unlock(&fl->lock);
1933
1934	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1935		err = -EFAULT;
1936		goto err_assign;
1937	}
1938
1939	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1940		buf->raddr, buf->size);
1941
1942	return 0;
1943
1944err_assign:
1945	fastrpc_req_munmap_impl(fl, buf);
1946err_invoke:
1947	fastrpc_buf_free(buf);
1948
1949	return err;
1950}
1951
1952static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1953{
1954	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1955	struct fastrpc_map *map = NULL, *iter, *m;
1956	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1957	int err = 0;
1958	u32 sc;
1959	struct device *dev = fl->sctx->dev;
1960
1961	spin_lock(&fl->lock);
1962	list_for_each_entry_safe(iter, m, &fl->maps, node) {
1963		if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1964			map = iter;
1965			break;
1966		}
1967	}
1968
1969	spin_unlock(&fl->lock);
1970
1971	if (!map) {
1972		dev_err(dev, "map not in list\n");
1973		return -EINVAL;
1974	}
1975
1976	req_msg.pgid = fl->tgid;
1977	req_msg.len = map->len;
1978	req_msg.vaddrin = map->raddr;
1979	req_msg.fd = map->fd;
1980
1981	args[0].ptr = (u64) (uintptr_t) &req_msg;
1982	args[0].length = sizeof(req_msg);
1983
1984	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
1985	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1986				      &args[0]);
1987	if (err) {
1988		dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n",  map->fd, map->raddr);
1989		return err;
1990	}
1991	fastrpc_map_put(map);
1992
1993	return 0;
1994}
1995
1996static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
1997{
1998	struct fastrpc_mem_unmap req;
1999
2000	if (copy_from_user(&req, argp, sizeof(req)))
2001		return -EFAULT;
2002
2003	return fastrpc_req_mem_unmap_impl(fl, &req);
2004}
2005
2006static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2007{
2008	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2009	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2010	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2011	struct fastrpc_mem_unmap req_unmap = { 0 };
2012	struct fastrpc_phy_page pages = { 0 };
2013	struct fastrpc_mem_map req;
2014	struct device *dev = fl->sctx->dev;
2015	struct fastrpc_map *map = NULL;
2016	int err;
2017	u32 sc;
2018
2019	if (copy_from_user(&req, argp, sizeof(req)))
2020		return -EFAULT;
2021
2022	/* create SMMU mapping */
2023	err = fastrpc_map_create(fl, req.fd, req.length, 0, &map);
2024	if (err) {
2025		dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2026		return err;
2027	}
2028
2029	req_msg.pgid = fl->tgid;
2030	req_msg.fd = req.fd;
2031	req_msg.offset = req.offset;
2032	req_msg.vaddrin = req.vaddrin;
2033	map->va = (void *) (uintptr_t) req.vaddrin;
2034	req_msg.flags = req.flags;
2035	req_msg.num = sizeof(pages);
2036	req_msg.data_len = 0;
2037
2038	args[0].ptr = (u64) (uintptr_t) &req_msg;
2039	args[0].length = sizeof(req_msg);
2040
2041	pages.addr = map->phys;
2042	pages.size = map->size;
2043
2044	args[1].ptr = (u64) (uintptr_t) &pages;
2045	args[1].length = sizeof(pages);
2046
2047	args[2].ptr = (u64) (uintptr_t) &pages;
2048	args[2].length = 0;
2049
2050	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2051	args[3].length = sizeof(rsp_msg);
2052
2053	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2054	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);
2055	if (err) {
2056		dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2057			req.fd, req.vaddrin, map->size);
2058		goto err_invoke;
2059	}
2060
2061	/* update the buffer to be able to deallocate the memory on the DSP */
2062	map->raddr = rsp_msg.vaddr;
2063
2064	/* let the client know the address to use */
2065	req.vaddrout = rsp_msg.vaddr;
2066
2067	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
2068		/* unmap the memory and release the buffer */
2069		req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2070		req_unmap.length = map->size;
2071		fastrpc_req_mem_unmap_impl(fl, &req_unmap);
2072		return -EFAULT;
2073	}
2074
2075	return 0;
2076
2077err_invoke:
2078	fastrpc_map_put(map);
2079
2080	return err;
2081}
2082
2083static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2084				 unsigned long arg)
2085{
2086	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2087	char __user *argp = (char __user *)arg;
2088	int err;
2089
2090	switch (cmd) {
2091	case FASTRPC_IOCTL_INVOKE:
2092		err = fastrpc_invoke(fl, argp);
2093		break;
2094	case FASTRPC_IOCTL_INIT_ATTACH:
2095		err = fastrpc_init_attach(fl, ROOT_PD);
2096		break;
2097	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2098		err = fastrpc_init_attach(fl, SENSORS_PD);
2099		break;
2100	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2101		err = fastrpc_init_create_static_process(fl, argp);
2102		break;
2103	case FASTRPC_IOCTL_INIT_CREATE:
2104		err = fastrpc_init_create_process(fl, argp);
2105		break;
2106	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2107		err = fastrpc_dmabuf_alloc(fl, argp);
2108		break;
2109	case FASTRPC_IOCTL_MMAP:
2110		err = fastrpc_req_mmap(fl, argp);
2111		break;
2112	case FASTRPC_IOCTL_MUNMAP:
2113		err = fastrpc_req_munmap(fl, argp);
2114		break;
2115	case FASTRPC_IOCTL_MEM_MAP:
2116		err = fastrpc_req_mem_map(fl, argp);
2117		break;
2118	case FASTRPC_IOCTL_MEM_UNMAP:
2119		err = fastrpc_req_mem_unmap(fl, argp);
2120		break;
2121	case FASTRPC_IOCTL_GET_DSP_INFO:
2122		err = fastrpc_get_dsp_info(fl, argp);
2123		break;
2124	default:
2125		err = -ENOTTY;
2126		break;
2127	}
2128
2129	return err;
2130}
2131
2132static const struct file_operations fastrpc_fops = {
2133	.open = fastrpc_device_open,
2134	.release = fastrpc_device_release,
2135	.unlocked_ioctl = fastrpc_device_ioctl,
2136	.compat_ioctl = fastrpc_device_ioctl,
2137};
2138
2139static int fastrpc_cb_probe(struct platform_device *pdev)
2140{
2141	struct fastrpc_channel_ctx *cctx;
2142	struct fastrpc_session_ctx *sess;
2143	struct device *dev = &pdev->dev;
2144	int i, sessions = 0;
2145	unsigned long flags;
2146	int rc;
2147
2148	cctx = dev_get_drvdata(dev->parent);
2149	if (!cctx)
2150		return -EINVAL;
2151
2152	of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
2153
2154	spin_lock_irqsave(&cctx->lock, flags);
2155	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2156		dev_err(&pdev->dev, "too many sessions\n");
2157		spin_unlock_irqrestore(&cctx->lock, flags);
2158		return -ENOSPC;
2159	}
2160	sess = &cctx->session[cctx->sesscount++];
2161	sess->used = false;
2162	sess->valid = true;
2163	sess->dev = dev;
2164	dev_set_drvdata(dev, sess);
2165
2166	if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
2167		dev_info(dev, "FastRPC Session ID not specified in DT\n");
2168
2169	if (sessions > 0) {
2170		struct fastrpc_session_ctx *dup_sess;
2171
2172		for (i = 1; i < sessions; i++) {
2173			if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2174				break;
2175			dup_sess = &cctx->session[cctx->sesscount++];
2176			memcpy(dup_sess, sess, sizeof(*dup_sess));
2177		}
2178	}
2179	spin_unlock_irqrestore(&cctx->lock, flags);
2180	rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2181	if (rc) {
2182		dev_err(dev, "32-bit DMA enable failed\n");
2183		return rc;
2184	}
2185
2186	return 0;
2187}
2188
2189static int fastrpc_cb_remove(struct platform_device *pdev)
2190{
2191	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
2192	struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
2193	unsigned long flags;
2194	int i;
2195
2196	spin_lock_irqsave(&cctx->lock, flags);
2197	for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2198		if (cctx->session[i].sid == sess->sid) {
2199			cctx->session[i].valid = false;
2200			cctx->sesscount--;
2201		}
2202	}
2203	spin_unlock_irqrestore(&cctx->lock, flags);
2204
2205	return 0;
2206}
2207
2208static const struct of_device_id fastrpc_match_table[] = {
2209	{ .compatible = "qcom,fastrpc-compute-cb", },
2210	{}
2211};
2212
2213static struct platform_driver fastrpc_cb_driver = {
2214	.probe = fastrpc_cb_probe,
2215	.remove = fastrpc_cb_remove,
2216	.driver = {
2217		.name = "qcom,fastrpc-cb",
2218		.of_match_table = fastrpc_match_table,
2219		.suppress_bind_attrs = true,
2220	},
2221};
2222
2223static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2224				   bool is_secured, const char *domain)
2225{
2226	struct fastrpc_device *fdev;
2227	int err;
2228
2229	fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL);
2230	if (!fdev)
2231		return -ENOMEM;
2232
2233	fdev->secure = is_secured;
2234	fdev->cctx = cctx;
2235	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2236	fdev->miscdev.fops = &fastrpc_fops;
2237	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s",
2238					    domain, is_secured ? "-secure" : "");
2239	if (!fdev->miscdev.name)
2240		return -ENOMEM;
2241
2242	err = misc_register(&fdev->miscdev);
2243	if (!err) {
2244		if (is_secured)
2245			cctx->secure_fdevice = fdev;
2246		else
2247			cctx->fdevice = fdev;
2248	}
2249
2250	return err;
2251}
2252
2253static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2254{
2255	struct device *rdev = &rpdev->dev;
2256	struct fastrpc_channel_ctx *data;
2257	int i, err, domain_id = -1, vmcount;
2258	const char *domain;
2259	bool secure_dsp;
 
 
2260	unsigned int vmids[FASTRPC_MAX_VMIDS];
2261
2262	err = of_property_read_string(rdev->of_node, "label", &domain);
2263	if (err) {
2264		dev_info(rdev, "FastRPC Domain not specified in DT\n");
2265		return err;
2266	}
2267
2268	for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
2269		if (!strcmp(domains[i], domain)) {
2270			domain_id = i;
2271			break;
2272		}
2273	}
2274
2275	if (domain_id < 0) {
2276		dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2277		return -EINVAL;
2278	}
2279
2280	if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))
2281		dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2282
2283	vmcount = of_property_read_variable_u32_array(rdev->of_node,
2284				"qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
2285	if (vmcount < 0)
2286		vmcount = 0;
2287	else if (!qcom_scm_is_available())
2288		return -EPROBE_DEFER;
2289
2290	data = kzalloc(sizeof(*data), GFP_KERNEL);
2291	if (!data)
2292		return -ENOMEM;
2293
2294	if (vmcount) {
2295		data->vmcount = vmcount;
2296		for (i = 0; i < data->vmcount; i++) {
2297			data->vmperms[i].vmid = vmids[i];
2298			data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2299		}
2300	}
2301
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2302	secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2303	data->secure = secure_dsp;
2304
2305	switch (domain_id) {
2306	case ADSP_DOMAIN_ID:
2307	case MDSP_DOMAIN_ID:
2308	case SDSP_DOMAIN_ID:
2309		/* Unsigned PD offloading is only supported on CDSP*/
2310		data->unsigned_support = false;
2311		err = fastrpc_device_register(rdev, data, secure_dsp, domains[domain_id]);
2312		if (err)
2313			goto fdev_error;
2314		break;
2315	case CDSP_DOMAIN_ID:
 
2316		data->unsigned_support = true;
2317		/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2318		err = fastrpc_device_register(rdev, data, true, domains[domain_id]);
2319		if (err)
2320			goto fdev_error;
2321
2322		err = fastrpc_device_register(rdev, data, false, domains[domain_id]);
2323		if (err)
2324			goto fdev_error;
2325		break;
2326	default:
2327		err = -EINVAL;
2328		goto fdev_error;
2329	}
2330
2331	kref_init(&data->refcount);
2332
2333	dev_set_drvdata(&rpdev->dev, data);
2334	rdev->dma_mask = &data->dma_mask;
2335	dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
2336	INIT_LIST_HEAD(&data->users);
2337	INIT_LIST_HEAD(&data->invoke_interrupted_mmaps);
2338	spin_lock_init(&data->lock);
2339	idr_init(&data->ctx_idr);
2340	data->domain_id = domain_id;
2341	data->rpdev = rpdev;
2342
2343	err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);
2344	if (err)
2345		goto populate_error;
2346
2347	return 0;
2348
2349populate_error:
2350	if (data->fdevice)
2351		misc_deregister(&data->fdevice->miscdev);
2352	if (data->secure_fdevice)
2353		misc_deregister(&data->secure_fdevice->miscdev);
2354
2355fdev_error:
2356	kfree(data);
2357	return err;
2358}
2359
2360static void fastrpc_notify_users(struct fastrpc_user *user)
2361{
2362	struct fastrpc_invoke_ctx *ctx;
2363
2364	spin_lock(&user->lock);
2365	list_for_each_entry(ctx, &user->pending, node) {
2366		ctx->retval = -EPIPE;
2367		complete(&ctx->work);
2368	}
2369	spin_unlock(&user->lock);
2370}
2371
2372static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2373{
2374	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2375	struct fastrpc_buf *buf, *b;
2376	struct fastrpc_user *user;
2377	unsigned long flags;
2378
2379	/* No invocations past this point */
2380	spin_lock_irqsave(&cctx->lock, flags);
2381	cctx->rpdev = NULL;
2382	list_for_each_entry(user, &cctx->users, user)
2383		fastrpc_notify_users(user);
2384	spin_unlock_irqrestore(&cctx->lock, flags);
2385
2386	if (cctx->fdevice)
2387		misc_deregister(&cctx->fdevice->miscdev);
2388
2389	if (cctx->secure_fdevice)
2390		misc_deregister(&cctx->secure_fdevice->miscdev);
2391
2392	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2393		list_del(&buf->node);
2394
2395	if (cctx->remote_heap)
2396		fastrpc_buf_free(cctx->remote_heap);
2397
2398	of_platform_depopulate(&rpdev->dev);
2399
2400	fastrpc_channel_ctx_put(cctx);
2401}
2402
2403static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2404				  int len, void *priv, u32 addr)
2405{
2406	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2407	struct fastrpc_invoke_rsp *rsp = data;
2408	struct fastrpc_invoke_ctx *ctx;
2409	unsigned long flags;
2410	unsigned long ctxid;
2411
2412	if (len < sizeof(*rsp))
2413		return -EINVAL;
2414
2415	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2416
2417	spin_lock_irqsave(&cctx->lock, flags);
2418	ctx = idr_find(&cctx->ctx_idr, ctxid);
2419	spin_unlock_irqrestore(&cctx->lock, flags);
2420
2421	if (!ctx) {
2422		dev_err(&rpdev->dev, "No context ID matches response\n");
2423		return -ENOENT;
2424	}
2425
2426	ctx->retval = rsp->retval;
2427	complete(&ctx->work);
2428
2429	/*
2430	 * The DMA buffer associated with the context cannot be freed in
2431	 * interrupt context so schedule it through a worker thread to
2432	 * avoid a kernel BUG.
2433	 */
2434	schedule_work(&ctx->put_work);
2435
2436	return 0;
2437}
2438
2439static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2440	{ .compatible = "qcom,fastrpc" },
2441	{ },
2442};
2443MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2444
2445static struct rpmsg_driver fastrpc_driver = {
2446	.probe = fastrpc_rpmsg_probe,
2447	.remove = fastrpc_rpmsg_remove,
2448	.callback = fastrpc_rpmsg_callback,
2449	.drv = {
2450		.name = "qcom,fastrpc",
2451		.of_match_table = fastrpc_rpmsg_of_match,
2452	},
2453};
2454
2455static int fastrpc_init(void)
2456{
2457	int ret;
2458
2459	ret = platform_driver_register(&fastrpc_cb_driver);
2460	if (ret < 0) {
2461		pr_err("fastrpc: failed to register cb driver\n");
2462		return ret;
2463	}
2464
2465	ret = register_rpmsg_driver(&fastrpc_driver);
2466	if (ret < 0) {
2467		pr_err("fastrpc: failed to register rpmsg driver\n");
2468		platform_driver_unregister(&fastrpc_cb_driver);
2469		return ret;
2470	}
2471
2472	return 0;
2473}
2474module_init(fastrpc_init);
2475
2476static void fastrpc_exit(void)
2477{
2478	platform_driver_unregister(&fastrpc_cb_driver);
2479	unregister_rpmsg_driver(&fastrpc_driver);
2480}
2481module_exit(fastrpc_exit);
2482
 
2483MODULE_LICENSE("GPL v2");
2484MODULE_IMPORT_NS(DMA_BUF);