1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
   3// Copyright (c) 2018, Linaro Limited
   4
   5#include <linux/completion.h>
   6#include <linux/device.h>
   7#include <linux/dma-buf.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/dma-resv.h>
  10#include <linux/idr.h>
  11#include <linux/list.h>
  12#include <linux/miscdevice.h>
  13#include <linux/module.h>
  14#include <linux/of_address.h>
  15#include <linux/of.h>
  16#include <linux/platform_device.h>
  17#include <linux/sort.h>
  18#include <linux/of_platform.h>
  19#include <linux/rpmsg.h>
  20#include <linux/scatterlist.h>
  21#include <linux/slab.h>
  22#include <linux/firmware/qcom/qcom_scm.h>
  23#include <uapi/misc/fastrpc.h>
  24#include <linux/of_reserved_mem.h>
  25
  26#define ADSP_DOMAIN_ID (0)
  27#define MDSP_DOMAIN_ID (1)
  28#define SDSP_DOMAIN_ID (2)
  29#define CDSP_DOMAIN_ID (3)
  30#define CDSP1_DOMAIN_ID (4)
  31#define FASTRPC_DEV_MAX		5 /* adsp, mdsp, slpi, cdsp, cdsp1 */
  32#define FASTRPC_MAX_SESSIONS	14
  33#define FASTRPC_MAX_VMIDS	16
  34#define FASTRPC_ALIGN		128
  35#define FASTRPC_MAX_FDLIST	16
  36#define FASTRPC_MAX_CRCLIST	64
  37#define FASTRPC_PHYS(p)	((p) & 0xffffffff)
  38#define FASTRPC_CTX_MAX (256)
  39#define FASTRPC_INIT_HANDLE	1
  40#define FASTRPC_DSP_UTILITIES_HANDLE	2
  41#define FASTRPC_CTXID_MASK (0xFF0)
  42#define INIT_FILELEN_MAX (2 * 1024 * 1024)
  43#define INIT_FILE_NAMELEN_MAX (128)
  44#define FASTRPC_DEVICE_NAME	"fastrpc"
  45
  46/* Add memory to static PD pool, protection thru XPU */
  47#define ADSP_MMAP_HEAP_ADDR  4
  48/* MAP static DMA buffer on DSP User PD */
  49#define ADSP_MMAP_DMA_BUFFER  6
  50/* Add memory to static PD pool protection thru hypervisor */
  51#define ADSP_MMAP_REMOTE_HEAP_ADDR  8
  52/* Add memory to userPD pool, for user heap */
  53#define ADSP_MMAP_ADD_PAGES 0x1000
  54/* Add memory to userPD pool, for LLC heap */
  55#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
  56
  57#define DSP_UNSUPPORTED_API (0x80000414)
  58/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
  59#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
  60#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
  61
  62/* Retrives number of input buffers from the scalars parameter */
  63#define REMOTE_SCALARS_INBUFS(sc)	(((sc) >> 16) & 0x0ff)
  64
  65/* Retrives number of output buffers from the scalars parameter */
  66#define REMOTE_SCALARS_OUTBUFS(sc)	(((sc) >> 8) & 0x0ff)
  67
  68/* Retrives number of input handles from the scalars parameter */
  69#define REMOTE_SCALARS_INHANDLES(sc)	(((sc) >> 4) & 0x0f)
  70
  71/* Retrives number of output handles from the scalars parameter */
  72#define REMOTE_SCALARS_OUTHANDLES(sc)	((sc) & 0x0f)
  73
  74#define REMOTE_SCALARS_LENGTH(sc)	(REMOTE_SCALARS_INBUFS(sc) +   \
  75					 REMOTE_SCALARS_OUTBUFS(sc) +  \
  76					 REMOTE_SCALARS_INHANDLES(sc)+ \
  77					 REMOTE_SCALARS_OUTHANDLES(sc))
  78#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout)  \
  79				(((attr & 0x07) << 29) |		\
  80				((method & 0x1f) << 24) |	\
  81				((in & 0xff) << 16) |		\
  82				((out & 0xff) <<  8) |		\
  83				((oin & 0x0f) <<  4) |		\
  84				(oout & 0x0f))
  85
  86#define FASTRPC_SCALARS(method, in, out) \
  87		FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
  88
  89#define FASTRPC_CREATE_PROCESS_NARGS	6
  90#define FASTRPC_CREATE_STATIC_PROCESS_NARGS	3
  91/* Remote Method id table */
  92#define FASTRPC_RMID_INIT_ATTACH	0
  93#define FASTRPC_RMID_INIT_RELEASE	1
  94#define FASTRPC_RMID_INIT_MMAP		4
  95#define FASTRPC_RMID_INIT_MUNMAP	5
  96#define FASTRPC_RMID_INIT_CREATE	6
  97#define FASTRPC_RMID_INIT_CREATE_ATTR	7
  98#define FASTRPC_RMID_INIT_CREATE_STATIC	8
  99#define FASTRPC_RMID_INIT_MEM_MAP      10
 100#define FASTRPC_RMID_INIT_MEM_UNMAP    11
 101
 102/* Protection Domain(PD) ids */
 103#define ROOT_PD		(0)
 104#define USER_PD		(1)
 105#define SENSORS_PD	(2)
 106
 107#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
 108
 109static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
 110						"sdsp", "cdsp", "cdsp1" };
 111struct fastrpc_phy_page {
 112	u64 addr;		/* physical address */
 113	u64 size;		/* size of contiguous region */
 114};
 115
 116struct fastrpc_invoke_buf {
 117	u32 num;		/* number of contiguous regions */
 118	u32 pgidx;		/* index to start of contiguous region */
 119};
 120
 121struct fastrpc_remote_dmahandle {
 122	s32 fd;		/* dma handle fd */
 123	u32 offset;	/* dma handle offset */
 124	u32 len;	/* dma handle length */
 125};
 126
 127struct fastrpc_remote_buf {
 128	u64 pv;		/* buffer pointer */
 129	u64 len;	/* length of buffer */
 130};
 131
 132union fastrpc_remote_arg {
 133	struct fastrpc_remote_buf buf;
 134	struct fastrpc_remote_dmahandle dma;
 135};
 136
 137struct fastrpc_mmap_rsp_msg {
 138	u64 vaddr;
 139};
 140
 141struct fastrpc_mmap_req_msg {
 142	s32 pgid;
 143	u32 flags;
 144	u64 vaddr;
 145	s32 num;
 146};
 147
 148struct fastrpc_mem_map_req_msg {
 149	s32 pgid;
 150	s32 fd;
 151	s32 offset;
 152	u32 flags;
 153	u64 vaddrin;
 154	s32 num;
 155	s32 data_len;
 156};
 157
 158struct fastrpc_munmap_req_msg {
 159	s32 pgid;
 160	u64 vaddr;
 161	u64 size;
 162};
 163
 164struct fastrpc_mem_unmap_req_msg {
 165	s32 pgid;
 166	s32 fd;
 167	u64 vaddrin;
 168	u64 len;
 169};
 170
 171struct fastrpc_msg {
 172	int pid;		/* process group id */
 173	int tid;		/* thread id */
 174	u64 ctx;		/* invoke caller context */
 175	u32 handle;	/* handle to invoke */
 176	u32 sc;		/* scalars structure describing the data */
 177	u64 addr;		/* physical address */
 178	u64 size;		/* size of contiguous region */
 179};
 180
 181struct fastrpc_invoke_rsp {
 182	u64 ctx;		/* invoke caller context */
 183	int retval;		/* invoke return value */
 184};
 185
 186struct fastrpc_buf_overlap {
 187	u64 start;
 188	u64 end;
 189	int raix;
 190	u64 mstart;
 191	u64 mend;
 192	u64 offset;
 193};
 194
 195struct fastrpc_buf {
 196	struct fastrpc_user *fl;
 197	struct dma_buf *dmabuf;
 198	struct device *dev;
 199	void *virt;
 200	u64 phys;
 201	u64 size;
 202	/* Lock for dma buf attachments */
 203	struct mutex lock;
 204	struct list_head attachments;
 205	/* mmap support */
 206	struct list_head node; /* list of user requested mmaps */
 207	uintptr_t raddr;
 208};
 209
 210struct fastrpc_dma_buf_attachment {
 211	struct device *dev;
 212	struct sg_table sgt;
 213	struct list_head node;
 214};
 215
 216struct fastrpc_map {
 217	struct list_head node;
 218	struct fastrpc_user *fl;
 219	int fd;
 220	struct dma_buf *buf;
 221	struct sg_table *table;
 222	struct dma_buf_attachment *attach;
 223	u64 phys;
 224	u64 size;
 225	void *va;
 226	u64 len;
 227	u64 raddr;
 228	u32 attr;
 229	struct kref refcount;
 230};
 231
 232struct fastrpc_invoke_ctx {
 233	int nscalars;
 234	int nbufs;
 235	int retval;
 236	int pid;
 237	int tgid;
 238	u32 sc;
 239	u32 *crc;
 240	u64 ctxid;
 241	u64 msg_sz;
 242	struct kref refcount;
 243	struct list_head node; /* list of ctxs */
 244	struct completion work;
 245	struct work_struct put_work;
 246	struct fastrpc_msg msg;
 247	struct fastrpc_user *fl;
 248	union fastrpc_remote_arg *rpra;
 249	struct fastrpc_map **maps;
 250	struct fastrpc_buf *buf;
 251	struct fastrpc_invoke_args *args;
 252	struct fastrpc_buf_overlap *olaps;
 253	struct fastrpc_channel_ctx *cctx;
 254};
 255
 256struct fastrpc_session_ctx {
 257	struct device *dev;
 258	int sid;
 259	bool used;
 260	bool valid;
 261};
 262
 263struct fastrpc_channel_ctx {
 264	int domain_id;
 265	int sesscount;
 266	int vmcount;
 267	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
 268	struct rpmsg_device *rpdev;
 269	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
 270	spinlock_t lock;
 271	struct idr ctx_idr;
 272	struct list_head users;
 273	struct kref refcount;
 274	/* Flag if dsp attributes are cached */
 275	bool valid_attributes;
 276	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
 277	struct fastrpc_device *secure_fdevice;
 278	struct fastrpc_device *fdevice;
 279	struct fastrpc_buf *remote_heap;
 280	struct list_head invoke_interrupted_mmaps;
 281	bool secure;
 282	bool unsigned_support;
 283	u64 dma_mask;
 284};
 285
 286struct fastrpc_device {
 287	struct fastrpc_channel_ctx *cctx;
 288	struct miscdevice miscdev;
 289	bool secure;
 290};
 291
 292struct fastrpc_user {
 293	struct list_head user;
 294	struct list_head maps;
 295	struct list_head pending;
 296	struct list_head mmaps;
 297
 298	struct fastrpc_channel_ctx *cctx;
 299	struct fastrpc_session_ctx *sctx;
 300	struct fastrpc_buf *init_mem;
 301
 302	int tgid;
 303	int pd;
 304	bool is_secure_dev;
 305	/* Lock for lists */
 306	spinlock_t lock;
 307	/* lock for allocations */
 308	struct mutex mutex;
 309};
 310
 311static void fastrpc_free_map(struct kref *ref)
 312{
 313	struct fastrpc_map *map;
 314
 315	map = container_of(ref, struct fastrpc_map, refcount);
 316
 317	if (map->table) {
 318		if (map->attr & FASTRPC_ATTR_SECUREMAP) {
 319			struct qcom_scm_vmperm perm;
 320			int vmid = map->fl->cctx->vmperms[0].vmid;
 321			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
 322			int err = 0;
 323
 324			perm.vmid = QCOM_SCM_VMID_HLOS;
 325			perm.perm = QCOM_SCM_PERM_RWX;
 326			err = qcom_scm_assign_mem(map->phys, map->size,
 327				&src_perms, &perm, 1);
 328			if (err) {
 329				dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
 330						map->phys, map->size, err);
 331				return;
 332			}
 333		}
 334		dma_buf_unmap_attachment_unlocked(map->attach, map->table,
 335						  DMA_BIDIRECTIONAL);
 336		dma_buf_detach(map->buf, map->attach);
 337		dma_buf_put(map->buf);
 338	}
 339
 340	if (map->fl) {
 341		spin_lock(&map->fl->lock);
 342		list_del(&map->node);
 343		spin_unlock(&map->fl->lock);
 344		map->fl = NULL;
 345	}
 346
 347	kfree(map);
 348}
 349
 350static void fastrpc_map_put(struct fastrpc_map *map)
 351{
 352	if (map)
 353		kref_put(&map->refcount, fastrpc_free_map);
 354}
 355
 356static int fastrpc_map_get(struct fastrpc_map *map)
 357{
 358	if (!map)
 359		return -ENOENT;
 360
 361	return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;
 362}
 363
 364
 365static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
 366			    struct fastrpc_map **ppmap, bool take_ref)
 367{
 368	struct fastrpc_session_ctx *sess = fl->sctx;
 369	struct fastrpc_map *map = NULL;
 370	int ret = -ENOENT;
 371
 372	spin_lock(&fl->lock);
 373	list_for_each_entry(map, &fl->maps, node) {
 374		if (map->fd != fd)
 375			continue;
 376
 377		if (take_ref) {
 378			ret = fastrpc_map_get(map);
 379			if (ret) {
 380				dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",
 381					__func__, fd, ret);
 382				break;
 383			}
 384		}
 385
 386		*ppmap = map;
 387		ret = 0;
 388		break;
 389	}
 390	spin_unlock(&fl->lock);
 391
 392	return ret;
 393}
 394
 395static void fastrpc_buf_free(struct fastrpc_buf *buf)
 396{
 397	dma_free_coherent(buf->dev, buf->size, buf->virt,
 398			  FASTRPC_PHYS(buf->phys));
 399	kfree(buf);
 400}
 401
 402static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
 403			     u64 size, struct fastrpc_buf **obuf)
 404{
 405	struct fastrpc_buf *buf;
 406
 407	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
 408	if (!buf)
 409		return -ENOMEM;
 410
 411	INIT_LIST_HEAD(&buf->attachments);
 412	INIT_LIST_HEAD(&buf->node);
 413	mutex_init(&buf->lock);
 414
 415	buf->fl = fl;
 416	buf->virt = NULL;
 417	buf->phys = 0;
 418	buf->size = size;
 419	buf->dev = dev;
 420	buf->raddr = 0;
 421
 422	buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
 423				       GFP_KERNEL);
 424	if (!buf->virt) {
 425		mutex_destroy(&buf->lock);
 426		kfree(buf);
 427		return -ENOMEM;
 428	}
 429
 430	*obuf = buf;
 431
 432	return 0;
 433}
 434
 435static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
 436			     u64 size, struct fastrpc_buf **obuf)
 437{
 438	int ret;
 439	struct fastrpc_buf *buf;
 440
 441	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
 442	if (ret)
 443		return ret;
 444
 445	buf = *obuf;
 446
 447	if (fl->sctx && fl->sctx->sid)
 448		buf->phys += ((u64)fl->sctx->sid << 32);
 449
 450	return 0;
 451}
 452
 453static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
 454				     u64 size, struct fastrpc_buf **obuf)
 455{
 456	struct device *rdev = &fl->cctx->rpdev->dev;
 457
 458	return  __fastrpc_buf_alloc(fl, rdev, size, obuf);
 459}
 460
 461static void fastrpc_channel_ctx_free(struct kref *ref)
 462{
 463	struct fastrpc_channel_ctx *cctx;
 464
 465	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
 466
 467	kfree(cctx);
 468}
 469
 470static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
 471{
 472	kref_get(&cctx->refcount);
 473}
 474
 475static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
 476{
 477	kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
 478}
 479
 480static void fastrpc_context_free(struct kref *ref)
 481{
 482	struct fastrpc_invoke_ctx *ctx;
 483	struct fastrpc_channel_ctx *cctx;
 484	unsigned long flags;
 485	int i;
 486
 487	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
 488	cctx = ctx->cctx;
 489
 490	for (i = 0; i < ctx->nbufs; i++)
 491		fastrpc_map_put(ctx->maps[i]);
 492
 493	if (ctx->buf)
 494		fastrpc_buf_free(ctx->buf);
 495
 496	spin_lock_irqsave(&cctx->lock, flags);
 497	idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
 498	spin_unlock_irqrestore(&cctx->lock, flags);
 499
 500	kfree(ctx->maps);
 501	kfree(ctx->olaps);
 502	kfree(ctx);
 503
 504	fastrpc_channel_ctx_put(cctx);
 505}
 506
 507static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
 508{
 509	kref_get(&ctx->refcount);
 510}
 511
 512static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
 513{
 514	kref_put(&ctx->refcount, fastrpc_context_free);
 515}
 516
 517static void fastrpc_context_put_wq(struct work_struct *work)
 518{
 519	struct fastrpc_invoke_ctx *ctx =
 520			container_of(work, struct fastrpc_invoke_ctx, put_work);
 521
 522	fastrpc_context_put(ctx);
 523}
 524
 525#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
 526static int olaps_cmp(const void *a, const void *b)
 527{
 528	struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
 529	struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
 530	/* sort with lowest starting buffer first */
 531	int st = CMP(pa->start, pb->start);
 532	/* sort with highest ending buffer first */
 533	int ed = CMP(pb->end, pa->end);
 534
 535	return st == 0 ? ed : st;
 536}
 537
 538static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
 539{
 540	u64 max_end = 0;
 541	int i;
 542
 543	for (i = 0; i < ctx->nbufs; ++i) {
 544		ctx->olaps[i].start = ctx->args[i].ptr;
 545		ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
 546		ctx->olaps[i].raix = i;
 547	}
 548
 549	sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
 550
 551	for (i = 0; i < ctx->nbufs; ++i) {
 552		/* Falling inside previous range */
 553		if (ctx->olaps[i].start < max_end) {
 554			ctx->olaps[i].mstart = max_end;
 555			ctx->olaps[i].mend = ctx->olaps[i].end;
 556			ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
 557
 558			if (ctx->olaps[i].end > max_end) {
 559				max_end = ctx->olaps[i].end;
 560			} else {
 561				ctx->olaps[i].mend = 0;
 562				ctx->olaps[i].mstart = 0;
 563			}
 564
 565		} else  {
 566			ctx->olaps[i].mend = ctx->olaps[i].end;
 567			ctx->olaps[i].mstart = ctx->olaps[i].start;
 568			ctx->olaps[i].offset = 0;
 569			max_end = ctx->olaps[i].end;
 570		}
 571	}
 572}
 573
 574static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
 575			struct fastrpc_user *user, u32 kernel, u32 sc,
 576			struct fastrpc_invoke_args *args)
 577{
 578	struct fastrpc_channel_ctx *cctx = user->cctx;
 579	struct fastrpc_invoke_ctx *ctx = NULL;
 580	unsigned long flags;
 581	int ret;
 582
 583	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 584	if (!ctx)
 585		return ERR_PTR(-ENOMEM);
 586
 587	INIT_LIST_HEAD(&ctx->node);
 588	ctx->fl = user;
 589	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
 590	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
 591		     REMOTE_SCALARS_OUTBUFS(sc);
 592
 593	if (ctx->nscalars) {
 594		ctx->maps = kcalloc(ctx->nscalars,
 595				    sizeof(*ctx->maps), GFP_KERNEL);
 596		if (!ctx->maps) {
 597			kfree(ctx);
 598			return ERR_PTR(-ENOMEM);
 599		}
 600		ctx->olaps = kcalloc(ctx->nscalars,
 601				    sizeof(*ctx->olaps), GFP_KERNEL);
 602		if (!ctx->olaps) {
 603			kfree(ctx->maps);
 604			kfree(ctx);
 605			return ERR_PTR(-ENOMEM);
 606		}
 607		ctx->args = args;
 608		fastrpc_get_buff_overlaps(ctx);
 609	}
 610
 611	/* Released in fastrpc_context_put() */
 612	fastrpc_channel_ctx_get(cctx);
 613
 614	ctx->sc = sc;
 615	ctx->retval = -1;
 616	ctx->pid = current->pid;
 617	ctx->tgid = user->tgid;
 618	ctx->cctx = cctx;
 619	init_completion(&ctx->work);
 620	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
 621
 622	spin_lock(&user->lock);
 623	list_add_tail(&ctx->node, &user->pending);
 624	spin_unlock(&user->lock);
 625
 626	spin_lock_irqsave(&cctx->lock, flags);
 627	ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
 628			       FASTRPC_CTX_MAX, GFP_ATOMIC);
 629	if (ret < 0) {
 630		spin_unlock_irqrestore(&cctx->lock, flags);
 631		goto err_idr;
 632	}
 633	ctx->ctxid = ret << 4;
 634	spin_unlock_irqrestore(&cctx->lock, flags);
 635
 636	kref_init(&ctx->refcount);
 637
 638	return ctx;
 639err_idr:
 640	spin_lock(&user->lock);
 641	list_del(&ctx->node);
 642	spin_unlock(&user->lock);
 643	fastrpc_channel_ctx_put(cctx);
 644	kfree(ctx->maps);
 645	kfree(ctx->olaps);
 646	kfree(ctx);
 647
 648	return ERR_PTR(ret);
 649}
 650
 651static struct sg_table *
 652fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
 653		    enum dma_data_direction dir)
 654{
 655	struct fastrpc_dma_buf_attachment *a = attachment->priv;
 656	struct sg_table *table;
 657	int ret;
 658
 659	table = &a->sgt;
 660
 661	ret = dma_map_sgtable(attachment->dev, table, dir, 0);
 662	if (ret)
 663		table = ERR_PTR(ret);
 664	return table;
 665}
 666
 667static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
 668				  struct sg_table *table,
 669				  enum dma_data_direction dir)
 670{
 671	dma_unmap_sgtable(attach->dev, table, dir, 0);
 672}
 673
 674static void fastrpc_release(struct dma_buf *dmabuf)
 675{
 676	struct fastrpc_buf *buffer = dmabuf->priv;
 677
 678	fastrpc_buf_free(buffer);
 679}
 680
 681static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
 682				  struct dma_buf_attachment *attachment)
 683{
 684	struct fastrpc_dma_buf_attachment *a;
 685	struct fastrpc_buf *buffer = dmabuf->priv;
 686	int ret;
 687
 688	a = kzalloc(sizeof(*a), GFP_KERNEL);
 689	if (!a)
 690		return -ENOMEM;
 691
 692	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
 693			      FASTRPC_PHYS(buffer->phys), buffer->size);
 694	if (ret < 0) {
 695		dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
 696		kfree(a);
 697		return -EINVAL;
 698	}
 699
 700	a->dev = attachment->dev;
 701	INIT_LIST_HEAD(&a->node);
 702	attachment->priv = a;
 703
 704	mutex_lock(&buffer->lock);
 705	list_add(&a->node, &buffer->attachments);
 706	mutex_unlock(&buffer->lock);
 707
 708	return 0;
 709}
 710
 711static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
 712				    struct dma_buf_attachment *attachment)
 713{
 714	struct fastrpc_dma_buf_attachment *a = attachment->priv;
 715	struct fastrpc_buf *buffer = dmabuf->priv;
 716
 717	mutex_lock(&buffer->lock);
 718	list_del(&a->node);
 719	mutex_unlock(&buffer->lock);
 720	sg_free_table(&a->sgt);
 721	kfree(a);
 722}
 723
 724static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
 725{
 726	struct fastrpc_buf *buf = dmabuf->priv;
 727
 728	iosys_map_set_vaddr(map, buf->virt);
 
 729
 730	return 0;
 
 
 
 
 731}
 732
 733static int fastrpc_mmap(struct dma_buf *dmabuf,
 734			struct vm_area_struct *vma)
 735{
 736	struct fastrpc_buf *buf = dmabuf->priv;
 737	size_t size = vma->vm_end - vma->vm_start;
 738
 739	dma_resv_assert_held(dmabuf->resv);
 740
 741	return dma_mmap_coherent(buf->dev, vma, buf->virt,
 742				 FASTRPC_PHYS(buf->phys), size);
 743}
 744
 745static const struct dma_buf_ops fastrpc_dma_buf_ops = {
 746	.attach = fastrpc_dma_buf_attach,
 747	.detach = fastrpc_dma_buf_detatch,
 748	.map_dma_buf = fastrpc_map_dma_buf,
 749	.unmap_dma_buf = fastrpc_unmap_dma_buf,
 750	.mmap = fastrpc_mmap,
 
 751	.vmap = fastrpc_vmap,
 752	.release = fastrpc_release,
 753};
 754
 755static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
 756			      u64 len, u32 attr, struct fastrpc_map **ppmap)
 757{
 758	struct fastrpc_session_ctx *sess = fl->sctx;
 759	struct fastrpc_map *map = NULL;
 760	struct sg_table *table;
 761	int err = 0;
 762
 763	if (!fastrpc_map_lookup(fl, fd, ppmap, true))
 764		return 0;
 765
 766	map = kzalloc(sizeof(*map), GFP_KERNEL);
 767	if (!map)
 768		return -ENOMEM;
 769
 770	INIT_LIST_HEAD(&map->node);
 771	kref_init(&map->refcount);
 772
 773	map->fl = fl;
 774	map->fd = fd;
 775	map->buf = dma_buf_get(fd);
 776	if (IS_ERR(map->buf)) {
 777		err = PTR_ERR(map->buf);
 778		goto get_err;
 779	}
 780
 781	map->attach = dma_buf_attach(map->buf, sess->dev);
 782	if (IS_ERR(map->attach)) {
 783		dev_err(sess->dev, "Failed to attach dmabuf\n");
 784		err = PTR_ERR(map->attach);
 785		goto attach_err;
 786	}
 787
 788	table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL);
 789	if (IS_ERR(table)) {
 790		err = PTR_ERR(table);
 791		goto map_err;
 792	}
 793	map->table = table;
 794
 795	if (attr & FASTRPC_ATTR_SECUREMAP) {
 796		map->phys = sg_phys(map->table->sgl);
 797	} else {
 798		map->phys = sg_dma_address(map->table->sgl);
 799		map->phys += ((u64)fl->sctx->sid << 32);
 800	}
 801	map->size = len;
 802	map->va = sg_virt(map->table->sgl);
 803	map->len = len;
 
 804
 805	if (attr & FASTRPC_ATTR_SECUREMAP) {
 806		/*
 807		 * If subsystem VMIDs are defined in DTSI, then do
 808		 * hyp_assign from HLOS to those VM(s)
 809		 */
 810		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
 811		struct qcom_scm_vmperm dst_perms[2] = {0};
 812
 813		dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
 814		dst_perms[0].perm = QCOM_SCM_PERM_RW;
 815		dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
 816		dst_perms[1].perm = QCOM_SCM_PERM_RWX;
 817		map->attr = attr;
 818		err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2);
 819		if (err) {
 820			dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
 821					map->phys, map->size, err);
 822			goto map_err;
 823		}
 824	}
 825	spin_lock(&fl->lock);
 826	list_add_tail(&map->node, &fl->maps);
 827	spin_unlock(&fl->lock);
 828	*ppmap = map;
 829
 830	return 0;
 831
 832map_err:
 833	dma_buf_detach(map->buf, map->attach);
 834attach_err:
 835	dma_buf_put(map->buf);
 836get_err:
 837	fastrpc_map_put(map);
 838
 839	return err;
 840}
 841
 842/*
 843 * Fastrpc payload buffer with metadata looks like:
 844 *
 845 * >>>>>>  START of METADATA <<<<<<<<<
 846 * +---------------------------------+
 847 * |           Arguments             |
 848 * | type:(union fastrpc_remote_arg)|
 849 * |             (0 - N)             |
 850 * +---------------------------------+
 851 * |         Invoke Buffer list      |
 852 * | type:(struct fastrpc_invoke_buf)|
 853 * |           (0 - N)               |
 854 * +---------------------------------+
 855 * |         Page info list          |
 856 * | type:(struct fastrpc_phy_page)  |
 857 * |             (0 - N)             |
 858 * +---------------------------------+
 859 * |         Optional info           |
 860 * |(can be specific to SoC/Firmware)|
 861 * +---------------------------------+
 862 * >>>>>>>>  END of METADATA <<<<<<<<<
 863 * +---------------------------------+
 864 * |         Inline ARGS             |
 865 * |            (0-N)                |
 866 * +---------------------------------+
 867 */
 868
 869static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
 870{
 871	int size = 0;
 872
 873	size = (sizeof(struct fastrpc_remote_buf) +
 874		sizeof(struct fastrpc_invoke_buf) +
 875		sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
 876		sizeof(u64) * FASTRPC_MAX_FDLIST +
 877		sizeof(u32) * FASTRPC_MAX_CRCLIST;
 878
 879	return size;
 880}
 881
 882static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
 883{
 884	u64 size = 0;
 885	int oix;
 886
 887	size = ALIGN(metalen, FASTRPC_ALIGN);
 888	for (oix = 0; oix < ctx->nbufs; oix++) {
 889		int i = ctx->olaps[oix].raix;
 890
 891		if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
 892
 893			if (ctx->olaps[oix].offset == 0)
 894				size = ALIGN(size, FASTRPC_ALIGN);
 895
 896			size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
 897		}
 898	}
 899
 900	return size;
 901}
 902
 903static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
 904{
 905	struct device *dev = ctx->fl->sctx->dev;
 906	int i, err;
 907
 908	for (i = 0; i < ctx->nscalars; ++i) {
 
 
 
 909
 910		if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
 911		    ctx->args[i].length == 0)
 912			continue;
 913
 914		err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
 915			 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
 916		if (err) {
 917			dev_err(dev, "Error Creating map %d\n", err);
 918			return -EINVAL;
 919		}
 920
 921	}
 922	return 0;
 923}
 924
 925static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
 926{
 927	return (struct fastrpc_invoke_buf *)(&pra[len]);
 928}
 929
 930static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
 931{
 932	return (struct fastrpc_phy_page *)(&buf[len]);
 933}
 934
 935static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
 936{
 937	struct device *dev = ctx->fl->sctx->dev;
 938	union fastrpc_remote_arg *rpra;
 939	struct fastrpc_invoke_buf *list;
 940	struct fastrpc_phy_page *pages;
 941	int inbufs, i, oix, err = 0;
 942	u64 len, rlen, pkt_size;
 943	u64 pg_start, pg_end;
 944	uintptr_t args;
 945	int metalen;
 946
 947	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
 948	metalen = fastrpc_get_meta_size(ctx);
 949	pkt_size = fastrpc_get_payload_size(ctx, metalen);
 950
 951	err = fastrpc_create_maps(ctx);
 952	if (err)
 953		return err;
 954
 955	ctx->msg_sz = pkt_size;
 956
 957	if (ctx->fl->sctx->sid)
 958		err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
 959	else
 960		err = fastrpc_remote_heap_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
 961	if (err)
 962		return err;
 963
 964	memset(ctx->buf->virt, 0, pkt_size);
 965	rpra = ctx->buf->virt;
 966	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
 967	pages = fastrpc_phy_page_start(list, ctx->nscalars);
 
 968	args = (uintptr_t)ctx->buf->virt + metalen;
 969	rlen = pkt_size - metalen;
 970	ctx->rpra = rpra;
 971
 972	for (oix = 0; oix < ctx->nbufs; ++oix) {
 973		int mlen;
 974
 975		i = ctx->olaps[oix].raix;
 976		len = ctx->args[i].length;
 977
 978		rpra[i].buf.pv = 0;
 979		rpra[i].buf.len = len;
 980		list[i].num = len ? 1 : 0;
 981		list[i].pgidx = i;
 982
 983		if (!len)
 984			continue;
 985
 986		if (ctx->maps[i]) {
 987			struct vm_area_struct *vma = NULL;
 988
 989			rpra[i].buf.pv = (u64) ctx->args[i].ptr;
 990			pages[i].addr = ctx->maps[i]->phys;
 991
 992			mmap_read_lock(current->mm);
 993			vma = find_vma(current->mm, ctx->args[i].ptr);
 994			if (vma)
 995				pages[i].addr += (ctx->args[i].ptr & PAGE_MASK) -
 996						 vma->vm_start;
 997			mmap_read_unlock(current->mm);
 998
 999			pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
1000			pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
1001				  PAGE_SHIFT;
1002			pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1003
1004		} else {
1005
1006			if (ctx->olaps[oix].offset == 0) {
1007				rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1008				args = ALIGN(args, FASTRPC_ALIGN);
1009			}
1010
1011			mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1012
1013			if (rlen < mlen)
1014				goto bail;
1015
1016			rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1017			pages[i].addr = ctx->buf->phys -
1018					ctx->olaps[oix].offset +
1019					(pkt_size - rlen);
1020			pages[i].addr = pages[i].addr &	PAGE_MASK;
1021
1022			pg_start = (rpra[i].buf.pv & PAGE_MASK) >> PAGE_SHIFT;
1023			pg_end = ((rpra[i].buf.pv + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1024			pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1025			args = args + mlen;
1026			rlen -= mlen;
1027		}
1028
1029		if (i < inbufs && !ctx->maps[i]) {
1030			void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1031			void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1032
1033			if (!kernel) {
1034				if (copy_from_user(dst, (void __user *)src,
1035						   len)) {
1036					err = -EFAULT;
1037					goto bail;
1038				}
1039			} else {
1040				memcpy(dst, src, len);
1041			}
1042		}
1043	}
1044
1045	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
 
 
1046		list[i].num = ctx->args[i].length ? 1 : 0;
1047		list[i].pgidx = i;
1048		if (ctx->maps[i]) {
1049			pages[i].addr = ctx->maps[i]->phys;
1050			pages[i].size = ctx->maps[i]->size;
1051		}
1052		rpra[i].dma.fd = ctx->args[i].fd;
1053		rpra[i].dma.len = ctx->args[i].length;
1054		rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1055	}
1056
1057bail:
1058	if (err)
1059		dev_err(dev, "Error: get invoke args failed:%d\n", err);
1060
1061	return err;
1062}
1063
1064static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1065			    u32 kernel)
1066{
1067	union fastrpc_remote_arg *rpra = ctx->rpra;
1068	struct fastrpc_user *fl = ctx->fl;
1069	struct fastrpc_map *mmap = NULL;
1070	struct fastrpc_invoke_buf *list;
1071	struct fastrpc_phy_page *pages;
1072	u64 *fdlist;
1073	int i, inbufs, outbufs, handles;
1074
1075	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1076	outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1077	handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1078	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
1079	pages = fastrpc_phy_page_start(list, ctx->nscalars);
1080	fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1081
1082	for (i = inbufs; i < ctx->nbufs; ++i) {
1083		if (!ctx->maps[i]) {
1084			void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1085			void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1086			u64 len = rpra[i].buf.len;
1087
1088			if (!kernel) {
1089				if (copy_to_user((void __user *)dst, src, len))
1090					return -EFAULT;
1091			} else {
1092				memcpy(dst, src, len);
1093			}
1094		}
1095	}
1096
1097	/* Clean up fdlist which is updated by DSP */
1098	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1099		if (!fdlist[i])
1100			break;
1101		if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false))
1102			fastrpc_map_put(mmap);
1103	}
1104
1105	return 0;
1106}
1107
1108static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1109			       struct fastrpc_invoke_ctx *ctx,
1110			       u32 kernel, uint32_t handle)
1111{
1112	struct fastrpc_channel_ctx *cctx;
1113	struct fastrpc_user *fl = ctx->fl;
1114	struct fastrpc_msg *msg = &ctx->msg;
1115	int ret;
1116
1117	cctx = fl->cctx;
1118	msg->pid = fl->tgid;
1119	msg->tid = current->pid;
1120
1121	if (kernel)
1122		msg->pid = 0;
1123
1124	msg->ctx = ctx->ctxid | fl->pd;
1125	msg->handle = handle;
1126	msg->sc = ctx->sc;
1127	msg->addr = ctx->buf ? ctx->buf->phys : 0;
1128	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1129	fastrpc_context_get(ctx);
1130
1131	ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
1132
1133	if (ret)
1134		fastrpc_context_put(ctx);
1135
1136	return ret;
1137
1138}
1139
1140static int fastrpc_internal_invoke(struct fastrpc_user *fl,  u32 kernel,
1141				   u32 handle, u32 sc,
1142				   struct fastrpc_invoke_args *args)
1143{
1144	struct fastrpc_invoke_ctx *ctx = NULL;
1145	struct fastrpc_buf *buf, *b;
1146
1147	int err = 0;
1148
1149	if (!fl->sctx)
1150		return -EINVAL;
1151
1152	if (!fl->cctx->rpdev)
1153		return -EPIPE;
1154
1155	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1156		dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n",  handle);
1157		return -EPERM;
1158	}
1159
1160	ctx = fastrpc_context_alloc(fl, kernel, sc, args);
1161	if (IS_ERR(ctx))
1162		return PTR_ERR(ctx);
1163
1164	err = fastrpc_get_args(kernel, ctx);
1165	if (err)
1166		goto bail;
 
 
1167
1168	/* make sure that all CPU memory writes are seen by DSP */
1169	dma_wmb();
1170	/* Send invoke buffer to remote dsp */
1171	err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
1172	if (err)
1173		goto bail;
1174
1175	if (kernel) {
1176		if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1177			err = -ETIMEDOUT;
1178	} else {
1179		err = wait_for_completion_interruptible(&ctx->work);
1180	}
1181
1182	if (err)
1183		goto bail;
1184
1185	/* make sure that all memory writes by DSP are seen by CPU */
1186	dma_rmb();
1187	/* populate all the output buffers with results */
1188	err = fastrpc_put_args(ctx, kernel);
1189	if (err)
1190		goto bail;
1191
1192	/* Check the response from remote dsp */
1193	err = ctx->retval;
1194	if (err)
1195		goto bail;
1196
1197bail:
1198	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1199		/* We are done with this compute context */
1200		spin_lock(&fl->lock);
1201		list_del(&ctx->node);
1202		spin_unlock(&fl->lock);
1203		fastrpc_context_put(ctx);
1204	}
1205
1206	if (err == -ERESTARTSYS) {
1207		list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1208			list_del(&buf->node);
1209			list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);
1210		}
1211	}
1212
1213	if (err)
1214		dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1215
1216	return err;
1217}
1218
1219static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1220{
1221	/* Check if the device node is non-secure and channel is secure*/
1222	if (!fl->is_secure_dev && fl->cctx->secure) {
1223		/*
1224		 * Allow untrusted applications to offload only to Unsigned PD when
1225		 * channel is configured as secure and block untrusted apps on channel
1226		 * that does not support unsigned PD offload
1227		 */
1228		if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1229			dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n");
1230			return true;
1231		}
1232	}
1233
1234	return false;
1235}
1236
1237static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1238					      char __user *argp)
1239{
1240	struct fastrpc_init_create_static init;
1241	struct fastrpc_invoke_args *args;
1242	struct fastrpc_phy_page pages[1];
1243	char *name;
1244	int err;
1245	bool scm_done = false;
1246	struct {
1247		int pgid;
1248		u32 namelen;
1249		u32 pageslen;
1250	} inbuf;
1251	u32 sc;
1252
1253	args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1254	if (!args)
1255		return -ENOMEM;
1256
1257	if (copy_from_user(&init, argp, sizeof(init))) {
1258		err = -EFAULT;
1259		goto err;
1260	}
1261
1262	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1263		err = -EINVAL;
1264		goto err;
1265	}
1266
1267	name = memdup_user(u64_to_user_ptr(init.name), init.namelen);
1268	if (IS_ERR(name)) {
1269		err = PTR_ERR(name);
1270		goto err;
1271	}
1272
1273	if (!fl->cctx->remote_heap) {
1274		err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
1275						&fl->cctx->remote_heap);
1276		if (err)
1277			goto err_name;
1278
1279		/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1280		if (fl->cctx->vmcount) {
1281			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1282
1283			err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1284							(u64)fl->cctx->remote_heap->size,
1285							&src_perms,
1286							fl->cctx->vmperms, fl->cctx->vmcount);
1287			if (err) {
1288				dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
1289					fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1290				goto err_map;
1291			}
1292			scm_done = true;
1293		}
1294	}
1295
1296	inbuf.pgid = fl->tgid;
1297	inbuf.namelen = init.namelen;
1298	inbuf.pageslen = 0;
1299	fl->pd = USER_PD;
1300
1301	args[0].ptr = (u64)(uintptr_t)&inbuf;
1302	args[0].length = sizeof(inbuf);
1303	args[0].fd = -1;
1304
1305	args[1].ptr = (u64)(uintptr_t)name;
1306	args[1].length = inbuf.namelen;
1307	args[1].fd = -1;
1308
1309	pages[0].addr = fl->cctx->remote_heap->phys;
1310	pages[0].size = fl->cctx->remote_heap->size;
1311
1312	args[2].ptr = (u64)(uintptr_t) pages;
1313	args[2].length = sizeof(*pages);
1314	args[2].fd = -1;
1315
1316	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1317
1318	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1319				      sc, args);
1320	if (err)
1321		goto err_invoke;
1322
1323	kfree(args);
1324	kfree(name);
1325
1326	return 0;
1327err_invoke:
1328	if (fl->cctx->vmcount && scm_done) {
1329		u64 src_perms = 0;
1330		struct qcom_scm_vmperm dst_perms;
1331		u32 i;
1332
1333		for (i = 0; i < fl->cctx->vmcount; i++)
1334			src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1335
1336		dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1337		dst_perms.perm = QCOM_SCM_PERM_RWX;
1338		err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1339						(u64)fl->cctx->remote_heap->size,
1340						&src_perms, &dst_perms, 1);
1341		if (err)
1342			dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
1343				fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1344	}
1345err_map:
1346	fastrpc_buf_free(fl->cctx->remote_heap);
1347err_name:
1348	kfree(name);
1349err:
1350	kfree(args);
1351
1352	return err;
1353}
1354
1355static int fastrpc_init_create_process(struct fastrpc_user *fl,
1356					char __user *argp)
1357{
1358	struct fastrpc_init_create init;
1359	struct fastrpc_invoke_args *args;
1360	struct fastrpc_phy_page pages[1];
1361	struct fastrpc_map *map = NULL;
1362	struct fastrpc_buf *imem = NULL;
1363	int memlen;
1364	int err;
1365	struct {
1366		int pgid;
1367		u32 namelen;
1368		u32 filelen;
1369		u32 pageslen;
1370		u32 attrs;
1371		u32 siglen;
1372	} inbuf;
1373	u32 sc;
1374	bool unsigned_module = false;
1375
1376	args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1377	if (!args)
1378		return -ENOMEM;
1379
1380	if (copy_from_user(&init, argp, sizeof(init))) {
1381		err = -EFAULT;
1382		goto err;
1383	}
1384
1385	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1386		unsigned_module = true;
1387
1388	if (is_session_rejected(fl, unsigned_module)) {
1389		err = -ECONNREFUSED;
1390		goto err;
1391	}
1392
1393	if (init.filelen > INIT_FILELEN_MAX) {
1394		err = -EINVAL;
1395		goto err;
1396	}
1397
1398	inbuf.pgid = fl->tgid;
1399	inbuf.namelen = strlen(current->comm) + 1;
1400	inbuf.filelen = init.filelen;
1401	inbuf.pageslen = 1;
1402	inbuf.attrs = init.attrs;
1403	inbuf.siglen = init.siglen;
1404	fl->pd = USER_PD;
1405
1406	if (init.filelen && init.filefd) {
1407		err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);
1408		if (err)
1409			goto err;
1410	}
1411
1412	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1413		       1024 * 1024);
1414	err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1415				&imem);
1416	if (err)
1417		goto err_alloc;
1418
1419	fl->init_mem = imem;
1420	args[0].ptr = (u64)(uintptr_t)&inbuf;
1421	args[0].length = sizeof(inbuf);
1422	args[0].fd = -1;
1423
1424	args[1].ptr = (u64)(uintptr_t)current->comm;
1425	args[1].length = inbuf.namelen;
1426	args[1].fd = -1;
1427
1428	args[2].ptr = (u64) init.file;
1429	args[2].length = inbuf.filelen;
1430	args[2].fd = init.filefd;
1431
1432	pages[0].addr = imem->phys;
1433	pages[0].size = imem->size;
1434
1435	args[3].ptr = (u64)(uintptr_t) pages;
1436	args[3].length = 1 * sizeof(*pages);
1437	args[3].fd = -1;
1438
1439	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1440	args[4].length = sizeof(inbuf.attrs);
1441	args[4].fd = -1;
1442
1443	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1444	args[5].length = sizeof(inbuf.siglen);
1445	args[5].fd = -1;
1446
1447	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1448	if (init.attrs)
1449		sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1450
1451	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1452				      sc, args);
1453	if (err)
1454		goto err_invoke;
1455
1456	kfree(args);
1457
1458	return 0;
1459
1460err_invoke:
1461	fl->init_mem = NULL;
1462	fastrpc_buf_free(imem);
1463err_alloc:
1464	fastrpc_map_put(map);
 
 
 
 
 
1465err:
1466	kfree(args);
1467
1468	return err;
1469}
1470
1471static struct fastrpc_session_ctx *fastrpc_session_alloc(
1472					struct fastrpc_channel_ctx *cctx)
1473{
1474	struct fastrpc_session_ctx *session = NULL;
1475	unsigned long flags;
1476	int i;
1477
1478	spin_lock_irqsave(&cctx->lock, flags);
1479	for (i = 0; i < cctx->sesscount; i++) {
1480		if (!cctx->session[i].used && cctx->session[i].valid) {
1481			cctx->session[i].used = true;
1482			session = &cctx->session[i];
1483			break;
1484		}
1485	}
1486	spin_unlock_irqrestore(&cctx->lock, flags);
1487
1488	return session;
1489}
1490
1491static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1492				 struct fastrpc_session_ctx *session)
1493{
1494	unsigned long flags;
1495
1496	spin_lock_irqsave(&cctx->lock, flags);
1497	session->used = false;
1498	spin_unlock_irqrestore(&cctx->lock, flags);
1499}
1500
1501static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1502{
1503	struct fastrpc_invoke_args args[1];
1504	int tgid = 0;
1505	u32 sc;
1506
1507	tgid = fl->tgid;
1508	args[0].ptr = (u64)(uintptr_t) &tgid;
1509	args[0].length = sizeof(tgid);
1510	args[0].fd = -1;
 
1511	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1512
1513	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1514				       sc, &args[0]);
1515}
1516
1517static int fastrpc_device_release(struct inode *inode, struct file *file)
1518{
1519	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1520	struct fastrpc_channel_ctx *cctx = fl->cctx;
1521	struct fastrpc_invoke_ctx *ctx, *n;
1522	struct fastrpc_map *map, *m;
1523	struct fastrpc_buf *buf, *b;
1524	unsigned long flags;
1525
1526	fastrpc_release_current_dsp_process(fl);
1527
1528	spin_lock_irqsave(&cctx->lock, flags);
1529	list_del(&fl->user);
1530	spin_unlock_irqrestore(&cctx->lock, flags);
1531
1532	if (fl->init_mem)
1533		fastrpc_buf_free(fl->init_mem);
1534
1535	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1536		list_del(&ctx->node);
1537		fastrpc_context_put(ctx);
1538	}
1539
1540	list_for_each_entry_safe(map, m, &fl->maps, node)
 
1541		fastrpc_map_put(map);
1542
1543	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1544		list_del(&buf->node);
1545		fastrpc_buf_free(buf);
1546	}
1547
1548	fastrpc_session_free(cctx, fl->sctx);
1549	fastrpc_channel_ctx_put(cctx);
1550
1551	mutex_destroy(&fl->mutex);
1552	kfree(fl);
1553	file->private_data = NULL;
1554
1555	return 0;
1556}
1557
1558static int fastrpc_device_open(struct inode *inode, struct file *filp)
1559{
1560	struct fastrpc_channel_ctx *cctx;
1561	struct fastrpc_device *fdevice;
1562	struct fastrpc_user *fl = NULL;
1563	unsigned long flags;
1564
1565	fdevice = miscdev_to_fdevice(filp->private_data);
1566	cctx = fdevice->cctx;
1567
1568	fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1569	if (!fl)
1570		return -ENOMEM;
1571
1572	/* Released in fastrpc_device_release() */
1573	fastrpc_channel_ctx_get(cctx);
1574
1575	filp->private_data = fl;
1576	spin_lock_init(&fl->lock);
1577	mutex_init(&fl->mutex);
1578	INIT_LIST_HEAD(&fl->pending);
1579	INIT_LIST_HEAD(&fl->maps);
1580	INIT_LIST_HEAD(&fl->mmaps);
1581	INIT_LIST_HEAD(&fl->user);
1582	fl->tgid = current->tgid;
1583	fl->cctx = cctx;
1584	fl->is_secure_dev = fdevice->secure;
1585
1586	fl->sctx = fastrpc_session_alloc(cctx);
1587	if (!fl->sctx) {
1588		dev_err(&cctx->rpdev->dev, "No session available\n");
1589		mutex_destroy(&fl->mutex);
1590		kfree(fl);
1591
1592		return -EBUSY;
1593	}
1594
1595	spin_lock_irqsave(&cctx->lock, flags);
1596	list_add_tail(&fl->user, &cctx->users);
1597	spin_unlock_irqrestore(&cctx->lock, flags);
1598
1599	return 0;
1600}
1601
1602static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1603{
1604	struct fastrpc_alloc_dma_buf bp;
1605	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1606	struct fastrpc_buf *buf = NULL;
1607	int err;
1608
1609	if (copy_from_user(&bp, argp, sizeof(bp)))
1610		return -EFAULT;
1611
1612	err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1613	if (err)
1614		return err;
1615	exp_info.ops = &fastrpc_dma_buf_ops;
1616	exp_info.size = bp.size;
1617	exp_info.flags = O_RDWR;
1618	exp_info.priv = buf;
1619	buf->dmabuf = dma_buf_export(&exp_info);
1620	if (IS_ERR(buf->dmabuf)) {
1621		err = PTR_ERR(buf->dmabuf);
1622		fastrpc_buf_free(buf);
1623		return err;
1624	}
1625
1626	bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1627	if (bp.fd < 0) {
1628		dma_buf_put(buf->dmabuf);
1629		return -EINVAL;
1630	}
1631
1632	if (copy_to_user(argp, &bp, sizeof(bp))) {
1633		/*
1634		 * The usercopy failed, but we can't do much about it, as
1635		 * dma_buf_fd() already called fd_install() and made the
1636		 * file descriptor accessible for the current process. It
1637		 * might already be closed and dmabuf no longer valid when
1638		 * we reach this point. Therefore "leak" the fd and rely on
1639		 * the process exit path to do any required cleanup.
1640		 */
1641		return -EFAULT;
1642	}
1643
1644	return 0;
1645}
1646
1647static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1648{
1649	struct fastrpc_invoke_args args[1];
1650	int tgid = fl->tgid;
1651	u32 sc;
1652
1653	args[0].ptr = (u64)(uintptr_t) &tgid;
1654	args[0].length = sizeof(tgid);
1655	args[0].fd = -1;
 
1656	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1657	fl->pd = pd;
1658
1659	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1660				       sc, &args[0]);
1661}
1662
1663static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1664{
1665	struct fastrpc_invoke_args *args = NULL;
1666	struct fastrpc_invoke inv;
1667	u32 nscalars;
1668	int err;
1669
1670	if (copy_from_user(&inv, argp, sizeof(inv)))
1671		return -EFAULT;
1672
1673	/* nscalars is truncated here to max supported value */
1674	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1675	if (nscalars) {
1676		args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1677		if (!args)
1678			return -ENOMEM;
1679
1680		if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1681				   nscalars * sizeof(*args))) {
1682			kfree(args);
1683			return -EFAULT;
1684		}
1685	}
1686
1687	err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1688	kfree(args);
1689
1690	return err;
1691}
1692
1693static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1694				     uint32_t dsp_attr_buf_len)
1695{
1696	struct fastrpc_invoke_args args[2] = { 0 };
1697
1698	/*
1699	 * Capability filled in userspace. This carries the information
1700	 * about the remoteproc support which is fetched from the remoteproc
1701	 * sysfs node by userspace.
1702	 */
1703	dsp_attr_buf[0] = 0;
1704	dsp_attr_buf_len -= 1;
1705
1706	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1707	args[0].length = sizeof(dsp_attr_buf_len);
1708	args[0].fd = -1;
1709	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1710	args[1].length = dsp_attr_buf_len * sizeof(u32);
1711	args[1].fd = -1;
1712
1713	return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE,
1714				       FASTRPC_SCALARS(0, 1, 1), args);
1715}
1716
1717static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1718					struct fastrpc_user *fl)
1719{
1720	struct fastrpc_channel_ctx *cctx = fl->cctx;
1721	uint32_t attribute_id = cap->attribute_id;
1722	uint32_t *dsp_attributes;
1723	unsigned long flags;
1724	uint32_t domain = cap->domain;
1725	int err;
1726
1727	spin_lock_irqsave(&cctx->lock, flags);
1728	/* check if we already have queried dsp for attributes */
1729	if (cctx->valid_attributes) {
1730		spin_unlock_irqrestore(&cctx->lock, flags);
1731		goto done;
1732	}
1733	spin_unlock_irqrestore(&cctx->lock, flags);
1734
1735	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1736	if (!dsp_attributes)
1737		return -ENOMEM;
1738
1739	err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES);
1740	if (err == DSP_UNSUPPORTED_API) {
1741		dev_info(&cctx->rpdev->dev,
1742			 "Warning: DSP capabilities not supported on domain: %d\n", domain);
1743		kfree(dsp_attributes);
1744		return -EOPNOTSUPP;
1745	} else if (err) {
1746		dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1747		kfree(dsp_attributes);
1748		return err;
1749	}
1750
1751	spin_lock_irqsave(&cctx->lock, flags);
1752	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1753	cctx->valid_attributes = true;
1754	spin_unlock_irqrestore(&cctx->lock, flags);
1755	kfree(dsp_attributes);
1756done:
1757	cap->capability = cctx->dsp_attributes[attribute_id];
1758	return 0;
1759}
1760
1761static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1762{
1763	struct fastrpc_ioctl_capability cap = {0};
1764	int err = 0;
1765
1766	if (copy_from_user(&cap, argp, sizeof(cap)))
1767		return  -EFAULT;
1768
1769	cap.capability = 0;
1770	if (cap.domain >= FASTRPC_DEV_MAX) {
1771		dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",
1772			cap.domain, err);
1773		return -ECHRNG;
1774	}
1775
1776	/* Fastrpc Capablities does not support modem domain */
1777	if (cap.domain == MDSP_DOMAIN_ID) {
1778		dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);
1779		return -ECHRNG;
1780	}
1781
1782	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1783		dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1784			cap.attribute_id, err);
1785		return -EOVERFLOW;
1786	}
1787
1788	err = fastrpc_get_info_from_kernel(&cap, fl);
1789	if (err)
1790		return err;
1791
1792	if (copy_to_user(argp, &cap, sizeof(cap)))
1793		return -EFAULT;
1794
1795	return 0;
1796}
1797
1798static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1799{
1800	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1801	struct fastrpc_munmap_req_msg req_msg;
1802	struct device *dev = fl->sctx->dev;
1803	int err;
1804	u32 sc;
1805
1806	req_msg.pgid = fl->tgid;
1807	req_msg.size = buf->size;
1808	req_msg.vaddr = buf->raddr;
1809
1810	args[0].ptr = (u64) (uintptr_t) &req_msg;
1811	args[0].length = sizeof(req_msg);
1812
1813	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1814	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1815				      &args[0]);
1816	if (!err) {
1817		dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1818		spin_lock(&fl->lock);
1819		list_del(&buf->node);
1820		spin_unlock(&fl->lock);
1821		fastrpc_buf_free(buf);
1822	} else {
1823		dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1824	}
1825
1826	return err;
1827}
1828
1829static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1830{
1831	struct fastrpc_buf *buf = NULL, *iter, *b;
1832	struct fastrpc_req_munmap req;
1833	struct device *dev = fl->sctx->dev;
1834
1835	if (copy_from_user(&req, argp, sizeof(req)))
1836		return -EFAULT;
1837
1838	spin_lock(&fl->lock);
1839	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1840		if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1841			buf = iter;
1842			break;
1843		}
1844	}
1845	spin_unlock(&fl->lock);
1846
1847	if (!buf) {
1848		dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1849			req.vaddrout, req.size);
1850		return -EINVAL;
1851	}
1852
1853	return fastrpc_req_munmap_impl(fl, buf);
1854}
1855
1856static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1857{
1858	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1859	struct fastrpc_buf *buf = NULL;
1860	struct fastrpc_mmap_req_msg req_msg;
1861	struct fastrpc_mmap_rsp_msg rsp_msg;
1862	struct fastrpc_phy_page pages;
1863	struct fastrpc_req_mmap req;
1864	struct device *dev = fl->sctx->dev;
1865	int err;
1866	u32 sc;
1867
1868	if (copy_from_user(&req, argp, sizeof(req)))
1869		return -EFAULT;
1870
1871	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1872		dev_err(dev, "flag not supported 0x%x\n", req.flags);
1873
1874		return -EINVAL;
1875	}
1876
1877	if (req.vaddrin) {
1878		dev_err(dev, "adding user allocated pages is not supported\n");
1879		return -EINVAL;
1880	}
1881
1882	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1883		err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);
1884	else
1885		err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
1886
1887	if (err) {
1888		dev_err(dev, "failed to allocate buffer\n");
1889		return err;
1890	}
1891
1892	req_msg.pgid = fl->tgid;
1893	req_msg.flags = req.flags;
1894	req_msg.vaddr = req.vaddrin;
1895	req_msg.num = sizeof(pages);
1896
1897	args[0].ptr = (u64) (uintptr_t) &req_msg;
1898	args[0].length = sizeof(req_msg);
1899
1900	pages.addr = buf->phys;
1901	pages.size = buf->size;
1902
1903	args[1].ptr = (u64) (uintptr_t) &pages;
1904	args[1].length = sizeof(pages);
1905
1906	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1907	args[2].length = sizeof(rsp_msg);
1908
1909	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1910	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1911				      &args[0]);
1912	if (err) {
1913		dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1914		fastrpc_buf_free(buf);
1915		return err;
1916	}
1917
1918	/* update the buffer to be able to deallocate the memory on the DSP */
1919	buf->raddr = (uintptr_t) rsp_msg.vaddr;
1920
1921	/* let the client know the address to use */
1922	req.vaddrout = rsp_msg.vaddr;
1923
1924	/* Add memory to static PD pool, protection thru hypervisor */
1925	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1926		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1927
1928		err = qcom_scm_assign_mem(buf->phys, (u64)buf->size,
1929			&src_perms, fl->cctx->vmperms, fl->cctx->vmcount);
1930		if (err) {
1931			dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1932					buf->phys, buf->size, err);
1933			goto err_assign;
1934		}
1935	}
1936
1937	spin_lock(&fl->lock);
1938	list_add_tail(&buf->node, &fl->mmaps);
1939	spin_unlock(&fl->lock);
1940
1941	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1942		err = -EFAULT;
1943		goto err_assign;
1944	}
1945
1946	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1947		buf->raddr, buf->size);
1948
1949	return 0;
1950
1951err_assign:
1952	fastrpc_req_munmap_impl(fl, buf);
1953
1954	return err;
1955}
1956
1957static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1958{
1959	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1960	struct fastrpc_map *map = NULL, *iter, *m;
1961	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1962	int err = 0;
1963	u32 sc;
1964	struct device *dev = fl->sctx->dev;
1965
1966	spin_lock(&fl->lock);
1967	list_for_each_entry_safe(iter, m, &fl->maps, node) {
1968		if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1969			map = iter;
1970			break;
1971		}
1972	}
1973
1974	spin_unlock(&fl->lock);
1975
1976	if (!map) {
1977		dev_err(dev, "map not in list\n");
1978		return -EINVAL;
1979	}
1980
1981	req_msg.pgid = fl->tgid;
1982	req_msg.len = map->len;
1983	req_msg.vaddrin = map->raddr;
1984	req_msg.fd = map->fd;
1985
1986	args[0].ptr = (u64) (uintptr_t) &req_msg;
1987	args[0].length = sizeof(req_msg);
1988
1989	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
1990	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1991				      &args[0]);
1992	if (err) {
1993		dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n",  map->fd, map->raddr);
1994		return err;
1995	}
1996	fastrpc_map_put(map);
1997
1998	return 0;
1999}
2000
2001static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
2002{
2003	struct fastrpc_mem_unmap req;
2004
2005	if (copy_from_user(&req, argp, sizeof(req)))
2006		return -EFAULT;
2007
2008	return fastrpc_req_mem_unmap_impl(fl, &req);
2009}
2010
2011static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2012{
2013	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2014	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2015	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2016	struct fastrpc_mem_unmap req_unmap = { 0 };
2017	struct fastrpc_phy_page pages = { 0 };
2018	struct fastrpc_mem_map req;
2019	struct device *dev = fl->sctx->dev;
2020	struct fastrpc_map *map = NULL;
2021	int err;
2022	u32 sc;
2023
2024	if (copy_from_user(&req, argp, sizeof(req)))
2025		return -EFAULT;
2026
2027	/* create SMMU mapping */
2028	err = fastrpc_map_create(fl, req.fd, req.length, 0, &map);
2029	if (err) {
2030		dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2031		return err;
2032	}
2033
2034	req_msg.pgid = fl->tgid;
2035	req_msg.fd = req.fd;
2036	req_msg.offset = req.offset;
2037	req_msg.vaddrin = req.vaddrin;
2038	map->va = (void *) (uintptr_t) req.vaddrin;
2039	req_msg.flags = req.flags;
2040	req_msg.num = sizeof(pages);
2041	req_msg.data_len = 0;
2042
2043	args[0].ptr = (u64) (uintptr_t) &req_msg;
2044	args[0].length = sizeof(req_msg);
2045
2046	pages.addr = map->phys;
2047	pages.size = map->size;
2048
2049	args[1].ptr = (u64) (uintptr_t) &pages;
2050	args[1].length = sizeof(pages);
2051
2052	args[2].ptr = (u64) (uintptr_t) &pages;
2053	args[2].length = 0;
2054
2055	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2056	args[3].length = sizeof(rsp_msg);
2057
2058	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2059	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);
2060	if (err) {
2061		dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2062			req.fd, req.vaddrin, map->size);
2063		goto err_invoke;
2064	}
2065
2066	/* update the buffer to be able to deallocate the memory on the DSP */
2067	map->raddr = rsp_msg.vaddr;
2068
2069	/* let the client know the address to use */
2070	req.vaddrout = rsp_msg.vaddr;
2071
2072	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
2073		/* unmap the memory and release the buffer */
2074		req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2075		req_unmap.length = map->size;
2076		fastrpc_req_mem_unmap_impl(fl, &req_unmap);
2077		return -EFAULT;
2078	}
2079
2080	return 0;
2081
2082err_invoke:
2083	fastrpc_map_put(map);
2084
2085	return err;
2086}
2087
2088static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2089				 unsigned long arg)
2090{
2091	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2092	char __user *argp = (char __user *)arg;
2093	int err;
2094
2095	switch (cmd) {
2096	case FASTRPC_IOCTL_INVOKE:
2097		err = fastrpc_invoke(fl, argp);
2098		break;
2099	case FASTRPC_IOCTL_INIT_ATTACH:
2100		err = fastrpc_init_attach(fl, ROOT_PD);
2101		break;
2102	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2103		err = fastrpc_init_attach(fl, SENSORS_PD);
2104		break;
2105	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2106		err = fastrpc_init_create_static_process(fl, argp);
2107		break;
2108	case FASTRPC_IOCTL_INIT_CREATE:
2109		err = fastrpc_init_create_process(fl, argp);
2110		break;
2111	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2112		err = fastrpc_dmabuf_alloc(fl, argp);
2113		break;
2114	case FASTRPC_IOCTL_MMAP:
2115		err = fastrpc_req_mmap(fl, argp);
2116		break;
2117	case FASTRPC_IOCTL_MUNMAP:
2118		err = fastrpc_req_munmap(fl, argp);
2119		break;
2120	case FASTRPC_IOCTL_MEM_MAP:
2121		err = fastrpc_req_mem_map(fl, argp);
2122		break;
2123	case FASTRPC_IOCTL_MEM_UNMAP:
2124		err = fastrpc_req_mem_unmap(fl, argp);
2125		break;
2126	case FASTRPC_IOCTL_GET_DSP_INFO:
2127		err = fastrpc_get_dsp_info(fl, argp);
2128		break;
2129	default:
2130		err = -ENOTTY;
2131		break;
2132	}
2133
2134	return err;
2135}
2136
2137static const struct file_operations fastrpc_fops = {
2138	.open = fastrpc_device_open,
2139	.release = fastrpc_device_release,
2140	.unlocked_ioctl = fastrpc_device_ioctl,
2141	.compat_ioctl = fastrpc_device_ioctl,
2142};
2143
2144static int fastrpc_cb_probe(struct platform_device *pdev)
2145{
2146	struct fastrpc_channel_ctx *cctx;
2147	struct fastrpc_session_ctx *sess;
2148	struct device *dev = &pdev->dev;
2149	int i, sessions = 0;
2150	unsigned long flags;
2151	int rc;
2152
2153	cctx = dev_get_drvdata(dev->parent);
2154	if (!cctx)
2155		return -EINVAL;
2156
2157	of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
2158
2159	spin_lock_irqsave(&cctx->lock, flags);
2160	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2161		dev_err(&pdev->dev, "too many sessions\n");
2162		spin_unlock_irqrestore(&cctx->lock, flags);
2163		return -ENOSPC;
2164	}
2165	sess = &cctx->session[cctx->sesscount++];
2166	sess->used = false;
2167	sess->valid = true;
2168	sess->dev = dev;
2169	dev_set_drvdata(dev, sess);
2170
2171	if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
2172		dev_info(dev, "FastRPC Session ID not specified in DT\n");
2173
2174	if (sessions > 0) {
2175		struct fastrpc_session_ctx *dup_sess;
2176
2177		for (i = 1; i < sessions; i++) {
2178			if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2179				break;
2180			dup_sess = &cctx->session[cctx->sesscount++];
2181			memcpy(dup_sess, sess, sizeof(*dup_sess));
2182		}
2183	}
 
2184	spin_unlock_irqrestore(&cctx->lock, flags);
2185	rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2186	if (rc) {
2187		dev_err(dev, "32-bit DMA enable failed\n");
2188		return rc;
2189	}
2190
2191	return 0;
2192}
2193
2194static void fastrpc_cb_remove(struct platform_device *pdev)
2195{
2196	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
2197	struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
2198	unsigned long flags;
2199	int i;
2200
2201	spin_lock_irqsave(&cctx->lock, flags);
2202	for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2203		if (cctx->session[i].sid == sess->sid) {
2204			cctx->session[i].valid = false;
2205			cctx->sesscount--;
2206		}
2207	}
2208	spin_unlock_irqrestore(&cctx->lock, flags);
 
 
2209}
2210
2211static const struct of_device_id fastrpc_match_table[] = {
2212	{ .compatible = "qcom,fastrpc-compute-cb", },
2213	{}
2214};
2215
2216static struct platform_driver fastrpc_cb_driver = {
2217	.probe = fastrpc_cb_probe,
2218	.remove = fastrpc_cb_remove,
2219	.driver = {
2220		.name = "qcom,fastrpc-cb",
2221		.of_match_table = fastrpc_match_table,
2222		.suppress_bind_attrs = true,
2223	},
2224};
2225
2226static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2227				   bool is_secured, const char *domain)
2228{
2229	struct fastrpc_device *fdev;
2230	int err;
2231
2232	fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL);
2233	if (!fdev)
2234		return -ENOMEM;
2235
2236	fdev->secure = is_secured;
2237	fdev->cctx = cctx;
2238	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2239	fdev->miscdev.fops = &fastrpc_fops;
2240	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s",
2241					    domain, is_secured ? "-secure" : "");
2242	if (!fdev->miscdev.name)
2243		return -ENOMEM;
2244
2245	err = misc_register(&fdev->miscdev);
2246	if (!err) {
2247		if (is_secured)
2248			cctx->secure_fdevice = fdev;
2249		else
2250			cctx->fdevice = fdev;
2251	}
2252
2253	return err;
2254}
2255
2256static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2257{
2258	struct device *rdev = &rpdev->dev;
2259	struct fastrpc_channel_ctx *data;
2260	int i, err, domain_id = -1, vmcount;
2261	const char *domain;
2262	bool secure_dsp;
2263	struct device_node *rmem_node;
2264	struct reserved_mem *rmem;
2265	unsigned int vmids[FASTRPC_MAX_VMIDS];
2266
2267	err = of_property_read_string(rdev->of_node, "label", &domain);
2268	if (err) {
2269		dev_info(rdev, "FastRPC Domain not specified in DT\n");
2270		return err;
2271	}
2272
2273	for (i = 0; i < FASTRPC_DEV_MAX; i++) {
2274		if (!strcmp(domains[i], domain)) {
2275			domain_id = i;
2276			break;
2277		}
2278	}
2279
2280	if (domain_id < 0) {
2281		dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2282		return -EINVAL;
2283	}
2284
2285	if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))
2286		dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2287
2288	vmcount = of_property_read_variable_u32_array(rdev->of_node,
2289				"qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
2290	if (vmcount < 0)
2291		vmcount = 0;
2292	else if (!qcom_scm_is_available())
2293		return -EPROBE_DEFER;
2294
2295	data = kzalloc(sizeof(*data), GFP_KERNEL);
2296	if (!data)
2297		return -ENOMEM;
2298
2299	if (vmcount) {
2300		data->vmcount = vmcount;
2301		for (i = 0; i < data->vmcount; i++) {
2302			data->vmperms[i].vmid = vmids[i];
2303			data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2304		}
2305	}
2306
2307	rmem_node = of_parse_phandle(rdev->of_node, "memory-region", 0);
2308	if (domain_id == SDSP_DOMAIN_ID && rmem_node) {
2309		u64 src_perms;
2310
2311		rmem = of_reserved_mem_lookup(rmem_node);
2312		if (!rmem) {
2313			err = -EINVAL;
2314			goto fdev_error;
2315		}
2316
2317		src_perms = BIT(QCOM_SCM_VMID_HLOS);
2318
2319		qcom_scm_assign_mem(rmem->base, rmem->size, &src_perms,
2320				    data->vmperms, data->vmcount);
2321
2322	}
2323
2324	secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2325	data->secure = secure_dsp;
2326
2327	switch (domain_id) {
2328	case ADSP_DOMAIN_ID:
2329	case MDSP_DOMAIN_ID:
2330	case SDSP_DOMAIN_ID:
2331		/* Unsigned PD offloading is only supported on CDSP and CDSP1 */
2332		data->unsigned_support = false;
2333		err = fastrpc_device_register(rdev, data, secure_dsp, domains[domain_id]);
2334		if (err)
2335			goto fdev_error;
2336		break;
2337	case CDSP_DOMAIN_ID:
2338	case CDSP1_DOMAIN_ID:
2339		data->unsigned_support = true;
2340		/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2341		err = fastrpc_device_register(rdev, data, true, domains[domain_id]);
2342		if (err)
2343			goto fdev_error;
2344
2345		err = fastrpc_device_register(rdev, data, false, domains[domain_id]);
2346		if (err)
2347			goto populate_error;
2348		break;
2349	default:
2350		err = -EINVAL;
2351		goto fdev_error;
2352	}
2353
2354	kref_init(&data->refcount);
2355
2356	dev_set_drvdata(&rpdev->dev, data);
2357	rdev->dma_mask = &data->dma_mask;
2358	dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
2359	INIT_LIST_HEAD(&data->users);
2360	INIT_LIST_HEAD(&data->invoke_interrupted_mmaps);
2361	spin_lock_init(&data->lock);
2362	idr_init(&data->ctx_idr);
2363	data->domain_id = domain_id;
2364	data->rpdev = rpdev;
2365
2366	err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);
2367	if (err)
2368		goto populate_error;
2369
2370	return 0;
2371
2372populate_error:
2373	if (data->fdevice)
2374		misc_deregister(&data->fdevice->miscdev);
2375	if (data->secure_fdevice)
2376		misc_deregister(&data->secure_fdevice->miscdev);
2377
2378fdev_error:
2379	kfree(data);
2380	return err;
2381}
2382
2383static void fastrpc_notify_users(struct fastrpc_user *user)
2384{
2385	struct fastrpc_invoke_ctx *ctx;
2386
2387	spin_lock(&user->lock);
2388	list_for_each_entry(ctx, &user->pending, node) {
2389		ctx->retval = -EPIPE;
2390		complete(&ctx->work);
2391	}
2392	spin_unlock(&user->lock);
2393}
2394
2395static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2396{
2397	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2398	struct fastrpc_buf *buf, *b;
2399	struct fastrpc_user *user;
2400	unsigned long flags;
2401
2402	/* No invocations past this point */
2403	spin_lock_irqsave(&cctx->lock, flags);
2404	cctx->rpdev = NULL;
2405	list_for_each_entry(user, &cctx->users, user)
2406		fastrpc_notify_users(user);
2407	spin_unlock_irqrestore(&cctx->lock, flags);
2408
2409	if (cctx->fdevice)
2410		misc_deregister(&cctx->fdevice->miscdev);
2411
2412	if (cctx->secure_fdevice)
2413		misc_deregister(&cctx->secure_fdevice->miscdev);
2414
2415	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2416		list_del(&buf->node);
2417
2418	if (cctx->remote_heap)
2419		fastrpc_buf_free(cctx->remote_heap);
2420
2421	of_platform_depopulate(&rpdev->dev);
2422
 
2423	fastrpc_channel_ctx_put(cctx);
2424}
2425
2426static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2427				  int len, void *priv, u32 addr)
2428{
2429	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2430	struct fastrpc_invoke_rsp *rsp = data;
2431	struct fastrpc_invoke_ctx *ctx;
2432	unsigned long flags;
2433	unsigned long ctxid;
2434
2435	if (len < sizeof(*rsp))
2436		return -EINVAL;
2437
2438	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2439
2440	spin_lock_irqsave(&cctx->lock, flags);
2441	ctx = idr_find(&cctx->ctx_idr, ctxid);
2442	spin_unlock_irqrestore(&cctx->lock, flags);
2443
2444	if (!ctx) {
2445		dev_err(&rpdev->dev, "No context ID matches response\n");
2446		return -ENOENT;
2447	}
2448
2449	ctx->retval = rsp->retval;
2450	complete(&ctx->work);
2451
2452	/*
2453	 * The DMA buffer associated with the context cannot be freed in
2454	 * interrupt context so schedule it through a worker thread to
2455	 * avoid a kernel BUG.
2456	 */
2457	schedule_work(&ctx->put_work);
2458
2459	return 0;
2460}
2461
2462static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2463	{ .compatible = "qcom,fastrpc" },
2464	{ },
2465};
2466MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2467
2468static struct rpmsg_driver fastrpc_driver = {
2469	.probe = fastrpc_rpmsg_probe,
2470	.remove = fastrpc_rpmsg_remove,
2471	.callback = fastrpc_rpmsg_callback,
2472	.drv = {
2473		.name = "qcom,fastrpc",
2474		.of_match_table = fastrpc_rpmsg_of_match,
2475	},
2476};
2477
2478static int fastrpc_init(void)
2479{
2480	int ret;
2481
2482	ret = platform_driver_register(&fastrpc_cb_driver);
2483	if (ret < 0) {
2484		pr_err("fastrpc: failed to register cb driver\n");
2485		return ret;
2486	}
2487
2488	ret = register_rpmsg_driver(&fastrpc_driver);
2489	if (ret < 0) {
2490		pr_err("fastrpc: failed to register rpmsg driver\n");
2491		platform_driver_unregister(&fastrpc_cb_driver);
2492		return ret;
2493	}
2494
2495	return 0;
2496}
2497module_init(fastrpc_init);
2498
2499static void fastrpc_exit(void)
2500{
2501	platform_driver_unregister(&fastrpc_cb_driver);
2502	unregister_rpmsg_driver(&fastrpc_driver);
2503}
2504module_exit(fastrpc_exit);
2505
2506MODULE_DESCRIPTION("Qualcomm FastRPC");
2507MODULE_LICENSE("GPL v2");
2508MODULE_IMPORT_NS("DMA_BUF");