Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2015-2021, 2023 Linaro Limited
4 * Copyright (c) 2016, EPAM Systems
5 */
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/arm-smccc.h>
10#include <linux/cpuhotplug.h>
11#include <linux/errno.h>
12#include <linux/firmware.h>
13#include <linux/interrupt.h>
14#include <linux/io.h>
15#include <linux/irqdomain.h>
16#include <linux/kernel.h>
17#include <linux/mm.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/of_irq.h>
21#include <linux/of_platform.h>
22#include <linux/platform_device.h>
23#include <linux/sched.h>
24#include <linux/slab.h>
25#include <linux/string.h>
26#include <linux/tee_drv.h>
27#include <linux/types.h>
28#include <linux/workqueue.h>
29#include "optee_private.h"
30#include "optee_smc.h"
31#include "optee_rpc_cmd.h"
32#include <linux/kmemleak.h>
33#define CREATE_TRACE_POINTS
34#include "optee_trace.h"
35
36/*
37 * This file implement the SMC ABI used when communicating with secure world
38 * OP-TEE OS via raw SMCs.
39 * This file is divided into the following sections:
40 * 1. Convert between struct tee_param and struct optee_msg_param
41 * 2. Low level support functions to register shared memory in secure world
42 * 3. Dynamic shared memory pool based on alloc_pages()
43 * 4. Do a normal scheduled call into secure world
44 * 5. Asynchronous notification
45 * 6. Driver initialization.
46 */
47
48/*
49 * A typical OP-TEE private shm allocation is 224 bytes (argument struct
50 * with 6 parameters, needed for open session). So with an alignment of 512
51 * we'll waste a bit more than 50%. However, it's only expected that we'll
52 * have a handful of these structs allocated at a time. Most memory will
53 * be allocated aligned to the page size, So all in all this should scale
54 * up and down quite well.
55 */
56#define OPTEE_MIN_STATIC_POOL_ALIGN 9 /* 512 bytes aligned */
57
58/* SMC ABI considers at most a single TEE firmware */
59static unsigned int pcpu_irq_num;
60
61static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
62{
63 enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE);
64
65 return 0;
66}
67
68static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
69{
70 disable_percpu_irq(pcpu_irq_num);
71
72 return 0;
73}
74
75/*
76 * 1. Convert between struct tee_param and struct optee_msg_param
77 *
78 * optee_from_msg_param() and optee_to_msg_param() are the main
79 * functions.
80 */
81
82static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
83 const struct optee_msg_param *mp)
84{
85 struct tee_shm *shm;
86 phys_addr_t pa;
87 int rc;
88
89 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
90 attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
91 p->u.memref.size = mp->u.tmem.size;
92 shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
93 if (!shm) {
94 p->u.memref.shm_offs = 0;
95 p->u.memref.shm = NULL;
96 return 0;
97 }
98
99 rc = tee_shm_get_pa(shm, 0, &pa);
100 if (rc)
101 return rc;
102
103 p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
104 p->u.memref.shm = shm;
105
106 return 0;
107}
108
109static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
110 const struct optee_msg_param *mp)
111{
112 struct tee_shm *shm;
113
114 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
115 attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
116 p->u.memref.size = mp->u.rmem.size;
117 shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
118
119 if (shm) {
120 p->u.memref.shm_offs = mp->u.rmem.offs;
121 p->u.memref.shm = shm;
122 } else {
123 p->u.memref.shm_offs = 0;
124 p->u.memref.shm = NULL;
125 }
126}
127
128/**
129 * optee_from_msg_param() - convert from OPTEE_MSG parameters to
130 * struct tee_param
131 * @optee: main service struct
132 * @params: subsystem internal parameter representation
133 * @num_params: number of elements in the parameter arrays
134 * @msg_params: OPTEE_MSG parameters
135 * Returns 0 on success or <0 on failure
136 */
137static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
138 size_t num_params,
139 const struct optee_msg_param *msg_params)
140{
141 int rc;
142 size_t n;
143
144 for (n = 0; n < num_params; n++) {
145 struct tee_param *p = params + n;
146 const struct optee_msg_param *mp = msg_params + n;
147 u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
148
149 switch (attr) {
150 case OPTEE_MSG_ATTR_TYPE_NONE:
151 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
152 memset(&p->u, 0, sizeof(p->u));
153 break;
154 case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
155 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
156 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
157 optee_from_msg_param_value(p, attr, mp);
158 break;
159 case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
160 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
161 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
162 rc = from_msg_param_tmp_mem(p, attr, mp);
163 if (rc)
164 return rc;
165 break;
166 case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
167 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
168 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
169 from_msg_param_reg_mem(p, attr, mp);
170 break;
171
172 default:
173 return -EINVAL;
174 }
175 }
176 return 0;
177}
178
179static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
180 const struct tee_param *p)
181{
182 int rc;
183 phys_addr_t pa;
184
185 mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
186 TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
187
188 mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
189 mp->u.tmem.size = p->u.memref.size;
190
191 if (!p->u.memref.shm) {
192 mp->u.tmem.buf_ptr = 0;
193 return 0;
194 }
195
196 rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa);
197 if (rc)
198 return rc;
199
200 mp->u.tmem.buf_ptr = pa;
201 mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
202 OPTEE_MSG_ATTR_CACHE_SHIFT;
203
204 return 0;
205}
206
207static int to_msg_param_reg_mem(struct optee_msg_param *mp,
208 const struct tee_param *p)
209{
210 mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
211 TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
212
213 mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
214 mp->u.rmem.size = p->u.memref.size;
215 mp->u.rmem.offs = p->u.memref.shm_offs;
216 return 0;
217}
218
219/**
220 * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
221 * @optee: main service struct
222 * @msg_params: OPTEE_MSG parameters
223 * @num_params: number of elements in the parameter arrays
224 * @params: subsystem itnernal parameter representation
225 * Returns 0 on success or <0 on failure
226 */
227static int optee_to_msg_param(struct optee *optee,
228 struct optee_msg_param *msg_params,
229 size_t num_params, const struct tee_param *params)
230{
231 int rc;
232 size_t n;
233
234 for (n = 0; n < num_params; n++) {
235 const struct tee_param *p = params + n;
236 struct optee_msg_param *mp = msg_params + n;
237
238 switch (p->attr) {
239 case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
240 mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
241 memset(&mp->u, 0, sizeof(mp->u));
242 break;
243 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
244 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
245 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
246 optee_to_msg_param_value(mp, p);
247 break;
248 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
249 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
250 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
251 if (tee_shm_is_dynamic(p->u.memref.shm))
252 rc = to_msg_param_reg_mem(mp, p);
253 else
254 rc = to_msg_param_tmp_mem(mp, p);
255 if (rc)
256 return rc;
257 break;
258 default:
259 return -EINVAL;
260 }
261 }
262 return 0;
263}
264
265/*
266 * 2. Low level support functions to register shared memory in secure world
267 *
268 * Functions to enable/disable shared memory caching in secure world, that
269 * is, lazy freeing of previously allocated shared memory. Freeing is
270 * performed when a request has been compled.
271 *
272 * Functions to register and unregister shared memory both for normal
273 * clients and for tee-supplicant.
274 */
275
276/**
277 * optee_enable_shm_cache() - Enables caching of some shared memory allocation
278 * in OP-TEE
279 * @optee: main service struct
280 */
281static void optee_enable_shm_cache(struct optee *optee)
282{
283 struct optee_call_waiter w;
284
285 /* We need to retry until secure world isn't busy. */
286 optee_cq_wait_init(&optee->call_queue, &w, false);
287 while (true) {
288 struct arm_smccc_res res;
289
290 optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
291 0, 0, 0, 0, 0, 0, 0, &res);
292 if (res.a0 == OPTEE_SMC_RETURN_OK)
293 break;
294 optee_cq_wait_for_completion(&optee->call_queue, &w);
295 }
296 optee_cq_wait_final(&optee->call_queue, &w);
297}
298
299/**
300 * __optee_disable_shm_cache() - Disables caching of some shared memory
301 * allocation in OP-TEE
302 * @optee: main service struct
303 * @is_mapped: true if the cached shared memory addresses were mapped by this
304 * kernel, are safe to dereference, and should be freed
305 */
306static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
307{
308 struct optee_call_waiter w;
309
310 /* We need to retry until secure world isn't busy. */
311 optee_cq_wait_init(&optee->call_queue, &w, false);
312 while (true) {
313 union {
314 struct arm_smccc_res smccc;
315 struct optee_smc_disable_shm_cache_result result;
316 } res;
317
318 optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
319 0, 0, 0, 0, 0, 0, 0, &res.smccc);
320 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
321 break; /* All shm's freed */
322 if (res.result.status == OPTEE_SMC_RETURN_OK) {
323 struct tee_shm *shm;
324
325 /*
326 * Shared memory references that were not mapped by
327 * this kernel must be ignored to prevent a crash.
328 */
329 if (!is_mapped)
330 continue;
331
332 shm = reg_pair_to_ptr(res.result.shm_upper32,
333 res.result.shm_lower32);
334 tee_shm_free(shm);
335 } else {
336 optee_cq_wait_for_completion(&optee->call_queue, &w);
337 }
338 }
339 optee_cq_wait_final(&optee->call_queue, &w);
340}
341
342/**
343 * optee_disable_shm_cache() - Disables caching of mapped shared memory
344 * allocations in OP-TEE
345 * @optee: main service struct
346 */
347static void optee_disable_shm_cache(struct optee *optee)
348{
349 return __optee_disable_shm_cache(optee, true);
350}
351
352/**
353 * optee_disable_unmapped_shm_cache() - Disables caching of shared memory
354 * allocations in OP-TEE which are not
355 * currently mapped
356 * @optee: main service struct
357 */
358static void optee_disable_unmapped_shm_cache(struct optee *optee)
359{
360 return __optee_disable_shm_cache(optee, false);
361}
362
363#define PAGELIST_ENTRIES_PER_PAGE \
364 ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
365
366/*
367 * The final entry in each pagelist page is a pointer to the next
368 * pagelist page.
369 */
370static size_t get_pages_list_size(size_t num_entries)
371{
372 int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
373
374 return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
375}
376
377static u64 *optee_allocate_pages_list(size_t num_entries)
378{
379 return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
380}
381
382static void optee_free_pages_list(void *list, size_t num_entries)
383{
384 free_pages_exact(list, get_pages_list_size(num_entries));
385}
386
387/**
388 * optee_fill_pages_list() - write list of user pages to given shared
389 * buffer.
390 *
391 * @dst: page-aligned buffer where list of pages will be stored
392 * @pages: array of pages that represents shared buffer
393 * @num_pages: number of entries in @pages
394 * @page_offset: offset of user buffer from page start
395 *
396 * @dst should be big enough to hold list of user page addresses and
397 * links to the next pages of buffer
398 */
399static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
400 size_t page_offset)
401{
402 int n = 0;
403 phys_addr_t optee_page;
404 /*
405 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
406 * for details.
407 */
408 struct {
409 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
410 u64 next_page_data;
411 } *pages_data;
412
413 /*
414 * Currently OP-TEE uses 4k page size and it does not looks
415 * like this will change in the future. On other hand, there are
416 * no know ARM architectures with page size < 4k.
417 * Thus the next built assert looks redundant. But the following
418 * code heavily relies on this assumption, so it is better be
419 * safe than sorry.
420 */
421 BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
422
423 pages_data = (void *)dst;
424 /*
425 * If linux page is bigger than 4k, and user buffer offset is
426 * larger than 4k/8k/12k/etc this will skip first 4k pages,
427 * because they bear no value data for OP-TEE.
428 */
429 optee_page = page_to_phys(*pages) +
430 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
431
432 while (true) {
433 pages_data->pages_list[n++] = optee_page;
434
435 if (n == PAGELIST_ENTRIES_PER_PAGE) {
436 pages_data->next_page_data =
437 virt_to_phys(pages_data + 1);
438 pages_data++;
439 n = 0;
440 }
441
442 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
443 if (!(optee_page & ~PAGE_MASK)) {
444 if (!--num_pages)
445 break;
446 pages++;
447 optee_page = page_to_phys(*pages);
448 }
449 }
450}
451
452static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
453 struct page **pages, size_t num_pages,
454 unsigned long start)
455{
456 struct optee *optee = tee_get_drvdata(ctx->teedev);
457 struct optee_msg_arg *msg_arg;
458 struct tee_shm *shm_arg;
459 u64 *pages_list;
460 size_t sz;
461 int rc;
462
463 if (!num_pages)
464 return -EINVAL;
465
466 rc = optee_check_mem_type(start, num_pages);
467 if (rc)
468 return rc;
469
470 pages_list = optee_allocate_pages_list(num_pages);
471 if (!pages_list)
472 return -ENOMEM;
473
474 /*
475 * We're about to register shared memory we can't register shared
476 * memory for this request or there's a catch-22.
477 *
478 * So in this we'll have to do the good old temporary private
479 * allocation instead of using optee_get_msg_arg().
480 */
481 sz = optee_msg_arg_size(optee->rpc_param_count);
482 shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
483 if (IS_ERR(shm_arg)) {
484 rc = PTR_ERR(shm_arg);
485 goto out;
486 }
487 msg_arg = tee_shm_get_va(shm_arg, 0);
488 if (IS_ERR(msg_arg)) {
489 rc = PTR_ERR(msg_arg);
490 goto out;
491 }
492
493 optee_fill_pages_list(pages_list, pages, num_pages,
494 tee_shm_get_page_offset(shm));
495
496 memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
497 msg_arg->num_params = 1;
498 msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
499 msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
500 OPTEE_MSG_ATTR_NONCONTIG;
501 msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
502 msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
503 /*
504 * In the least bits of msg_arg->params->u.tmem.buf_ptr we
505 * store buffer offset from 4k page, as described in OP-TEE ABI.
506 */
507 msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
508 (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
509
510 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
511 msg_arg->ret != TEEC_SUCCESS)
512 rc = -EINVAL;
513
514 tee_shm_free(shm_arg);
515out:
516 optee_free_pages_list(pages_list, num_pages);
517 return rc;
518}
519
520static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
521{
522 struct optee *optee = tee_get_drvdata(ctx->teedev);
523 struct optee_msg_arg *msg_arg;
524 struct tee_shm *shm_arg;
525 int rc = 0;
526 size_t sz;
527
528 /*
529 * We're about to unregister shared memory and we may not be able
530 * register shared memory for this request in case we're called
531 * from optee_shm_arg_cache_uninit().
532 *
533 * So in order to keep things simple in this function just as in
534 * optee_shm_register() we'll use temporary private allocation
535 * instead of using optee_get_msg_arg().
536 */
537 sz = optee_msg_arg_size(optee->rpc_param_count);
538 shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
539 if (IS_ERR(shm_arg))
540 return PTR_ERR(shm_arg);
541 msg_arg = tee_shm_get_va(shm_arg, 0);
542 if (IS_ERR(msg_arg)) {
543 rc = PTR_ERR(msg_arg);
544 goto out;
545 }
546
547 memset(msg_arg, 0, sz);
548 msg_arg->num_params = 1;
549 msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
550 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
551 msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
552
553 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
554 msg_arg->ret != TEEC_SUCCESS)
555 rc = -EINVAL;
556out:
557 tee_shm_free(shm_arg);
558 return rc;
559}
560
561static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
562 struct page **pages, size_t num_pages,
563 unsigned long start)
564{
565 /*
566 * We don't want to register supplicant memory in OP-TEE.
567 * Instead information about it will be passed in RPC code.
568 */
569 return optee_check_mem_type(start, num_pages);
570}
571
572static int optee_shm_unregister_supp(struct tee_context *ctx,
573 struct tee_shm *shm)
574{
575 return 0;
576}
577
578/*
579 * 3. Dynamic shared memory pool based on alloc_pages()
580 *
581 * Implements an OP-TEE specific shared memory pool which is used
582 * when dynamic shared memory is supported by secure world.
583 *
584 * The main function is optee_shm_pool_alloc_pages().
585 */
586
587static int pool_op_alloc(struct tee_shm_pool *pool,
588 struct tee_shm *shm, size_t size, size_t align)
589{
590 /*
591 * Shared memory private to the OP-TEE driver doesn't need
592 * to be registered with OP-TEE.
593 */
594 if (shm->flags & TEE_SHM_PRIV)
595 return optee_pool_op_alloc_helper(pool, shm, size, align, NULL);
596
597 return optee_pool_op_alloc_helper(pool, shm, size, align,
598 optee_shm_register);
599}
600
601static void pool_op_free(struct tee_shm_pool *pool,
602 struct tee_shm *shm)
603{
604 if (!(shm->flags & TEE_SHM_PRIV))
605 optee_pool_op_free_helper(pool, shm, optee_shm_unregister);
606 else
607 optee_pool_op_free_helper(pool, shm, NULL);
608}
609
610static void pool_op_destroy_pool(struct tee_shm_pool *pool)
611{
612 kfree(pool);
613}
614
615static const struct tee_shm_pool_ops pool_ops = {
616 .alloc = pool_op_alloc,
617 .free = pool_op_free,
618 .destroy_pool = pool_op_destroy_pool,
619};
620
621/**
622 * optee_shm_pool_alloc_pages() - create page-based allocator pool
623 *
624 * This pool is used when OP-TEE supports dymanic SHM. In this case
625 * command buffers and such are allocated from kernel's own memory.
626 */
627static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
628{
629 struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
630
631 if (!pool)
632 return ERR_PTR(-ENOMEM);
633
634 pool->ops = &pool_ops;
635
636 return pool;
637}
638
639/*
640 * 4. Do a normal scheduled call into secure world
641 *
642 * The function optee_smc_do_call_with_arg() performs a normal scheduled
643 * call into secure world. During this call may normal world request help
644 * from normal world using RPCs, Remote Procedure Calls. This includes
645 * delivery of non-secure interrupts to for instance allow rescheduling of
646 * the current task.
647 */
648
649static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
650 struct optee_msg_arg *arg)
651{
652 struct tee_shm *shm;
653
654 arg->ret_origin = TEEC_ORIGIN_COMMS;
655
656 if (arg->num_params != 1 ||
657 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
658 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
659 return;
660 }
661
662 shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
663 switch (arg->params[0].u.value.a) {
664 case OPTEE_RPC_SHM_TYPE_APPL:
665 optee_rpc_cmd_free_suppl(ctx, shm);
666 break;
667 case OPTEE_RPC_SHM_TYPE_KERNEL:
668 tee_shm_free(shm);
669 break;
670 default:
671 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
672 }
673 arg->ret = TEEC_SUCCESS;
674}
675
676static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
677 struct optee *optee,
678 struct optee_msg_arg *arg,
679 struct optee_call_ctx *call_ctx)
680{
681 struct tee_shm *shm;
682 size_t sz;
683 size_t n;
684 struct page **pages;
685 size_t page_count;
686
687 arg->ret_origin = TEEC_ORIGIN_COMMS;
688
689 if (!arg->num_params ||
690 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
691 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
692 return;
693 }
694
695 for (n = 1; n < arg->num_params; n++) {
696 if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
697 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
698 return;
699 }
700 }
701
702 sz = arg->params[0].u.value.b;
703 switch (arg->params[0].u.value.a) {
704 case OPTEE_RPC_SHM_TYPE_APPL:
705 shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
706 break;
707 case OPTEE_RPC_SHM_TYPE_KERNEL:
708 shm = tee_shm_alloc_priv_buf(optee->ctx, sz);
709 break;
710 default:
711 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
712 return;
713 }
714
715 if (IS_ERR(shm)) {
716 arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
717 return;
718 }
719
720 /*
721 * If there are pages it's dynamically allocated shared memory (not
722 * from the reserved shared memory pool) and needs to be
723 * registered.
724 */
725 pages = tee_shm_get_pages(shm, &page_count);
726 if (pages) {
727 u64 *pages_list;
728
729 pages_list = optee_allocate_pages_list(page_count);
730 if (!pages_list) {
731 arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
732 goto bad;
733 }
734
735 call_ctx->pages_list = pages_list;
736 call_ctx->num_entries = page_count;
737
738 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
739 OPTEE_MSG_ATTR_NONCONTIG;
740 /*
741 * In the least bits of u.tmem.buf_ptr we store buffer offset
742 * from 4k page, as described in OP-TEE ABI.
743 */
744 arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) |
745 (tee_shm_get_page_offset(shm) &
746 (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
747
748 optee_fill_pages_list(pages_list, pages, page_count,
749 tee_shm_get_page_offset(shm));
750 } else {
751 phys_addr_t pa;
752
753 if (tee_shm_get_pa(shm, 0, &pa)) {
754 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
755 goto bad;
756 }
757
758 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
759 arg->params[0].u.tmem.buf_ptr = pa;
760 }
761 arg->params[0].u.tmem.size = tee_shm_get_size(shm);
762 arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
763
764 arg->ret = TEEC_SUCCESS;
765 return;
766bad:
767 tee_shm_free(shm);
768}
769
770static void free_pages_list(struct optee_call_ctx *call_ctx)
771{
772 if (call_ctx->pages_list) {
773 optee_free_pages_list(call_ctx->pages_list,
774 call_ctx->num_entries);
775 call_ctx->pages_list = NULL;
776 call_ctx->num_entries = 0;
777 }
778}
779
780static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
781{
782 free_pages_list(call_ctx);
783}
784
785static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
786 struct optee_msg_arg *arg,
787 struct optee_call_ctx *call_ctx)
788{
789
790 switch (arg->cmd) {
791 case OPTEE_RPC_CMD_SHM_ALLOC:
792 free_pages_list(call_ctx);
793 handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
794 break;
795 case OPTEE_RPC_CMD_SHM_FREE:
796 handle_rpc_func_cmd_shm_free(ctx, arg);
797 break;
798 default:
799 optee_rpc_cmd(ctx, optee, arg);
800 }
801}
802
803/**
804 * optee_handle_rpc() - handle RPC from secure world
805 * @ctx: context doing the RPC
806 * @rpc_arg: pointer to RPC arguments if any, or NULL if none
807 * @param: value of registers for the RPC
808 * @call_ctx: call context. Preserved during one OP-TEE invocation
809 *
810 * Result of RPC is written back into @param.
811 */
812static void optee_handle_rpc(struct tee_context *ctx,
813 struct optee_msg_arg *rpc_arg,
814 struct optee_rpc_param *param,
815 struct optee_call_ctx *call_ctx)
816{
817 struct tee_device *teedev = ctx->teedev;
818 struct optee *optee = tee_get_drvdata(teedev);
819 struct optee_msg_arg *arg;
820 struct tee_shm *shm;
821 phys_addr_t pa;
822
823 switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
824 case OPTEE_SMC_RPC_FUNC_ALLOC:
825 shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1);
826 if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {
827 reg_pair_from_64(¶m->a1, ¶m->a2, pa);
828 reg_pair_from_64(¶m->a4, ¶m->a5,
829 (unsigned long)shm);
830 } else {
831 param->a1 = 0;
832 param->a2 = 0;
833 param->a4 = 0;
834 param->a5 = 0;
835 }
836 kmemleak_not_leak(shm);
837 break;
838 case OPTEE_SMC_RPC_FUNC_FREE:
839 shm = reg_pair_to_ptr(param->a1, param->a2);
840 tee_shm_free(shm);
841 break;
842 case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
843 /*
844 * A foreign interrupt was raised while secure world was
845 * executing, since they are handled in Linux a dummy RPC is
846 * performed to let Linux take the interrupt through the normal
847 * vector.
848 */
849 break;
850 case OPTEE_SMC_RPC_FUNC_CMD:
851 if (rpc_arg) {
852 arg = rpc_arg;
853 } else {
854 shm = reg_pair_to_ptr(param->a1, param->a2);
855 arg = tee_shm_get_va(shm, 0);
856 if (IS_ERR(arg)) {
857 pr_err("%s: tee_shm_get_va %p failed\n",
858 __func__, shm);
859 break;
860 }
861 }
862
863 handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
864 break;
865 default:
866 pr_warn("Unknown RPC func 0x%x\n",
867 (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
868 break;
869 }
870
871 param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
872}
873
874/**
875 * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
876 * @ctx: calling context
877 * @shm: shared memory holding the message to pass to secure world
878 * @offs: offset of the message in @shm
879 * @system_thread: true if caller requests TEE system thread support
880 *
881 * Does and SMC to OP-TEE in secure world and handles eventual resulting
882 * Remote Procedure Calls (RPC) from OP-TEE.
883 *
884 * Returns return code from secure world, 0 is OK
885 */
886static int optee_smc_do_call_with_arg(struct tee_context *ctx,
887 struct tee_shm *shm, u_int offs,
888 bool system_thread)
889{
890 struct optee *optee = tee_get_drvdata(ctx->teedev);
891 struct optee_call_waiter w;
892 struct optee_rpc_param param = { };
893 struct optee_call_ctx call_ctx = { };
894 struct optee_msg_arg *rpc_arg = NULL;
895 int rc;
896
897 if (optee->rpc_param_count) {
898 struct optee_msg_arg *arg;
899 unsigned int rpc_arg_offs;
900
901 arg = tee_shm_get_va(shm, offs);
902 if (IS_ERR(arg))
903 return PTR_ERR(arg);
904
905 rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
906 rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);
907 if (IS_ERR(rpc_arg))
908 return PTR_ERR(rpc_arg);
909 }
910
911 if (rpc_arg && tee_shm_is_dynamic(shm)) {
912 param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
913 reg_pair_from_64(¶m.a1, ¶m.a2, (u_long)shm);
914 param.a3 = offs;
915 } else {
916 phys_addr_t parg;
917
918 rc = tee_shm_get_pa(shm, offs, &parg);
919 if (rc)
920 return rc;
921
922 if (rpc_arg)
923 param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
924 else
925 param.a0 = OPTEE_SMC_CALL_WITH_ARG;
926 reg_pair_from_64(¶m.a1, ¶m.a2, parg);
927 }
928 /* Initialize waiter */
929 optee_cq_wait_init(&optee->call_queue, &w, system_thread);
930 while (true) {
931 struct arm_smccc_res res;
932
933 trace_optee_invoke_fn_begin(¶m);
934 optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
935 param.a4, param.a5, param.a6, param.a7,
936 &res);
937 trace_optee_invoke_fn_end(¶m, &res);
938
939 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
940 /*
941 * Out of threads in secure world, wait for a thread
942 * become available.
943 */
944 optee_cq_wait_for_completion(&optee->call_queue, &w);
945 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
946 cond_resched();
947 param.a0 = res.a0;
948 param.a1 = res.a1;
949 param.a2 = res.a2;
950 param.a3 = res.a3;
951 optee_handle_rpc(ctx, rpc_arg, ¶m, &call_ctx);
952 } else {
953 rc = res.a0;
954 break;
955 }
956 }
957
958 optee_rpc_finalize_call(&call_ctx);
959 /*
960 * We're done with our thread in secure world, if there's any
961 * thread waiters wake up one.
962 */
963 optee_cq_wait_final(&optee->call_queue, &w);
964
965 return rc;
966}
967
968/*
969 * 5. Asynchronous notification
970 */
971
972static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
973 bool *value_pending)
974{
975 struct arm_smccc_res res;
976
977 invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
978
979 if (res.a0) {
980 *value_valid = false;
981 return 0;
982 }
983 *value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
984 *value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
985 return res.a1;
986}
987
988static irqreturn_t irq_handler(struct optee *optee)
989{
990 bool do_bottom_half = false;
991 bool value_valid;
992 bool value_pending;
993 u32 value;
994
995 do {
996 value = get_async_notif_value(optee->smc.invoke_fn,
997 &value_valid, &value_pending);
998 if (!value_valid)
999 break;
1000
1001 if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
1002 do_bottom_half = true;
1003 else
1004 optee_notif_send(optee, value);
1005 } while (value_pending);
1006
1007 if (do_bottom_half)
1008 return IRQ_WAKE_THREAD;
1009 return IRQ_HANDLED;
1010}
1011
1012static irqreturn_t notif_irq_handler(int irq, void *dev_id)
1013{
1014 struct optee *optee = dev_id;
1015
1016 return irq_handler(optee);
1017}
1018
1019static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
1020{
1021 struct optee *optee = dev_id;
1022
1023 optee_do_bottom_half(optee->ctx);
1024
1025 return IRQ_HANDLED;
1026}
1027
1028static int init_irq(struct optee *optee, u_int irq)
1029{
1030 int rc;
1031
1032 rc = request_threaded_irq(irq, notif_irq_handler,
1033 notif_irq_thread_fn,
1034 0, "optee_notification", optee);
1035 if (rc)
1036 return rc;
1037
1038 optee->smc.notif_irq = irq;
1039
1040 return 0;
1041}
1042
1043static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
1044{
1045 struct optee_pcpu *pcpu = dev_id;
1046 struct optee *optee = pcpu->optee;
1047
1048 if (irq_handler(optee) == IRQ_WAKE_THREAD)
1049 queue_work(optee->smc.notif_pcpu_wq,
1050 &optee->smc.notif_pcpu_work);
1051
1052 return IRQ_HANDLED;
1053}
1054
1055static void notif_pcpu_irq_work_fn(struct work_struct *work)
1056{
1057 struct optee_smc *optee_smc = container_of(work, struct optee_smc,
1058 notif_pcpu_work);
1059 struct optee *optee = container_of(optee_smc, struct optee, smc);
1060
1061 optee_do_bottom_half(optee->ctx);
1062}
1063
1064static int init_pcpu_irq(struct optee *optee, u_int irq)
1065{
1066 struct optee_pcpu __percpu *optee_pcpu;
1067 int cpu, rc;
1068
1069 optee_pcpu = alloc_percpu(struct optee_pcpu);
1070 if (!optee_pcpu)
1071 return -ENOMEM;
1072
1073 for_each_present_cpu(cpu)
1074 per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
1075
1076 rc = request_percpu_irq(irq, notif_pcpu_irq_handler,
1077 "optee_pcpu_notification", optee_pcpu);
1078 if (rc)
1079 goto err_free_pcpu;
1080
1081 INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
1082 optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
1083 if (!optee->smc.notif_pcpu_wq) {
1084 rc = -EINVAL;
1085 goto err_free_pcpu_irq;
1086 }
1087
1088 optee->smc.optee_pcpu = optee_pcpu;
1089 optee->smc.notif_irq = irq;
1090
1091 pcpu_irq_num = irq;
1092 rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting",
1093 optee_cpuhp_enable_pcpu_irq,
1094 optee_cpuhp_disable_pcpu_irq);
1095 if (!rc)
1096 rc = -EINVAL;
1097 if (rc < 0)
1098 goto err_free_pcpu_irq;
1099
1100 optee->smc.notif_cpuhp_state = rc;
1101
1102 return 0;
1103
1104err_free_pcpu_irq:
1105 free_percpu_irq(irq, optee_pcpu);
1106err_free_pcpu:
1107 free_percpu(optee_pcpu);
1108
1109 return rc;
1110}
1111
1112static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
1113{
1114 if (irq_is_percpu_devid(irq))
1115 return init_pcpu_irq(optee, irq);
1116 else
1117 return init_irq(optee, irq);
1118}
1119
1120static void uninit_pcpu_irq(struct optee *optee)
1121{
1122 cpuhp_remove_state(optee->smc.notif_cpuhp_state);
1123
1124 destroy_workqueue(optee->smc.notif_pcpu_wq);
1125
1126 free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
1127 free_percpu(optee->smc.optee_pcpu);
1128}
1129
1130static void optee_smc_notif_uninit_irq(struct optee *optee)
1131{
1132 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1133 optee_stop_async_notif(optee->ctx);
1134 if (optee->smc.notif_irq) {
1135 if (irq_is_percpu_devid(optee->smc.notif_irq))
1136 uninit_pcpu_irq(optee);
1137 else
1138 free_irq(optee->smc.notif_irq, optee);
1139
1140 irq_dispose_mapping(optee->smc.notif_irq);
1141 }
1142 }
1143}
1144
1145/*
1146 * 6. Driver initialization
1147 *
1148 * During driver initialization is secure world probed to find out which
1149 * features it supports so the driver can be initialized with a matching
1150 * configuration. This involves for instance support for dynamic shared
1151 * memory instead of a static memory carvout.
1152 */
1153
1154static void optee_get_version(struct tee_device *teedev,
1155 struct tee_ioctl_version_data *vers)
1156{
1157 struct tee_ioctl_version_data v = {
1158 .impl_id = TEE_IMPL_ID_OPTEE,
1159 .impl_caps = TEE_OPTEE_CAP_TZ,
1160 .gen_caps = TEE_GEN_CAP_GP,
1161 };
1162 struct optee *optee = tee_get_drvdata(teedev);
1163
1164 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1165 v.gen_caps |= TEE_GEN_CAP_REG_MEM;
1166 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
1167 v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
1168 *vers = v;
1169}
1170
1171static int optee_smc_open(struct tee_context *ctx)
1172{
1173 struct optee *optee = tee_get_drvdata(ctx->teedev);
1174 u32 sec_caps = optee->smc.sec_caps;
1175
1176 return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
1177}
1178
1179static const struct tee_driver_ops optee_clnt_ops = {
1180 .get_version = optee_get_version,
1181 .open = optee_smc_open,
1182 .release = optee_release,
1183 .open_session = optee_open_session,
1184 .close_session = optee_close_session,
1185 .system_session = optee_system_session,
1186 .invoke_func = optee_invoke_func,
1187 .cancel_req = optee_cancel_req,
1188 .shm_register = optee_shm_register,
1189 .shm_unregister = optee_shm_unregister,
1190};
1191
1192static const struct tee_desc optee_clnt_desc = {
1193 .name = DRIVER_NAME "-clnt",
1194 .ops = &optee_clnt_ops,
1195 .owner = THIS_MODULE,
1196};
1197
1198static const struct tee_driver_ops optee_supp_ops = {
1199 .get_version = optee_get_version,
1200 .open = optee_smc_open,
1201 .release = optee_release_supp,
1202 .supp_recv = optee_supp_recv,
1203 .supp_send = optee_supp_send,
1204 .shm_register = optee_shm_register_supp,
1205 .shm_unregister = optee_shm_unregister_supp,
1206};
1207
1208static const struct tee_desc optee_supp_desc = {
1209 .name = DRIVER_NAME "-supp",
1210 .ops = &optee_supp_ops,
1211 .owner = THIS_MODULE,
1212 .flags = TEE_DESC_PRIVILEGED,
1213};
1214
1215static const struct optee_ops optee_ops = {
1216 .do_call_with_arg = optee_smc_do_call_with_arg,
1217 .to_msg_param = optee_to_msg_param,
1218 .from_msg_param = optee_from_msg_param,
1219};
1220
1221static int enable_async_notif(optee_invoke_fn *invoke_fn)
1222{
1223 struct arm_smccc_res res;
1224
1225 invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
1226
1227 if (res.a0)
1228 return -EINVAL;
1229 return 0;
1230}
1231
1232static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
1233{
1234 struct arm_smccc_res res;
1235
1236 invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1237
1238 if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
1239 res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
1240 return true;
1241 return false;
1242}
1243
1244#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1245static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
1246{
1247 struct arm_smccc_res res;
1248
1249 invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1250
1251 if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
1252 res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
1253 res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
1254 res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
1255 return true;
1256 return false;
1257}
1258#endif
1259
1260static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
1261{
1262 union {
1263 struct arm_smccc_res smccc;
1264 struct optee_smc_call_get_os_revision_result result;
1265 } res = {
1266 .result = {
1267 .build_id = 0
1268 }
1269 };
1270
1271 invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
1272 &res.smccc);
1273
1274 if (res.result.build_id)
1275 pr_info("revision %lu.%lu (%08lx)", res.result.major,
1276 res.result.minor, res.result.build_id);
1277 else
1278 pr_info("revision %lu.%lu", res.result.major, res.result.minor);
1279}
1280
1281static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
1282{
1283 union {
1284 struct arm_smccc_res smccc;
1285 struct optee_smc_calls_revision_result result;
1286 } res;
1287
1288 invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1289
1290 if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
1291 (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
1292 return true;
1293 return false;
1294}
1295
1296static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
1297 u32 *sec_caps, u32 *max_notif_value,
1298 unsigned int *rpc_param_count)
1299{
1300 union {
1301 struct arm_smccc_res smccc;
1302 struct optee_smc_exchange_capabilities_result result;
1303 } res;
1304 u32 a1 = 0;
1305
1306 /*
1307 * TODO This isn't enough to tell if it's UP system (from kernel
1308 * point of view) or not, is_smp() returns the information
1309 * needed, but can't be called directly from here.
1310 */
1311 if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
1312 a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
1313
1314 invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
1315 &res.smccc);
1316
1317 if (res.result.status != OPTEE_SMC_RETURN_OK)
1318 return false;
1319
1320 *sec_caps = res.result.capabilities;
1321 if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
1322 *max_notif_value = res.result.max_notif_value;
1323 else
1324 *max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
1325 if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1326 *rpc_param_count = (u8)res.result.data;
1327 else
1328 *rpc_param_count = 0;
1329
1330 return true;
1331}
1332
1333static unsigned int optee_msg_get_thread_count(optee_invoke_fn *invoke_fn)
1334{
1335 struct arm_smccc_res res;
1336
1337 invoke_fn(OPTEE_SMC_GET_THREAD_COUNT, 0, 0, 0, 0, 0, 0, 0, &res);
1338 if (res.a0)
1339 return 0;
1340 return res.a1;
1341}
1342
1343static struct tee_shm_pool *
1344optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
1345{
1346 union {
1347 struct arm_smccc_res smccc;
1348 struct optee_smc_get_shm_config_result result;
1349 } res;
1350 unsigned long vaddr;
1351 phys_addr_t paddr;
1352 size_t size;
1353 phys_addr_t begin;
1354 phys_addr_t end;
1355 void *va;
1356 void *rc;
1357
1358 invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1359 if (res.result.status != OPTEE_SMC_RETURN_OK) {
1360 pr_err("static shm service not available\n");
1361 return ERR_PTR(-ENOENT);
1362 }
1363
1364 if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
1365 pr_err("only normal cached shared memory supported\n");
1366 return ERR_PTR(-EINVAL);
1367 }
1368
1369 begin = roundup(res.result.start, PAGE_SIZE);
1370 end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
1371 paddr = begin;
1372 size = end - begin;
1373
1374 va = memremap(paddr, size, MEMREMAP_WB);
1375 if (!va) {
1376 pr_err("shared memory ioremap failed\n");
1377 return ERR_PTR(-EINVAL);
1378 }
1379 vaddr = (unsigned long)va;
1380
1381 rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
1382 OPTEE_MIN_STATIC_POOL_ALIGN);
1383 if (IS_ERR(rc))
1384 memunmap(va);
1385 else
1386 *memremaped_shm = va;
1387
1388 return rc;
1389}
1390
1391/* Simple wrapper functions to be able to use a function pointer */
1392static void optee_smccc_smc(unsigned long a0, unsigned long a1,
1393 unsigned long a2, unsigned long a3,
1394 unsigned long a4, unsigned long a5,
1395 unsigned long a6, unsigned long a7,
1396 struct arm_smccc_res *res)
1397{
1398 arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1399}
1400
1401static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
1402 unsigned long a2, unsigned long a3,
1403 unsigned long a4, unsigned long a5,
1404 unsigned long a6, unsigned long a7,
1405 struct arm_smccc_res *res)
1406{
1407 arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1408}
1409
1410static optee_invoke_fn *get_invoke_func(struct device *dev)
1411{
1412 const char *method;
1413
1414 pr_info("probing for conduit method.\n");
1415
1416 if (device_property_read_string(dev, "method", &method)) {
1417 pr_warn("missing \"method\" property\n");
1418 return ERR_PTR(-ENXIO);
1419 }
1420
1421 if (!strcmp("hvc", method))
1422 return optee_smccc_hvc;
1423 else if (!strcmp("smc", method))
1424 return optee_smccc_smc;
1425
1426 pr_warn("invalid \"method\" property: %s\n", method);
1427 return ERR_PTR(-EINVAL);
1428}
1429
1430/* optee_remove - Device Removal Routine
1431 * @pdev: platform device information struct
1432 *
1433 * optee_remove is called by platform subsystem to alert the driver
1434 * that it should release the device
1435 */
1436static int optee_smc_remove(struct platform_device *pdev)
1437{
1438 struct optee *optee = platform_get_drvdata(pdev);
1439
1440 /*
1441 * Ask OP-TEE to free all cached shared memory objects to decrease
1442 * reference counters and also avoid wild pointers in secure world
1443 * into the old shared memory range.
1444 */
1445 if (!optee->rpc_param_count)
1446 optee_disable_shm_cache(optee);
1447
1448 optee_smc_notif_uninit_irq(optee);
1449
1450 optee_remove_common(optee);
1451
1452 if (optee->smc.memremaped_shm)
1453 memunmap(optee->smc.memremaped_shm);
1454
1455 kfree(optee);
1456
1457 return 0;
1458}
1459
1460/* optee_shutdown - Device Removal Routine
1461 * @pdev: platform device information struct
1462 *
1463 * platform_shutdown is called by the platform subsystem to alert
1464 * the driver that a shutdown, reboot, or kexec is happening and
1465 * device must be disabled.
1466 */
1467static void optee_shutdown(struct platform_device *pdev)
1468{
1469 struct optee *optee = platform_get_drvdata(pdev);
1470
1471 if (!optee->rpc_param_count)
1472 optee_disable_shm_cache(optee);
1473}
1474
1475#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1476
1477#define OPTEE_FW_IMAGE "optee/tee.bin"
1478
1479static optee_invoke_fn *cpuhp_invoke_fn;
1480
1481static int optee_cpuhp_probe(unsigned int cpu)
1482{
1483 /*
1484 * Invoking a call on a CPU will cause OP-TEE to perform the required
1485 * setup for that CPU. Just invoke the call to get the UID since that
1486 * has no side effects.
1487 */
1488 if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
1489 return 0;
1490 else
1491 return -EINVAL;
1492}
1493
1494static int optee_load_fw(struct platform_device *pdev,
1495 optee_invoke_fn *invoke_fn)
1496{
1497 const struct firmware *fw = NULL;
1498 struct arm_smccc_res res;
1499 phys_addr_t data_pa;
1500 u8 *data_buf = NULL;
1501 u64 data_size;
1502 u32 data_pa_high, data_pa_low;
1503 u32 data_size_high, data_size_low;
1504 int rc;
1505 int hp_state;
1506
1507 if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
1508 return 0;
1509
1510 rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
1511 if (rc) {
1512 /*
1513 * The firmware in the rootfs will not be accessible until we
1514 * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
1515 * that point.
1516 */
1517 if (system_state < SYSTEM_RUNNING)
1518 return -EPROBE_DEFER;
1519 goto fw_err;
1520 }
1521
1522 data_size = fw->size;
1523 /*
1524 * This uses the GFP_DMA flag to ensure we are allocated memory in the
1525 * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
1526 */
1527 data_buf = kmemdup(fw->data, fw->size, GFP_KERNEL | GFP_DMA);
1528 if (!data_buf) {
1529 rc = -ENOMEM;
1530 goto fw_err;
1531 }
1532 data_pa = virt_to_phys(data_buf);
1533 reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
1534 reg_pair_from_64(&data_size_high, &data_size_low, data_size);
1535 goto fw_load;
1536
1537fw_err:
1538 pr_warn("image loading failed\n");
1539 data_pa_high = 0;
1540 data_pa_low = 0;
1541 data_size_high = 0;
1542 data_size_low = 0;
1543
1544fw_load:
1545 /*
1546 * Always invoke the SMC, even if loading the image fails, to indicate
1547 * to EL3 that we have passed the point where it should allow invoking
1548 * this SMC.
1549 */
1550 pr_warn("OP-TEE image loaded from kernel, this can be insecure");
1551 invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
1552 data_pa_high, data_pa_low, 0, 0, 0, &res);
1553 if (!rc)
1554 rc = res.a0;
1555 if (fw)
1556 release_firmware(fw);
1557 kfree(data_buf);
1558
1559 if (!rc) {
1560 /*
1561 * We need to initialize OP-TEE on all other running cores as
1562 * well. Any cores that aren't running yet will get initialized
1563 * when they are brought up by the power management functions in
1564 * TF-A which are registered by the OP-TEE SPD. Due to that we
1565 * can un-register the callback right after registering it.
1566 */
1567 cpuhp_invoke_fn = invoke_fn;
1568 hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
1569 optee_cpuhp_probe, NULL);
1570 if (hp_state < 0) {
1571 pr_warn("Failed with CPU hotplug setup for OP-TEE");
1572 return -EINVAL;
1573 }
1574 cpuhp_remove_state(hp_state);
1575 cpuhp_invoke_fn = NULL;
1576 }
1577
1578 return rc;
1579}
1580#else
1581static inline int optee_load_fw(struct platform_device *pdev,
1582 optee_invoke_fn *invoke_fn)
1583{
1584 return 0;
1585}
1586#endif
1587
1588static int optee_probe(struct platform_device *pdev)
1589{
1590 optee_invoke_fn *invoke_fn;
1591 struct tee_shm_pool *pool = ERR_PTR(-EINVAL);
1592 struct optee *optee = NULL;
1593 void *memremaped_shm = NULL;
1594 unsigned int rpc_param_count;
1595 unsigned int thread_count;
1596 struct tee_device *teedev;
1597 struct tee_context *ctx;
1598 u32 max_notif_value;
1599 u32 arg_cache_flags;
1600 u32 sec_caps;
1601 int rc;
1602
1603 invoke_fn = get_invoke_func(&pdev->dev);
1604 if (IS_ERR(invoke_fn))
1605 return PTR_ERR(invoke_fn);
1606
1607 rc = optee_load_fw(pdev, invoke_fn);
1608 if (rc)
1609 return rc;
1610
1611 if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
1612 pr_warn("api uid mismatch\n");
1613 return -EINVAL;
1614 }
1615
1616 optee_msg_get_os_revision(invoke_fn);
1617
1618 if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
1619 pr_warn("api revision mismatch\n");
1620 return -EINVAL;
1621 }
1622
1623 thread_count = optee_msg_get_thread_count(invoke_fn);
1624 if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,
1625 &max_notif_value,
1626 &rpc_param_count)) {
1627 pr_warn("capabilities mismatch\n");
1628 return -EINVAL;
1629 }
1630
1631 /*
1632 * Try to use dynamic shared memory if possible
1633 */
1634 if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
1635 /*
1636 * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
1637 * optee_get_msg_arg() to pre-register (by having
1638 * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
1639 * an argument struct.
1640 *
1641 * With the page is pre-registered we can use a non-zero
1642 * offset for argument struct, this is indicated with
1643 * OPTEE_SHM_ARG_SHARED.
1644 *
1645 * This means that optee_smc_do_call_with_arg() will use
1646 * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
1647 */
1648 if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1649 arg_cache_flags = OPTEE_SHM_ARG_SHARED;
1650 else
1651 arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
1652
1653 pool = optee_shm_pool_alloc_pages();
1654 }
1655
1656 /*
1657 * If dynamic shared memory is not available or failed - try static one
1658 */
1659 if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
1660 /*
1661 * The static memory pool can use non-zero page offsets so
1662 * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
1663 *
1664 * optee_get_msg_arg() should not pre-register the
1665 * allocated page used to pass an argument struct, this is
1666 * indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
1667 *
1668 * This means that optee_smc_do_call_with_arg() will use
1669 * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
1670 * OPTEE_SMC_CALL_WITH_RPC_ARG.
1671 */
1672 arg_cache_flags = OPTEE_SHM_ARG_SHARED |
1673 OPTEE_SHM_ARG_ALLOC_PRIV;
1674 pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm);
1675 }
1676
1677 if (IS_ERR(pool))
1678 return PTR_ERR(pool);
1679
1680 optee = kzalloc(sizeof(*optee), GFP_KERNEL);
1681 if (!optee) {
1682 rc = -ENOMEM;
1683 goto err_free_pool;
1684 }
1685
1686 optee->ops = &optee_ops;
1687 optee->smc.invoke_fn = invoke_fn;
1688 optee->smc.sec_caps = sec_caps;
1689 optee->rpc_param_count = rpc_param_count;
1690
1691 teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee);
1692 if (IS_ERR(teedev)) {
1693 rc = PTR_ERR(teedev);
1694 goto err_free_optee;
1695 }
1696 optee->teedev = teedev;
1697
1698 teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee);
1699 if (IS_ERR(teedev)) {
1700 rc = PTR_ERR(teedev);
1701 goto err_unreg_teedev;
1702 }
1703 optee->supp_teedev = teedev;
1704
1705 rc = tee_device_register(optee->teedev);
1706 if (rc)
1707 goto err_unreg_supp_teedev;
1708
1709 rc = tee_device_register(optee->supp_teedev);
1710 if (rc)
1711 goto err_unreg_supp_teedev;
1712
1713 optee_cq_init(&optee->call_queue, thread_count);
1714 optee_supp_init(&optee->supp);
1715 optee->smc.memremaped_shm = memremaped_shm;
1716 optee->pool = pool;
1717 optee_shm_arg_cache_init(optee, arg_cache_flags);
1718
1719 platform_set_drvdata(pdev, optee);
1720 ctx = teedev_open(optee->teedev);
1721 if (IS_ERR(ctx)) {
1722 rc = PTR_ERR(ctx);
1723 goto err_supp_uninit;
1724 }
1725 optee->ctx = ctx;
1726 rc = optee_notif_init(optee, max_notif_value);
1727 if (rc)
1728 goto err_close_ctx;
1729
1730 if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1731 unsigned int irq;
1732
1733 rc = platform_get_irq(pdev, 0);
1734 if (rc < 0) {
1735 pr_err("platform_get_irq: ret %d\n", rc);
1736 goto err_notif_uninit;
1737 }
1738 irq = rc;
1739
1740 rc = optee_smc_notif_init_irq(optee, irq);
1741 if (rc) {
1742 irq_dispose_mapping(irq);
1743 goto err_notif_uninit;
1744 }
1745 enable_async_notif(optee->smc.invoke_fn);
1746 pr_info("Asynchronous notifications enabled\n");
1747 }
1748
1749 /*
1750 * Ensure that there are no pre-existing shm objects before enabling
1751 * the shm cache so that there's no chance of receiving an invalid
1752 * address during shutdown. This could occur, for example, if we're
1753 * kexec booting from an older kernel that did not properly cleanup the
1754 * shm cache.
1755 */
1756 optee_disable_unmapped_shm_cache(optee);
1757
1758 /*
1759 * Only enable the shm cache in case we're not able to pass the RPC
1760 * arg struct right after the normal arg struct.
1761 */
1762 if (!optee->rpc_param_count)
1763 optee_enable_shm_cache(optee);
1764
1765 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1766 pr_info("dynamic shared memory is enabled\n");
1767
1768 rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1769 if (rc)
1770 goto err_disable_shm_cache;
1771
1772 pr_info("initialized driver\n");
1773 return 0;
1774
1775err_disable_shm_cache:
1776 if (!optee->rpc_param_count)
1777 optee_disable_shm_cache(optee);
1778 optee_smc_notif_uninit_irq(optee);
1779 optee_unregister_devices();
1780err_notif_uninit:
1781 optee_notif_uninit(optee);
1782err_close_ctx:
1783 teedev_close_context(ctx);
1784err_supp_uninit:
1785 optee_shm_arg_cache_uninit(optee);
1786 optee_supp_uninit(&optee->supp);
1787 mutex_destroy(&optee->call_queue.mutex);
1788err_unreg_supp_teedev:
1789 tee_device_unregister(optee->supp_teedev);
1790err_unreg_teedev:
1791 tee_device_unregister(optee->teedev);
1792err_free_optee:
1793 kfree(optee);
1794err_free_pool:
1795 tee_shm_pool_free(pool);
1796 if (memremaped_shm)
1797 memunmap(memremaped_shm);
1798 return rc;
1799}
1800
1801static const struct of_device_id optee_dt_match[] = {
1802 { .compatible = "linaro,optee-tz" },
1803 {},
1804};
1805MODULE_DEVICE_TABLE(of, optee_dt_match);
1806
1807static struct platform_driver optee_driver = {
1808 .probe = optee_probe,
1809 .remove = optee_smc_remove,
1810 .shutdown = optee_shutdown,
1811 .driver = {
1812 .name = "optee",
1813 .of_match_table = optee_dt_match,
1814 },
1815};
1816
1817int optee_smc_abi_register(void)
1818{
1819 return platform_driver_register(&optee_driver);
1820}
1821
1822void optee_smc_abi_unregister(void)
1823{
1824 platform_driver_unregister(&optee_driver);
1825}