Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2015-2021, 2023 Linaro Limited
4 */
5#include <linux/device.h>
6#include <linux/err.h>
7#include <linux/errno.h>
8#include <linux/mm.h>
9#include <linux/slab.h>
10#include <linux/tee_drv.h>
11#include <linux/types.h>
12#include "optee_private.h"
13
14#define MAX_ARG_PARAM_COUNT 6
15
16/*
17 * How much memory we allocate for each entry. This doesn't have to be a
18 * single page, but it makes sense to keep at least keep it as multiples of
19 * the page size.
20 */
21#define SHM_ENTRY_SIZE PAGE_SIZE
22
23/*
24 * We need to have a compile time constant to be able to determine the
25 * maximum needed size of the bit field.
26 */
27#define MIN_ARG_SIZE OPTEE_MSG_GET_ARG_SIZE(MAX_ARG_PARAM_COUNT)
28#define MAX_ARG_COUNT_PER_ENTRY (SHM_ENTRY_SIZE / MIN_ARG_SIZE)
29
30/*
31 * Shared memory for argument structs are cached here. The number of
32 * arguments structs that can fit is determined at runtime depending on the
33 * needed RPC parameter count reported by secure world
34 * (optee->rpc_param_count).
35 */
36struct optee_shm_arg_entry {
37 struct list_head list_node;
38 struct tee_shm *shm;
39 DECLARE_BITMAP(map, MAX_ARG_COUNT_PER_ENTRY);
40};
41
42void optee_cq_init(struct optee_call_queue *cq, int thread_count)
43{
44 mutex_init(&cq->mutex);
45 INIT_LIST_HEAD(&cq->waiters);
46
47 /*
48 * If cq->total_thread_count is 0 then we're not trying to keep
49 * track of how many free threads we have, instead we're relying on
50 * the secure world to tell us when we're out of thread and have to
51 * wait for another thread to become available.
52 */
53 cq->total_thread_count = thread_count;
54 cq->free_thread_count = thread_count;
55}
56
57void optee_cq_wait_init(struct optee_call_queue *cq,
58 struct optee_call_waiter *w, bool sys_thread)
59{
60 unsigned int free_thread_threshold;
61 bool need_wait = false;
62
63 memset(w, 0, sizeof(*w));
64
65 /*
66 * We're preparing to make a call to secure world. In case we can't
67 * allocate a thread in secure world we'll end up waiting in
68 * optee_cq_wait_for_completion().
69 *
70 * Normally if there's no contention in secure world the call will
71 * complete and we can cleanup directly with optee_cq_wait_final().
72 */
73 mutex_lock(&cq->mutex);
74
75 /*
76 * We add ourselves to the queue, but we don't wait. This
77 * guarantees that we don't lose a completion if secure world
78 * returns busy and another thread just exited and try to complete
79 * someone.
80 */
81 init_completion(&w->c);
82 list_add_tail(&w->list_node, &cq->waiters);
83 w->sys_thread = sys_thread;
84
85 if (cq->total_thread_count) {
86 if (sys_thread || !cq->sys_thread_req_count)
87 free_thread_threshold = 0;
88 else
89 free_thread_threshold = 1;
90
91 if (cq->free_thread_count > free_thread_threshold)
92 cq->free_thread_count--;
93 else
94 need_wait = true;
95 }
96
97 mutex_unlock(&cq->mutex);
98
99 while (need_wait) {
100 optee_cq_wait_for_completion(cq, w);
101 mutex_lock(&cq->mutex);
102
103 if (sys_thread || !cq->sys_thread_req_count)
104 free_thread_threshold = 0;
105 else
106 free_thread_threshold = 1;
107
108 if (cq->free_thread_count > free_thread_threshold) {
109 cq->free_thread_count--;
110 need_wait = false;
111 }
112
113 mutex_unlock(&cq->mutex);
114 }
115}
116
117void optee_cq_wait_for_completion(struct optee_call_queue *cq,
118 struct optee_call_waiter *w)
119{
120 wait_for_completion(&w->c);
121
122 mutex_lock(&cq->mutex);
123
124 /* Move to end of list to get out of the way for other waiters */
125 list_del(&w->list_node);
126 reinit_completion(&w->c);
127 list_add_tail(&w->list_node, &cq->waiters);
128
129 mutex_unlock(&cq->mutex);
130}
131
132static void optee_cq_complete_one(struct optee_call_queue *cq)
133{
134 struct optee_call_waiter *w;
135
136 /* Wake a waiting system session if any, prior to a normal session */
137 list_for_each_entry(w, &cq->waiters, list_node) {
138 if (w->sys_thread && !completion_done(&w->c)) {
139 complete(&w->c);
140 return;
141 }
142 }
143
144 list_for_each_entry(w, &cq->waiters, list_node) {
145 if (!completion_done(&w->c)) {
146 complete(&w->c);
147 break;
148 }
149 }
150}
151
152void optee_cq_wait_final(struct optee_call_queue *cq,
153 struct optee_call_waiter *w)
154{
155 /*
156 * We're done with the call to secure world. The thread in secure
157 * world that was used for this call is now available for some
158 * other task to use.
159 */
160 mutex_lock(&cq->mutex);
161
162 /* Get out of the list */
163 list_del(&w->list_node);
164
165 cq->free_thread_count++;
166
167 /* Wake up one eventual waiting task */
168 optee_cq_complete_one(cq);
169
170 /*
171 * If we're completed we've got a completion from another task that
172 * was just done with its call to secure world. Since yet another
173 * thread now is available in secure world wake up another eventual
174 * waiting task.
175 */
176 if (completion_done(&w->c))
177 optee_cq_complete_one(cq);
178
179 mutex_unlock(&cq->mutex);
180}
181
182/* Count registered system sessions to reserved a system thread or not */
183static bool optee_cq_incr_sys_thread_count(struct optee_call_queue *cq)
184{
185 if (cq->total_thread_count <= 1)
186 return false;
187
188 mutex_lock(&cq->mutex);
189 cq->sys_thread_req_count++;
190 mutex_unlock(&cq->mutex);
191
192 return true;
193}
194
195static void optee_cq_decr_sys_thread_count(struct optee_call_queue *cq)
196{
197 mutex_lock(&cq->mutex);
198 cq->sys_thread_req_count--;
199 /* If there's someone waiting, let it resume */
200 optee_cq_complete_one(cq);
201 mutex_unlock(&cq->mutex);
202}
203
204/* Requires the filpstate mutex to be held */
205static struct optee_session *find_session(struct optee_context_data *ctxdata,
206 u32 session_id)
207{
208 struct optee_session *sess;
209
210 list_for_each_entry(sess, &ctxdata->sess_list, list_node)
211 if (sess->session_id == session_id)
212 return sess;
213
214 return NULL;
215}
216
217void optee_shm_arg_cache_init(struct optee *optee, u32 flags)
218{
219 INIT_LIST_HEAD(&optee->shm_arg_cache.shm_args);
220 mutex_init(&optee->shm_arg_cache.mutex);
221 optee->shm_arg_cache.flags = flags;
222}
223
224void optee_shm_arg_cache_uninit(struct optee *optee)
225{
226 struct list_head *head = &optee->shm_arg_cache.shm_args;
227 struct optee_shm_arg_entry *entry;
228
229 mutex_destroy(&optee->shm_arg_cache.mutex);
230 while (!list_empty(head)) {
231 entry = list_first_entry(head, struct optee_shm_arg_entry,
232 list_node);
233 list_del(&entry->list_node);
234 if (find_first_bit(entry->map, MAX_ARG_COUNT_PER_ENTRY) !=
235 MAX_ARG_COUNT_PER_ENTRY) {
236 pr_err("Freeing non-free entry\n");
237 }
238 tee_shm_free(entry->shm);
239 kfree(entry);
240 }
241}
242
243size_t optee_msg_arg_size(size_t rpc_param_count)
244{
245 size_t sz = OPTEE_MSG_GET_ARG_SIZE(MAX_ARG_PARAM_COUNT);
246
247 if (rpc_param_count)
248 sz += OPTEE_MSG_GET_ARG_SIZE(rpc_param_count);
249
250 return sz;
251}
252
253/**
254 * optee_get_msg_arg() - Provide shared memory for argument struct
255 * @ctx: Caller TEE context
256 * @num_params: Number of parameter to store
257 * @entry_ret: Entry pointer, needed when freeing the buffer
258 * @shm_ret: Shared memory buffer
259 * @offs_ret: Offset of argument strut in shared memory buffer
260 *
261 * @returns a pointer to the argument struct in memory, else an ERR_PTR
262 */
263struct optee_msg_arg *optee_get_msg_arg(struct tee_context *ctx,
264 size_t num_params,
265 struct optee_shm_arg_entry **entry_ret,
266 struct tee_shm **shm_ret,
267 u_int *offs_ret)
268{
269 struct optee *optee = tee_get_drvdata(ctx->teedev);
270 size_t sz = optee_msg_arg_size(optee->rpc_param_count);
271 struct optee_shm_arg_entry *entry;
272 struct optee_msg_arg *ma;
273 size_t args_per_entry;
274 u_long bit;
275 u_int offs;
276 void *res;
277
278 if (num_params > MAX_ARG_PARAM_COUNT)
279 return ERR_PTR(-EINVAL);
280
281 if (optee->shm_arg_cache.flags & OPTEE_SHM_ARG_SHARED)
282 args_per_entry = SHM_ENTRY_SIZE / sz;
283 else
284 args_per_entry = 1;
285
286 mutex_lock(&optee->shm_arg_cache.mutex);
287 list_for_each_entry(entry, &optee->shm_arg_cache.shm_args, list_node) {
288 bit = find_first_zero_bit(entry->map, MAX_ARG_COUNT_PER_ENTRY);
289 if (bit < args_per_entry)
290 goto have_entry;
291 }
292
293 /*
294 * No entry was found, let's allocate a new.
295 */
296 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
297 if (!entry) {
298 res = ERR_PTR(-ENOMEM);
299 goto out;
300 }
301
302 if (optee->shm_arg_cache.flags & OPTEE_SHM_ARG_ALLOC_PRIV)
303 res = tee_shm_alloc_priv_buf(ctx, SHM_ENTRY_SIZE);
304 else
305 res = tee_shm_alloc_kernel_buf(ctx, SHM_ENTRY_SIZE);
306
307 if (IS_ERR(res)) {
308 kfree(entry);
309 goto out;
310 }
311 entry->shm = res;
312 list_add(&entry->list_node, &optee->shm_arg_cache.shm_args);
313 bit = 0;
314
315have_entry:
316 offs = bit * sz;
317 res = tee_shm_get_va(entry->shm, offs);
318 if (IS_ERR(res))
319 goto out;
320 ma = res;
321 set_bit(bit, entry->map);
322 memset(ma, 0, sz);
323 ma->num_params = num_params;
324 *entry_ret = entry;
325 *shm_ret = entry->shm;
326 *offs_ret = offs;
327out:
328 mutex_unlock(&optee->shm_arg_cache.mutex);
329 return res;
330}
331
332/**
333 * optee_free_msg_arg() - Free previsouly obtained shared memory
334 * @ctx: Caller TEE context
335 * @entry: Pointer returned when the shared memory was obtained
336 * @offs: Offset of shared memory buffer to free
337 *
338 * This function frees the shared memory obtained with optee_get_msg_arg().
339 */
340void optee_free_msg_arg(struct tee_context *ctx,
341 struct optee_shm_arg_entry *entry, u_int offs)
342{
343 struct optee *optee = tee_get_drvdata(ctx->teedev);
344 size_t sz = optee_msg_arg_size(optee->rpc_param_count);
345 u_long bit;
346
347 if (offs > SHM_ENTRY_SIZE || offs % sz) {
348 pr_err("Invalid offs %u\n", offs);
349 return;
350 }
351 bit = offs / sz;
352
353 mutex_lock(&optee->shm_arg_cache.mutex);
354
355 if (!test_bit(bit, entry->map))
356 pr_err("Bit pos %lu is already free\n", bit);
357 clear_bit(bit, entry->map);
358
359 mutex_unlock(&optee->shm_arg_cache.mutex);
360}
361
362int optee_open_session(struct tee_context *ctx,
363 struct tee_ioctl_open_session_arg *arg,
364 struct tee_param *param)
365{
366 struct optee *optee = tee_get_drvdata(ctx->teedev);
367 struct optee_context_data *ctxdata = ctx->data;
368 struct optee_shm_arg_entry *entry;
369 struct tee_shm *shm;
370 struct optee_msg_arg *msg_arg;
371 struct optee_session *sess = NULL;
372 uuid_t client_uuid;
373 u_int offs;
374 int rc;
375
376 /* +2 for the meta parameters added below */
377 msg_arg = optee_get_msg_arg(ctx, arg->num_params + 2,
378 &entry, &shm, &offs);
379 if (IS_ERR(msg_arg))
380 return PTR_ERR(msg_arg);
381
382 msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION;
383 msg_arg->cancel_id = arg->cancel_id;
384
385 /*
386 * Initialize and add the meta parameters needed when opening a
387 * session.
388 */
389 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
390 OPTEE_MSG_ATTR_META;
391 msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
392 OPTEE_MSG_ATTR_META;
393 memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid));
394 msg_arg->params[1].u.value.c = arg->clnt_login;
395
396 rc = tee_session_calc_client_uuid(&client_uuid, arg->clnt_login,
397 arg->clnt_uuid);
398 if (rc)
399 goto out;
400 export_uuid(msg_arg->params[1].u.octets, &client_uuid);
401
402 rc = optee->ops->to_msg_param(optee, msg_arg->params + 2,
403 arg->num_params, param);
404 if (rc)
405 goto out;
406
407 sess = kzalloc(sizeof(*sess), GFP_KERNEL);
408 if (!sess) {
409 rc = -ENOMEM;
410 goto out;
411 }
412
413 if (optee->ops->do_call_with_arg(ctx, shm, offs,
414 sess->use_sys_thread)) {
415 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
416 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
417 }
418
419 if (msg_arg->ret == TEEC_SUCCESS) {
420 /* A new session has been created, add it to the list. */
421 sess->session_id = msg_arg->session;
422 mutex_lock(&ctxdata->mutex);
423 list_add(&sess->list_node, &ctxdata->sess_list);
424 mutex_unlock(&ctxdata->mutex);
425 } else {
426 kfree(sess);
427 }
428
429 if (optee->ops->from_msg_param(optee, param, arg->num_params,
430 msg_arg->params + 2)) {
431 arg->ret = TEEC_ERROR_COMMUNICATION;
432 arg->ret_origin = TEEC_ORIGIN_COMMS;
433 /* Close session again to avoid leakage */
434 optee_close_session(ctx, msg_arg->session);
435 } else {
436 arg->session = msg_arg->session;
437 arg->ret = msg_arg->ret;
438 arg->ret_origin = msg_arg->ret_origin;
439 }
440out:
441 optee_free_msg_arg(ctx, entry, offs);
442
443 return rc;
444}
445
446int optee_system_session(struct tee_context *ctx, u32 session)
447{
448 struct optee *optee = tee_get_drvdata(ctx->teedev);
449 struct optee_context_data *ctxdata = ctx->data;
450 struct optee_session *sess;
451 int rc = -EINVAL;
452
453 mutex_lock(&ctxdata->mutex);
454
455 sess = find_session(ctxdata, session);
456 if (sess && (sess->use_sys_thread ||
457 optee_cq_incr_sys_thread_count(&optee->call_queue))) {
458 sess->use_sys_thread = true;
459 rc = 0;
460 }
461
462 mutex_unlock(&ctxdata->mutex);
463
464 return rc;
465}
466
467int optee_close_session_helper(struct tee_context *ctx, u32 session,
468 bool system_thread)
469{
470 struct optee *optee = tee_get_drvdata(ctx->teedev);
471 struct optee_shm_arg_entry *entry;
472 struct optee_msg_arg *msg_arg;
473 struct tee_shm *shm;
474 u_int offs;
475
476 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
477 if (IS_ERR(msg_arg))
478 return PTR_ERR(msg_arg);
479
480 msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION;
481 msg_arg->session = session;
482 optee->ops->do_call_with_arg(ctx, shm, offs, system_thread);
483
484 optee_free_msg_arg(ctx, entry, offs);
485
486 if (system_thread)
487 optee_cq_decr_sys_thread_count(&optee->call_queue);
488
489 return 0;
490}
491
492int optee_close_session(struct tee_context *ctx, u32 session)
493{
494 struct optee_context_data *ctxdata = ctx->data;
495 struct optee_session *sess;
496 bool system_thread;
497
498 /* Check that the session is valid and remove it from the list */
499 mutex_lock(&ctxdata->mutex);
500 sess = find_session(ctxdata, session);
501 if (sess)
502 list_del(&sess->list_node);
503 mutex_unlock(&ctxdata->mutex);
504 if (!sess)
505 return -EINVAL;
506 system_thread = sess->use_sys_thread;
507 kfree(sess);
508
509 return optee_close_session_helper(ctx, session, system_thread);
510}
511
512int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
513 struct tee_param *param)
514{
515 struct optee *optee = tee_get_drvdata(ctx->teedev);
516 struct optee_context_data *ctxdata = ctx->data;
517 struct optee_shm_arg_entry *entry;
518 struct optee_msg_arg *msg_arg;
519 struct optee_session *sess;
520 struct tee_shm *shm;
521 bool system_thread;
522 u_int offs;
523 int rc;
524
525 /* Check that the session is valid */
526 mutex_lock(&ctxdata->mutex);
527 sess = find_session(ctxdata, arg->session);
528 if (sess)
529 system_thread = sess->use_sys_thread;
530 mutex_unlock(&ctxdata->mutex);
531 if (!sess)
532 return -EINVAL;
533
534 msg_arg = optee_get_msg_arg(ctx, arg->num_params,
535 &entry, &shm, &offs);
536 if (IS_ERR(msg_arg))
537 return PTR_ERR(msg_arg);
538 msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND;
539 msg_arg->func = arg->func;
540 msg_arg->session = arg->session;
541 msg_arg->cancel_id = arg->cancel_id;
542
543 rc = optee->ops->to_msg_param(optee, msg_arg->params, arg->num_params,
544 param);
545 if (rc)
546 goto out;
547
548 if (optee->ops->do_call_with_arg(ctx, shm, offs, system_thread)) {
549 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
550 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
551 }
552
553 if (optee->ops->from_msg_param(optee, param, arg->num_params,
554 msg_arg->params)) {
555 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
556 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
557 }
558
559 arg->ret = msg_arg->ret;
560 arg->ret_origin = msg_arg->ret_origin;
561out:
562 optee_free_msg_arg(ctx, entry, offs);
563 return rc;
564}
565
566int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)
567{
568 struct optee *optee = tee_get_drvdata(ctx->teedev);
569 struct optee_context_data *ctxdata = ctx->data;
570 struct optee_shm_arg_entry *entry;
571 struct optee_msg_arg *msg_arg;
572 struct optee_session *sess;
573 bool system_thread;
574 struct tee_shm *shm;
575 u_int offs;
576
577 /* Check that the session is valid */
578 mutex_lock(&ctxdata->mutex);
579 sess = find_session(ctxdata, session);
580 if (sess)
581 system_thread = sess->use_sys_thread;
582 mutex_unlock(&ctxdata->mutex);
583 if (!sess)
584 return -EINVAL;
585
586 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
587 if (IS_ERR(msg_arg))
588 return PTR_ERR(msg_arg);
589
590 msg_arg->cmd = OPTEE_MSG_CMD_CANCEL;
591 msg_arg->session = session;
592 msg_arg->cancel_id = cancel_id;
593 optee->ops->do_call_with_arg(ctx, shm, offs, system_thread);
594
595 optee_free_msg_arg(ctx, entry, offs);
596 return 0;
597}
598
599static bool is_normal_memory(pgprot_t p)
600{
601#if defined(CONFIG_ARM)
602 return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) ||
603 ((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK));
604#elif defined(CONFIG_ARM64)
605 return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);
606#else
607#error "Unsupported architecture"
608#endif
609}
610
611static int __check_mem_type(struct mm_struct *mm, unsigned long start,
612 unsigned long end)
613{
614 struct vm_area_struct *vma;
615 VMA_ITERATOR(vmi, mm, start);
616
617 for_each_vma_range(vmi, vma, end) {
618 if (!is_normal_memory(vma->vm_page_prot))
619 return -EINVAL;
620 }
621
622 return 0;
623}
624
625int optee_check_mem_type(unsigned long start, size_t num_pages)
626{
627 struct mm_struct *mm = current->mm;
628 int rc;
629
630 /*
631 * Allow kernel address to register with OP-TEE as kernel
632 * pages are configured as normal memory only.
633 */
634 if (virt_addr_valid((void *)start) || is_vmalloc_addr((void *)start))
635 return 0;
636
637 mmap_read_lock(mm);
638 rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE);
639 mmap_read_unlock(mm);
640
641 return rc;
642}
643
644static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
645{
646 struct optee *optee = tee_get_drvdata(ctx->teedev);
647 struct optee_shm_arg_entry *entry;
648 struct optee_msg_arg *msg_arg;
649 struct tee_shm *shm;
650 u_int offs;
651
652 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
653 if (IS_ERR(msg_arg))
654 return PTR_ERR(msg_arg);
655
656 msg_arg->cmd = cmd;
657 optee->ops->do_call_with_arg(ctx, shm, offs, false);
658
659 optee_free_msg_arg(ctx, entry, offs);
660 return 0;
661}
662
663int optee_do_bottom_half(struct tee_context *ctx)
664{
665 return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
666}
667
668int optee_stop_async_notif(struct tee_context *ctx)
669{
670 return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
671}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2015, Linaro Limited
4 */
5#include <linux/arm-smccc.h>
6#include <linux/device.h>
7#include <linux/err.h>
8#include <linux/errno.h>
9#include <linux/mm.h>
10#include <linux/slab.h>
11#include <linux/tee_drv.h>
12#include <linux/types.h>
13#include <linux/uaccess.h>
14#include "optee_private.h"
15#include "optee_smc.h"
16
17struct optee_call_waiter {
18 struct list_head list_node;
19 struct completion c;
20};
21
22static void optee_cq_wait_init(struct optee_call_queue *cq,
23 struct optee_call_waiter *w)
24{
25 /*
26 * We're preparing to make a call to secure world. In case we can't
27 * allocate a thread in secure world we'll end up waiting in
28 * optee_cq_wait_for_completion().
29 *
30 * Normally if there's no contention in secure world the call will
31 * complete and we can cleanup directly with optee_cq_wait_final().
32 */
33 mutex_lock(&cq->mutex);
34
35 /*
36 * We add ourselves to the queue, but we don't wait. This
37 * guarantees that we don't lose a completion if secure world
38 * returns busy and another thread just exited and try to complete
39 * someone.
40 */
41 init_completion(&w->c);
42 list_add_tail(&w->list_node, &cq->waiters);
43
44 mutex_unlock(&cq->mutex);
45}
46
47static void optee_cq_wait_for_completion(struct optee_call_queue *cq,
48 struct optee_call_waiter *w)
49{
50 wait_for_completion(&w->c);
51
52 mutex_lock(&cq->mutex);
53
54 /* Move to end of list to get out of the way for other waiters */
55 list_del(&w->list_node);
56 reinit_completion(&w->c);
57 list_add_tail(&w->list_node, &cq->waiters);
58
59 mutex_unlock(&cq->mutex);
60}
61
62static void optee_cq_complete_one(struct optee_call_queue *cq)
63{
64 struct optee_call_waiter *w;
65
66 list_for_each_entry(w, &cq->waiters, list_node) {
67 if (!completion_done(&w->c)) {
68 complete(&w->c);
69 break;
70 }
71 }
72}
73
74static void optee_cq_wait_final(struct optee_call_queue *cq,
75 struct optee_call_waiter *w)
76{
77 /*
78 * We're done with the call to secure world. The thread in secure
79 * world that was used for this call is now available for some
80 * other task to use.
81 */
82 mutex_lock(&cq->mutex);
83
84 /* Get out of the list */
85 list_del(&w->list_node);
86
87 /* Wake up one eventual waiting task */
88 optee_cq_complete_one(cq);
89
90 /*
91 * If we're completed we've got a completion from another task that
92 * was just done with its call to secure world. Since yet another
93 * thread now is available in secure world wake up another eventual
94 * waiting task.
95 */
96 if (completion_done(&w->c))
97 optee_cq_complete_one(cq);
98
99 mutex_unlock(&cq->mutex);
100}
101
102/* Requires the filpstate mutex to be held */
103static struct optee_session *find_session(struct optee_context_data *ctxdata,
104 u32 session_id)
105{
106 struct optee_session *sess;
107
108 list_for_each_entry(sess, &ctxdata->sess_list, list_node)
109 if (sess->session_id == session_id)
110 return sess;
111
112 return NULL;
113}
114
115/**
116 * optee_do_call_with_arg() - Do an SMC to OP-TEE in secure world
117 * @ctx: calling context
118 * @parg: physical address of message to pass to secure world
119 *
120 * Does and SMC to OP-TEE in secure world and handles eventual resulting
121 * Remote Procedure Calls (RPC) from OP-TEE.
122 *
123 * Returns return code from secure world, 0 is OK
124 */
125u32 optee_do_call_with_arg(struct tee_context *ctx, phys_addr_t parg)
126{
127 struct optee *optee = tee_get_drvdata(ctx->teedev);
128 struct optee_call_waiter w;
129 struct optee_rpc_param param = { };
130 struct optee_call_ctx call_ctx = { };
131 u32 ret;
132
133 param.a0 = OPTEE_SMC_CALL_WITH_ARG;
134 reg_pair_from_64(¶m.a1, ¶m.a2, parg);
135 /* Initialize waiter */
136 optee_cq_wait_init(&optee->call_queue, &w);
137 while (true) {
138 struct arm_smccc_res res;
139
140 optee->invoke_fn(param.a0, param.a1, param.a2, param.a3,
141 param.a4, param.a5, param.a6, param.a7,
142 &res);
143
144 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
145 /*
146 * Out of threads in secure world, wait for a thread
147 * become available.
148 */
149 optee_cq_wait_for_completion(&optee->call_queue, &w);
150 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
151 might_sleep();
152 param.a0 = res.a0;
153 param.a1 = res.a1;
154 param.a2 = res.a2;
155 param.a3 = res.a3;
156 optee_handle_rpc(ctx, ¶m, &call_ctx);
157 } else {
158 ret = res.a0;
159 break;
160 }
161 }
162
163 optee_rpc_finalize_call(&call_ctx);
164 /*
165 * We're done with our thread in secure world, if there's any
166 * thread waiters wake up one.
167 */
168 optee_cq_wait_final(&optee->call_queue, &w);
169
170 return ret;
171}
172
173static struct tee_shm *get_msg_arg(struct tee_context *ctx, size_t num_params,
174 struct optee_msg_arg **msg_arg,
175 phys_addr_t *msg_parg)
176{
177 int rc;
178 struct tee_shm *shm;
179 struct optee_msg_arg *ma;
180
181 shm = tee_shm_alloc(ctx, OPTEE_MSG_GET_ARG_SIZE(num_params),
182 TEE_SHM_MAPPED);
183 if (IS_ERR(shm))
184 return shm;
185
186 ma = tee_shm_get_va(shm, 0);
187 if (IS_ERR(ma)) {
188 rc = PTR_ERR(ma);
189 goto out;
190 }
191
192 rc = tee_shm_get_pa(shm, 0, msg_parg);
193 if (rc)
194 goto out;
195
196 memset(ma, 0, OPTEE_MSG_GET_ARG_SIZE(num_params));
197 ma->num_params = num_params;
198 *msg_arg = ma;
199out:
200 if (rc) {
201 tee_shm_free(shm);
202 return ERR_PTR(rc);
203 }
204
205 return shm;
206}
207
208int optee_open_session(struct tee_context *ctx,
209 struct tee_ioctl_open_session_arg *arg,
210 struct tee_param *param)
211{
212 struct optee_context_data *ctxdata = ctx->data;
213 int rc;
214 struct tee_shm *shm;
215 struct optee_msg_arg *msg_arg;
216 phys_addr_t msg_parg;
217 struct optee_session *sess = NULL;
218
219 /* +2 for the meta parameters added below */
220 shm = get_msg_arg(ctx, arg->num_params + 2, &msg_arg, &msg_parg);
221 if (IS_ERR(shm))
222 return PTR_ERR(shm);
223
224 msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION;
225 msg_arg->cancel_id = arg->cancel_id;
226
227 /*
228 * Initialize and add the meta parameters needed when opening a
229 * session.
230 */
231 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
232 OPTEE_MSG_ATTR_META;
233 msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
234 OPTEE_MSG_ATTR_META;
235 memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid));
236 memcpy(&msg_arg->params[1].u.value, arg->uuid, sizeof(arg->clnt_uuid));
237 msg_arg->params[1].u.value.c = arg->clnt_login;
238
239 rc = optee_to_msg_param(msg_arg->params + 2, arg->num_params, param);
240 if (rc)
241 goto out;
242
243 sess = kzalloc(sizeof(*sess), GFP_KERNEL);
244 if (!sess) {
245 rc = -ENOMEM;
246 goto out;
247 }
248
249 if (optee_do_call_with_arg(ctx, msg_parg)) {
250 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
251 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
252 }
253
254 if (msg_arg->ret == TEEC_SUCCESS) {
255 /* A new session has been created, add it to the list. */
256 sess->session_id = msg_arg->session;
257 mutex_lock(&ctxdata->mutex);
258 list_add(&sess->list_node, &ctxdata->sess_list);
259 mutex_unlock(&ctxdata->mutex);
260 } else {
261 kfree(sess);
262 }
263
264 if (optee_from_msg_param(param, arg->num_params, msg_arg->params + 2)) {
265 arg->ret = TEEC_ERROR_COMMUNICATION;
266 arg->ret_origin = TEEC_ORIGIN_COMMS;
267 /* Close session again to avoid leakage */
268 optee_close_session(ctx, msg_arg->session);
269 } else {
270 arg->session = msg_arg->session;
271 arg->ret = msg_arg->ret;
272 arg->ret_origin = msg_arg->ret_origin;
273 }
274out:
275 tee_shm_free(shm);
276
277 return rc;
278}
279
280int optee_close_session(struct tee_context *ctx, u32 session)
281{
282 struct optee_context_data *ctxdata = ctx->data;
283 struct tee_shm *shm;
284 struct optee_msg_arg *msg_arg;
285 phys_addr_t msg_parg;
286 struct optee_session *sess;
287
288 /* Check that the session is valid and remove it from the list */
289 mutex_lock(&ctxdata->mutex);
290 sess = find_session(ctxdata, session);
291 if (sess)
292 list_del(&sess->list_node);
293 mutex_unlock(&ctxdata->mutex);
294 if (!sess)
295 return -EINVAL;
296 kfree(sess);
297
298 shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg);
299 if (IS_ERR(shm))
300 return PTR_ERR(shm);
301
302 msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION;
303 msg_arg->session = session;
304 optee_do_call_with_arg(ctx, msg_parg);
305
306 tee_shm_free(shm);
307 return 0;
308}
309
310int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
311 struct tee_param *param)
312{
313 struct optee_context_data *ctxdata = ctx->data;
314 struct tee_shm *shm;
315 struct optee_msg_arg *msg_arg;
316 phys_addr_t msg_parg;
317 struct optee_session *sess;
318 int rc;
319
320 /* Check that the session is valid */
321 mutex_lock(&ctxdata->mutex);
322 sess = find_session(ctxdata, arg->session);
323 mutex_unlock(&ctxdata->mutex);
324 if (!sess)
325 return -EINVAL;
326
327 shm = get_msg_arg(ctx, arg->num_params, &msg_arg, &msg_parg);
328 if (IS_ERR(shm))
329 return PTR_ERR(shm);
330 msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND;
331 msg_arg->func = arg->func;
332 msg_arg->session = arg->session;
333 msg_arg->cancel_id = arg->cancel_id;
334
335 rc = optee_to_msg_param(msg_arg->params, arg->num_params, param);
336 if (rc)
337 goto out;
338
339 if (optee_do_call_with_arg(ctx, msg_parg)) {
340 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
341 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
342 }
343
344 if (optee_from_msg_param(param, arg->num_params, msg_arg->params)) {
345 msg_arg->ret = TEEC_ERROR_COMMUNICATION;
346 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
347 }
348
349 arg->ret = msg_arg->ret;
350 arg->ret_origin = msg_arg->ret_origin;
351out:
352 tee_shm_free(shm);
353 return rc;
354}
355
356int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)
357{
358 struct optee_context_data *ctxdata = ctx->data;
359 struct tee_shm *shm;
360 struct optee_msg_arg *msg_arg;
361 phys_addr_t msg_parg;
362 struct optee_session *sess;
363
364 /* Check that the session is valid */
365 mutex_lock(&ctxdata->mutex);
366 sess = find_session(ctxdata, session);
367 mutex_unlock(&ctxdata->mutex);
368 if (!sess)
369 return -EINVAL;
370
371 shm = get_msg_arg(ctx, 0, &msg_arg, &msg_parg);
372 if (IS_ERR(shm))
373 return PTR_ERR(shm);
374
375 msg_arg->cmd = OPTEE_MSG_CMD_CANCEL;
376 msg_arg->session = session;
377 msg_arg->cancel_id = cancel_id;
378 optee_do_call_with_arg(ctx, msg_parg);
379
380 tee_shm_free(shm);
381 return 0;
382}
383
384/**
385 * optee_enable_shm_cache() - Enables caching of some shared memory allocation
386 * in OP-TEE
387 * @optee: main service struct
388 */
389void optee_enable_shm_cache(struct optee *optee)
390{
391 struct optee_call_waiter w;
392
393 /* We need to retry until secure world isn't busy. */
394 optee_cq_wait_init(&optee->call_queue, &w);
395 while (true) {
396 struct arm_smccc_res res;
397
398 optee->invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0,
399 0, &res);
400 if (res.a0 == OPTEE_SMC_RETURN_OK)
401 break;
402 optee_cq_wait_for_completion(&optee->call_queue, &w);
403 }
404 optee_cq_wait_final(&optee->call_queue, &w);
405}
406
407/**
408 * optee_disable_shm_cache() - Disables caching of some shared memory allocation
409 * in OP-TEE
410 * @optee: main service struct
411 */
412void optee_disable_shm_cache(struct optee *optee)
413{
414 struct optee_call_waiter w;
415
416 /* We need to retry until secure world isn't busy. */
417 optee_cq_wait_init(&optee->call_queue, &w);
418 while (true) {
419 union {
420 struct arm_smccc_res smccc;
421 struct optee_smc_disable_shm_cache_result result;
422 } res;
423
424 optee->invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE, 0, 0, 0, 0, 0, 0,
425 0, &res.smccc);
426 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
427 break; /* All shm's freed */
428 if (res.result.status == OPTEE_SMC_RETURN_OK) {
429 struct tee_shm *shm;
430
431 shm = reg_pair_to_ptr(res.result.shm_upper32,
432 res.result.shm_lower32);
433 tee_shm_free(shm);
434 } else {
435 optee_cq_wait_for_completion(&optee->call_queue, &w);
436 }
437 }
438 optee_cq_wait_final(&optee->call_queue, &w);
439}
440
441#define PAGELIST_ENTRIES_PER_PAGE \
442 ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
443
444/**
445 * optee_fill_pages_list() - write list of user pages to given shared
446 * buffer.
447 *
448 * @dst: page-aligned buffer where list of pages will be stored
449 * @pages: array of pages that represents shared buffer
450 * @num_pages: number of entries in @pages
451 * @page_offset: offset of user buffer from page start
452 *
453 * @dst should be big enough to hold list of user page addresses and
454 * links to the next pages of buffer
455 */
456void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
457 size_t page_offset)
458{
459 int n = 0;
460 phys_addr_t optee_page;
461 /*
462 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
463 * for details.
464 */
465 struct {
466 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
467 u64 next_page_data;
468 } *pages_data;
469
470 /*
471 * Currently OP-TEE uses 4k page size and it does not looks
472 * like this will change in the future. On other hand, there are
473 * no know ARM architectures with page size < 4k.
474 * Thus the next built assert looks redundant. But the following
475 * code heavily relies on this assumption, so it is better be
476 * safe than sorry.
477 */
478 BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
479
480 pages_data = (void *)dst;
481 /*
482 * If linux page is bigger than 4k, and user buffer offset is
483 * larger than 4k/8k/12k/etc this will skip first 4k pages,
484 * because they bear no value data for OP-TEE.
485 */
486 optee_page = page_to_phys(*pages) +
487 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
488
489 while (true) {
490 pages_data->pages_list[n++] = optee_page;
491
492 if (n == PAGELIST_ENTRIES_PER_PAGE) {
493 pages_data->next_page_data =
494 virt_to_phys(pages_data + 1);
495 pages_data++;
496 n = 0;
497 }
498
499 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
500 if (!(optee_page & ~PAGE_MASK)) {
501 if (!--num_pages)
502 break;
503 pages++;
504 optee_page = page_to_phys(*pages);
505 }
506 }
507}
508
509/*
510 * The final entry in each pagelist page is a pointer to the next
511 * pagelist page.
512 */
513static size_t get_pages_list_size(size_t num_entries)
514{
515 int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
516
517 return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
518}
519
520u64 *optee_allocate_pages_list(size_t num_entries)
521{
522 return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
523}
524
525void optee_free_pages_list(void *list, size_t num_entries)
526{
527 free_pages_exact(list, get_pages_list_size(num_entries));
528}
529
530static bool is_normal_memory(pgprot_t p)
531{
532#if defined(CONFIG_ARM)
533 return (pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC;
534#elif defined(CONFIG_ARM64)
535 return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);
536#else
537#error "Unuspported architecture"
538#endif
539}
540
541static int __check_mem_type(struct vm_area_struct *vma, unsigned long end)
542{
543 while (vma && is_normal_memory(vma->vm_page_prot)) {
544 if (vma->vm_end >= end)
545 return 0;
546 vma = vma->vm_next;
547 }
548
549 return -EINVAL;
550}
551
552static int check_mem_type(unsigned long start, size_t num_pages)
553{
554 struct mm_struct *mm = current->mm;
555 int rc;
556
557 down_read(&mm->mmap_sem);
558 rc = __check_mem_type(find_vma(mm, start),
559 start + num_pages * PAGE_SIZE);
560 up_read(&mm->mmap_sem);
561
562 return rc;
563}
564
565int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
566 struct page **pages, size_t num_pages,
567 unsigned long start)
568{
569 struct tee_shm *shm_arg = NULL;
570 struct optee_msg_arg *msg_arg;
571 u64 *pages_list;
572 phys_addr_t msg_parg;
573 int rc;
574
575 if (!num_pages)
576 return -EINVAL;
577
578 rc = check_mem_type(start, num_pages);
579 if (rc)
580 return rc;
581
582 pages_list = optee_allocate_pages_list(num_pages);
583 if (!pages_list)
584 return -ENOMEM;
585
586 shm_arg = get_msg_arg(ctx, 1, &msg_arg, &msg_parg);
587 if (IS_ERR(shm_arg)) {
588 rc = PTR_ERR(shm_arg);
589 goto out;
590 }
591
592 optee_fill_pages_list(pages_list, pages, num_pages,
593 tee_shm_get_page_offset(shm));
594
595 msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
596 msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
597 OPTEE_MSG_ATTR_NONCONTIG;
598 msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
599 msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
600 /*
601 * In the least bits of msg_arg->params->u.tmem.buf_ptr we
602 * store buffer offset from 4k page, as described in OP-TEE ABI.
603 */
604 msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
605 (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
606
607 if (optee_do_call_with_arg(ctx, msg_parg) ||
608 msg_arg->ret != TEEC_SUCCESS)
609 rc = -EINVAL;
610
611 tee_shm_free(shm_arg);
612out:
613 optee_free_pages_list(pages_list, num_pages);
614 return rc;
615}
616
617int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
618{
619 struct tee_shm *shm_arg;
620 struct optee_msg_arg *msg_arg;
621 phys_addr_t msg_parg;
622 int rc = 0;
623
624 shm_arg = get_msg_arg(ctx, 1, &msg_arg, &msg_parg);
625 if (IS_ERR(shm_arg))
626 return PTR_ERR(shm_arg);
627
628 msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
629
630 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
631 msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
632
633 if (optee_do_call_with_arg(ctx, msg_parg) ||
634 msg_arg->ret != TEEC_SUCCESS)
635 rc = -EINVAL;
636 tee_shm_free(shm_arg);
637 return rc;
638}
639
640int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
641 struct page **pages, size_t num_pages,
642 unsigned long start)
643{
644 /*
645 * We don't want to register supplicant memory in OP-TEE.
646 * Instead information about it will be passed in RPC code.
647 */
648 return check_mem_type(start, num_pages);
649}
650
651int optee_shm_unregister_supp(struct tee_context *ctx, struct tee_shm *shm)
652{
653 return 0;
654}