cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

smc_abi.c (41625B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (c) 2015-2021, Linaro Limited
      4 * Copyright (c) 2016, EPAM Systems
      5 */
      6
      7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
      8
      9#include <linux/arm-smccc.h>
     10#include <linux/errno.h>
     11#include <linux/interrupt.h>
     12#include <linux/io.h>
     13#include <linux/irqdomain.h>
     14#include <linux/mm.h>
     15#include <linux/module.h>
     16#include <linux/of.h>
     17#include <linux/of_irq.h>
     18#include <linux/of_platform.h>
     19#include <linux/platform_device.h>
     20#include <linux/sched.h>
     21#include <linux/slab.h>
     22#include <linux/string.h>
     23#include <linux/tee_drv.h>
     24#include <linux/types.h>
     25#include <linux/workqueue.h>
     26#include "optee_private.h"
     27#include "optee_smc.h"
     28#include "optee_rpc_cmd.h"
     29#include <linux/kmemleak.h>
     30#define CREATE_TRACE_POINTS
     31#include "optee_trace.h"
     32
     33/*
     34 * This file implement the SMC ABI used when communicating with secure world
     35 * OP-TEE OS via raw SMCs.
     36 * This file is divided into the following sections:
     37 * 1. Convert between struct tee_param and struct optee_msg_param
     38 * 2. Low level support functions to register shared memory in secure world
     39 * 3. Dynamic shared memory pool based on alloc_pages()
     40 * 4. Do a normal scheduled call into secure world
     41 * 5. Asynchronous notification
     42 * 6. Driver initialization.
     43 */
     44
     45/*
     46 * A typical OP-TEE private shm allocation is 224 bytes (argument struct
     47 * with 6 parameters, needed for open session). So with an alignment of 512
     48 * we'll waste a bit more than 50%. However, it's only expected that we'll
     49 * have a handful of these structs allocated at a time. Most memory will
     50 * be allocated aligned to the page size, So all in all this should scale
     51 * up and down quite well.
     52 */
     53#define OPTEE_MIN_STATIC_POOL_ALIGN    9 /* 512 bytes aligned */
     54
     55/*
     56 * 1. Convert between struct tee_param and struct optee_msg_param
     57 *
     58 * optee_from_msg_param() and optee_to_msg_param() are the main
     59 * functions.
     60 */
     61
     62static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
     63				  const struct optee_msg_param *mp)
     64{
     65	struct tee_shm *shm;
     66	phys_addr_t pa;
     67	int rc;
     68
     69	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
     70		  attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
     71	p->u.memref.size = mp->u.tmem.size;
     72	shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
     73	if (!shm) {
     74		p->u.memref.shm_offs = 0;
     75		p->u.memref.shm = NULL;
     76		return 0;
     77	}
     78
     79	rc = tee_shm_get_pa(shm, 0, &pa);
     80	if (rc)
     81		return rc;
     82
     83	p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
     84	p->u.memref.shm = shm;
     85
     86	return 0;
     87}
     88
     89static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
     90				   const struct optee_msg_param *mp)
     91{
     92	struct tee_shm *shm;
     93
     94	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
     95		  attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
     96	p->u.memref.size = mp->u.rmem.size;
     97	shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
     98
     99	if (shm) {
    100		p->u.memref.shm_offs = mp->u.rmem.offs;
    101		p->u.memref.shm = shm;
    102	} else {
    103		p->u.memref.shm_offs = 0;
    104		p->u.memref.shm = NULL;
    105	}
    106}
    107
    108/**
    109 * optee_from_msg_param() - convert from OPTEE_MSG parameters to
    110 *			    struct tee_param
    111 * @optee:	main service struct
    112 * @params:	subsystem internal parameter representation
    113 * @num_params:	number of elements in the parameter arrays
    114 * @msg_params:	OPTEE_MSG parameters
    115 * Returns 0 on success or <0 on failure
    116 */
    117static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
    118				size_t num_params,
    119				const struct optee_msg_param *msg_params)
    120{
    121	int rc;
    122	size_t n;
    123
    124	for (n = 0; n < num_params; n++) {
    125		struct tee_param *p = params + n;
    126		const struct optee_msg_param *mp = msg_params + n;
    127		u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
    128
    129		switch (attr) {
    130		case OPTEE_MSG_ATTR_TYPE_NONE:
    131			p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
    132			memset(&p->u, 0, sizeof(p->u));
    133			break;
    134		case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
    135		case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
    136		case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
    137			optee_from_msg_param_value(p, attr, mp);
    138			break;
    139		case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
    140		case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
    141		case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
    142			rc = from_msg_param_tmp_mem(p, attr, mp);
    143			if (rc)
    144				return rc;
    145			break;
    146		case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
    147		case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
    148		case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
    149			from_msg_param_reg_mem(p, attr, mp);
    150			break;
    151
    152		default:
    153			return -EINVAL;
    154		}
    155	}
    156	return 0;
    157}
    158
    159static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
    160				const struct tee_param *p)
    161{
    162	int rc;
    163	phys_addr_t pa;
    164
    165	mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
    166		   TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
    167
    168	mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
    169	mp->u.tmem.size = p->u.memref.size;
    170
    171	if (!p->u.memref.shm) {
    172		mp->u.tmem.buf_ptr = 0;
    173		return 0;
    174	}
    175
    176	rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa);
    177	if (rc)
    178		return rc;
    179
    180	mp->u.tmem.buf_ptr = pa;
    181	mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
    182		    OPTEE_MSG_ATTR_CACHE_SHIFT;
    183
    184	return 0;
    185}
    186
    187static int to_msg_param_reg_mem(struct optee_msg_param *mp,
    188				const struct tee_param *p)
    189{
    190	mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
    191		   TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
    192
    193	mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
    194	mp->u.rmem.size = p->u.memref.size;
    195	mp->u.rmem.offs = p->u.memref.shm_offs;
    196	return 0;
    197}
    198
    199/**
    200 * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
    201 * @optee:	main service struct
    202 * @msg_params:	OPTEE_MSG parameters
    203 * @num_params:	number of elements in the parameter arrays
    204 * @params:	subsystem itnernal parameter representation
    205 * Returns 0 on success or <0 on failure
    206 */
    207static int optee_to_msg_param(struct optee *optee,
    208			      struct optee_msg_param *msg_params,
    209			      size_t num_params, const struct tee_param *params)
    210{
    211	int rc;
    212	size_t n;
    213
    214	for (n = 0; n < num_params; n++) {
    215		const struct tee_param *p = params + n;
    216		struct optee_msg_param *mp = msg_params + n;
    217
    218		switch (p->attr) {
    219		case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
    220			mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
    221			memset(&mp->u, 0, sizeof(mp->u));
    222			break;
    223		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
    224		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
    225		case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
    226			optee_to_msg_param_value(mp, p);
    227			break;
    228		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
    229		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
    230		case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
    231			if (tee_shm_is_dynamic(p->u.memref.shm))
    232				rc = to_msg_param_reg_mem(mp, p);
    233			else
    234				rc = to_msg_param_tmp_mem(mp, p);
    235			if (rc)
    236				return rc;
    237			break;
    238		default:
    239			return -EINVAL;
    240		}
    241	}
    242	return 0;
    243}
    244
    245/*
    246 * 2. Low level support functions to register shared memory in secure world
    247 *
    248 * Functions to enable/disable shared memory caching in secure world, that
    249 * is, lazy freeing of previously allocated shared memory. Freeing is
    250 * performed when a request has been compled.
    251 *
    252 * Functions to register and unregister shared memory both for normal
    253 * clients and for tee-supplicant.
    254 */
    255
    256/**
    257 * optee_enable_shm_cache() - Enables caching of some shared memory allocation
    258 *			      in OP-TEE
    259 * @optee:	main service struct
    260 */
    261static void optee_enable_shm_cache(struct optee *optee)
    262{
    263	struct optee_call_waiter w;
    264
    265	/* We need to retry until secure world isn't busy. */
    266	optee_cq_wait_init(&optee->call_queue, &w);
    267	while (true) {
    268		struct arm_smccc_res res;
    269
    270		optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
    271				     0, 0, 0, 0, 0, 0, 0, &res);
    272		if (res.a0 == OPTEE_SMC_RETURN_OK)
    273			break;
    274		optee_cq_wait_for_completion(&optee->call_queue, &w);
    275	}
    276	optee_cq_wait_final(&optee->call_queue, &w);
    277}
    278
    279/**
    280 * __optee_disable_shm_cache() - Disables caching of some shared memory
    281 *				 allocation in OP-TEE
    282 * @optee:	main service struct
    283 * @is_mapped:	true if the cached shared memory addresses were mapped by this
    284 *		kernel, are safe to dereference, and should be freed
    285 */
    286static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
    287{
    288	struct optee_call_waiter w;
    289
    290	/* We need to retry until secure world isn't busy. */
    291	optee_cq_wait_init(&optee->call_queue, &w);
    292	while (true) {
    293		union {
    294			struct arm_smccc_res smccc;
    295			struct optee_smc_disable_shm_cache_result result;
    296		} res;
    297
    298		optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
    299				     0, 0, 0, 0, 0, 0, 0, &res.smccc);
    300		if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
    301			break; /* All shm's freed */
    302		if (res.result.status == OPTEE_SMC_RETURN_OK) {
    303			struct tee_shm *shm;
    304
    305			/*
    306			 * Shared memory references that were not mapped by
    307			 * this kernel must be ignored to prevent a crash.
    308			 */
    309			if (!is_mapped)
    310				continue;
    311
    312			shm = reg_pair_to_ptr(res.result.shm_upper32,
    313					      res.result.shm_lower32);
    314			tee_shm_free(shm);
    315		} else {
    316			optee_cq_wait_for_completion(&optee->call_queue, &w);
    317		}
    318	}
    319	optee_cq_wait_final(&optee->call_queue, &w);
    320}
    321
    322/**
    323 * optee_disable_shm_cache() - Disables caching of mapped shared memory
    324 *			       allocations in OP-TEE
    325 * @optee:	main service struct
    326 */
    327static void optee_disable_shm_cache(struct optee *optee)
    328{
    329	return __optee_disable_shm_cache(optee, true);
    330}
    331
    332/**
    333 * optee_disable_unmapped_shm_cache() - Disables caching of shared memory
    334 *					allocations in OP-TEE which are not
    335 *					currently mapped
    336 * @optee:	main service struct
    337 */
    338static void optee_disable_unmapped_shm_cache(struct optee *optee)
    339{
    340	return __optee_disable_shm_cache(optee, false);
    341}
    342
    343#define PAGELIST_ENTRIES_PER_PAGE				\
    344	((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
    345
    346/*
    347 * The final entry in each pagelist page is a pointer to the next
    348 * pagelist page.
    349 */
    350static size_t get_pages_list_size(size_t num_entries)
    351{
    352	int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
    353
    354	return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
    355}
    356
    357static u64 *optee_allocate_pages_list(size_t num_entries)
    358{
    359	return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
    360}
    361
    362static void optee_free_pages_list(void *list, size_t num_entries)
    363{
    364	free_pages_exact(list, get_pages_list_size(num_entries));
    365}
    366
    367/**
    368 * optee_fill_pages_list() - write list of user pages to given shared
    369 * buffer.
    370 *
    371 * @dst: page-aligned buffer where list of pages will be stored
    372 * @pages: array of pages that represents shared buffer
    373 * @num_pages: number of entries in @pages
    374 * @page_offset: offset of user buffer from page start
    375 *
    376 * @dst should be big enough to hold list of user page addresses and
    377 *	links to the next pages of buffer
    378 */
    379static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
    380				  size_t page_offset)
    381{
    382	int n = 0;
    383	phys_addr_t optee_page;
    384	/*
    385	 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
    386	 * for details.
    387	 */
    388	struct {
    389		u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
    390		u64 next_page_data;
    391	} *pages_data;
    392
    393	/*
    394	 * Currently OP-TEE uses 4k page size and it does not looks
    395	 * like this will change in the future.  On other hand, there are
    396	 * no know ARM architectures with page size < 4k.
    397	 * Thus the next built assert looks redundant. But the following
    398	 * code heavily relies on this assumption, so it is better be
    399	 * safe than sorry.
    400	 */
    401	BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
    402
    403	pages_data = (void *)dst;
    404	/*
    405	 * If linux page is bigger than 4k, and user buffer offset is
    406	 * larger than 4k/8k/12k/etc this will skip first 4k pages,
    407	 * because they bear no value data for OP-TEE.
    408	 */
    409	optee_page = page_to_phys(*pages) +
    410		round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
    411
    412	while (true) {
    413		pages_data->pages_list[n++] = optee_page;
    414
    415		if (n == PAGELIST_ENTRIES_PER_PAGE) {
    416			pages_data->next_page_data =
    417				virt_to_phys(pages_data + 1);
    418			pages_data++;
    419			n = 0;
    420		}
    421
    422		optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
    423		if (!(optee_page & ~PAGE_MASK)) {
    424			if (!--num_pages)
    425				break;
    426			pages++;
    427			optee_page = page_to_phys(*pages);
    428		}
    429	}
    430}
    431
    432static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
    433			      struct page **pages, size_t num_pages,
    434			      unsigned long start)
    435{
    436	struct optee *optee = tee_get_drvdata(ctx->teedev);
    437	struct optee_msg_arg *msg_arg;
    438	struct tee_shm *shm_arg;
    439	u64 *pages_list;
    440	size_t sz;
    441	int rc;
    442
    443	if (!num_pages)
    444		return -EINVAL;
    445
    446	rc = optee_check_mem_type(start, num_pages);
    447	if (rc)
    448		return rc;
    449
    450	pages_list = optee_allocate_pages_list(num_pages);
    451	if (!pages_list)
    452		return -ENOMEM;
    453
    454	/*
    455	 * We're about to register shared memory we can't register shared
    456	 * memory for this request or there's a catch-22.
    457	 *
    458	 * So in this we'll have to do the good old temporary private
    459	 * allocation instead of using optee_get_msg_arg().
    460	 */
    461	sz = optee_msg_arg_size(optee->rpc_param_count);
    462	shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
    463	if (IS_ERR(shm_arg)) {
    464		rc = PTR_ERR(shm_arg);
    465		goto out;
    466	}
    467	msg_arg = tee_shm_get_va(shm_arg, 0);
    468	if (IS_ERR(msg_arg)) {
    469		rc = PTR_ERR(msg_arg);
    470		goto out;
    471	}
    472
    473	optee_fill_pages_list(pages_list, pages, num_pages,
    474			      tee_shm_get_page_offset(shm));
    475
    476	memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
    477	msg_arg->num_params = 1;
    478	msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
    479	msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
    480				OPTEE_MSG_ATTR_NONCONTIG;
    481	msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
    482	msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
    483	/*
    484	 * In the least bits of msg_arg->params->u.tmem.buf_ptr we
    485	 * store buffer offset from 4k page, as described in OP-TEE ABI.
    486	 */
    487	msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
    488	  (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
    489
    490	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
    491	    msg_arg->ret != TEEC_SUCCESS)
    492		rc = -EINVAL;
    493
    494	tee_shm_free(shm_arg);
    495out:
    496	optee_free_pages_list(pages_list, num_pages);
    497	return rc;
    498}
    499
    500static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
    501{
    502	struct optee *optee = tee_get_drvdata(ctx->teedev);
    503	struct optee_msg_arg *msg_arg;
    504	struct tee_shm *shm_arg;
    505	int rc = 0;
    506	size_t sz;
    507
    508	/*
    509	 * We're about to unregister shared memory and we may not be able
    510	 * register shared memory for this request in case we're called
    511	 * from optee_shm_arg_cache_uninit().
    512	 *
    513	 * So in order to keep things simple in this function just as in
    514	 * optee_shm_register() we'll use temporary private allocation
    515	 * instead of using optee_get_msg_arg().
    516	 */
    517	sz = optee_msg_arg_size(optee->rpc_param_count);
    518	shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
    519	if (IS_ERR(shm_arg))
    520		return PTR_ERR(shm_arg);
    521	msg_arg = tee_shm_get_va(shm_arg, 0);
    522	if (IS_ERR(msg_arg)) {
    523		rc = PTR_ERR(msg_arg);
    524		goto out;
    525	}
    526
    527	memset(msg_arg, 0, sz);
    528	msg_arg->num_params = 1;
    529	msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
    530	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
    531	msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
    532
    533	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
    534	    msg_arg->ret != TEEC_SUCCESS)
    535		rc = -EINVAL;
    536out:
    537	tee_shm_free(shm_arg);
    538	return rc;
    539}
    540
    541static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
    542				   struct page **pages, size_t num_pages,
    543				   unsigned long start)
    544{
    545	/*
    546	 * We don't want to register supplicant memory in OP-TEE.
    547	 * Instead information about it will be passed in RPC code.
    548	 */
    549	return optee_check_mem_type(start, num_pages);
    550}
    551
    552static int optee_shm_unregister_supp(struct tee_context *ctx,
    553				     struct tee_shm *shm)
    554{
    555	return 0;
    556}
    557
    558/*
    559 * 3. Dynamic shared memory pool based on alloc_pages()
    560 *
    561 * Implements an OP-TEE specific shared memory pool which is used
    562 * when dynamic shared memory is supported by secure world.
    563 *
    564 * The main function is optee_shm_pool_alloc_pages().
    565 */
    566
    567static int pool_op_alloc(struct tee_shm_pool *pool,
    568			 struct tee_shm *shm, size_t size, size_t align)
    569{
    570	/*
    571	 * Shared memory private to the OP-TEE driver doesn't need
    572	 * to be registered with OP-TEE.
    573	 */
    574	if (shm->flags & TEE_SHM_PRIV)
    575		return optee_pool_op_alloc_helper(pool, shm, size, align, NULL);
    576
    577	return optee_pool_op_alloc_helper(pool, shm, size, align,
    578					  optee_shm_register);
    579}
    580
    581static void pool_op_free(struct tee_shm_pool *pool,
    582			 struct tee_shm *shm)
    583{
    584	if (!(shm->flags & TEE_SHM_PRIV))
    585		optee_pool_op_free_helper(pool, shm, optee_shm_unregister);
    586	else
    587		optee_pool_op_free_helper(pool, shm, NULL);
    588}
    589
    590static void pool_op_destroy_pool(struct tee_shm_pool *pool)
    591{
    592	kfree(pool);
    593}
    594
    595static const struct tee_shm_pool_ops pool_ops = {
    596	.alloc = pool_op_alloc,
    597	.free = pool_op_free,
    598	.destroy_pool = pool_op_destroy_pool,
    599};
    600
    601/**
    602 * optee_shm_pool_alloc_pages() - create page-based allocator pool
    603 *
    604 * This pool is used when OP-TEE supports dymanic SHM. In this case
    605 * command buffers and such are allocated from kernel's own memory.
    606 */
    607static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
    608{
    609	struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
    610
    611	if (!pool)
    612		return ERR_PTR(-ENOMEM);
    613
    614	pool->ops = &pool_ops;
    615
    616	return pool;
    617}
    618
    619/*
    620 * 4. Do a normal scheduled call into secure world
    621 *
    622 * The function optee_smc_do_call_with_arg() performs a normal scheduled
    623 * call into secure world. During this call may normal world request help
    624 * from normal world using RPCs, Remote Procedure Calls. This includes
    625 * delivery of non-secure interrupts to for instance allow rescheduling of
    626 * the current task.
    627 */
    628
    629static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
    630					 struct optee_msg_arg *arg)
    631{
    632	struct tee_shm *shm;
    633
    634	arg->ret_origin = TEEC_ORIGIN_COMMS;
    635
    636	if (arg->num_params != 1 ||
    637	    arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
    638		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
    639		return;
    640	}
    641
    642	shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
    643	switch (arg->params[0].u.value.a) {
    644	case OPTEE_RPC_SHM_TYPE_APPL:
    645		optee_rpc_cmd_free_suppl(ctx, shm);
    646		break;
    647	case OPTEE_RPC_SHM_TYPE_KERNEL:
    648		tee_shm_free(shm);
    649		break;
    650	default:
    651		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
    652	}
    653	arg->ret = TEEC_SUCCESS;
    654}
    655
    656static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
    657					  struct optee *optee,
    658					  struct optee_msg_arg *arg,
    659					  struct optee_call_ctx *call_ctx)
    660{
    661	phys_addr_t pa;
    662	struct tee_shm *shm;
    663	size_t sz;
    664	size_t n;
    665
    666	arg->ret_origin = TEEC_ORIGIN_COMMS;
    667
    668	if (!arg->num_params ||
    669	    arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
    670		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
    671		return;
    672	}
    673
    674	for (n = 1; n < arg->num_params; n++) {
    675		if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
    676			arg->ret = TEEC_ERROR_BAD_PARAMETERS;
    677			return;
    678		}
    679	}
    680
    681	sz = arg->params[0].u.value.b;
    682	switch (arg->params[0].u.value.a) {
    683	case OPTEE_RPC_SHM_TYPE_APPL:
    684		shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
    685		break;
    686	case OPTEE_RPC_SHM_TYPE_KERNEL:
    687		shm = tee_shm_alloc_priv_buf(optee->ctx, sz);
    688		break;
    689	default:
    690		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
    691		return;
    692	}
    693
    694	if (IS_ERR(shm)) {
    695		arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
    696		return;
    697	}
    698
    699	if (tee_shm_get_pa(shm, 0, &pa)) {
    700		arg->ret = TEEC_ERROR_BAD_PARAMETERS;
    701		goto bad;
    702	}
    703
    704	sz = tee_shm_get_size(shm);
    705
    706	if (tee_shm_is_dynamic(shm)) {
    707		struct page **pages;
    708		u64 *pages_list;
    709		size_t page_num;
    710
    711		pages = tee_shm_get_pages(shm, &page_num);
    712		if (!pages || !page_num) {
    713			arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
    714			goto bad;
    715		}
    716
    717		pages_list = optee_allocate_pages_list(page_num);
    718		if (!pages_list) {
    719			arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
    720			goto bad;
    721		}
    722
    723		call_ctx->pages_list = pages_list;
    724		call_ctx->num_entries = page_num;
    725
    726		arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
    727				      OPTEE_MSG_ATTR_NONCONTIG;
    728		/*
    729		 * In the least bits of u.tmem.buf_ptr we store buffer offset
    730		 * from 4k page, as described in OP-TEE ABI.
    731		 */
    732		arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) |
    733			(tee_shm_get_page_offset(shm) &
    734			 (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
    735		arg->params[0].u.tmem.size = tee_shm_get_size(shm);
    736		arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
    737
    738		optee_fill_pages_list(pages_list, pages, page_num,
    739				      tee_shm_get_page_offset(shm));
    740	} else {
    741		arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
    742		arg->params[0].u.tmem.buf_ptr = pa;
    743		arg->params[0].u.tmem.size = sz;
    744		arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
    745	}
    746
    747	arg->ret = TEEC_SUCCESS;
    748	return;
    749bad:
    750	tee_shm_free(shm);
    751}
    752
    753static void free_pages_list(struct optee_call_ctx *call_ctx)
    754{
    755	if (call_ctx->pages_list) {
    756		optee_free_pages_list(call_ctx->pages_list,
    757				      call_ctx->num_entries);
    758		call_ctx->pages_list = NULL;
    759		call_ctx->num_entries = 0;
    760	}
    761}
    762
    763static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
    764{
    765	free_pages_list(call_ctx);
    766}
    767
    768static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
    769				struct optee_msg_arg *arg,
    770				struct optee_call_ctx *call_ctx)
    771{
    772
    773	switch (arg->cmd) {
    774	case OPTEE_RPC_CMD_SHM_ALLOC:
    775		free_pages_list(call_ctx);
    776		handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
    777		break;
    778	case OPTEE_RPC_CMD_SHM_FREE:
    779		handle_rpc_func_cmd_shm_free(ctx, arg);
    780		break;
    781	default:
    782		optee_rpc_cmd(ctx, optee, arg);
    783	}
    784}
    785
    786/**
    787 * optee_handle_rpc() - handle RPC from secure world
    788 * @ctx:	context doing the RPC
    789 * @param:	value of registers for the RPC
    790 * @call_ctx:	call context. Preserved during one OP-TEE invocation
    791 *
    792 * Result of RPC is written back into @param.
    793 */
    794static void optee_handle_rpc(struct tee_context *ctx,
    795			     struct optee_msg_arg *rpc_arg,
    796			     struct optee_rpc_param *param,
    797			     struct optee_call_ctx *call_ctx)
    798{
    799	struct tee_device *teedev = ctx->teedev;
    800	struct optee *optee = tee_get_drvdata(teedev);
    801	struct optee_msg_arg *arg;
    802	struct tee_shm *shm;
    803	phys_addr_t pa;
    804
    805	switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
    806	case OPTEE_SMC_RPC_FUNC_ALLOC:
    807		shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1);
    808		if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {
    809			reg_pair_from_64(&param->a1, &param->a2, pa);
    810			reg_pair_from_64(&param->a4, &param->a5,
    811					 (unsigned long)shm);
    812		} else {
    813			param->a1 = 0;
    814			param->a2 = 0;
    815			param->a4 = 0;
    816			param->a5 = 0;
    817		}
    818		kmemleak_not_leak(shm);
    819		break;
    820	case OPTEE_SMC_RPC_FUNC_FREE:
    821		shm = reg_pair_to_ptr(param->a1, param->a2);
    822		tee_shm_free(shm);
    823		break;
    824	case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
    825		/*
    826		 * A foreign interrupt was raised while secure world was
    827		 * executing, since they are handled in Linux a dummy RPC is
    828		 * performed to let Linux take the interrupt through the normal
    829		 * vector.
    830		 */
    831		break;
    832	case OPTEE_SMC_RPC_FUNC_CMD:
    833		if (rpc_arg) {
    834			arg = rpc_arg;
    835		} else {
    836			shm = reg_pair_to_ptr(param->a1, param->a2);
    837			arg = tee_shm_get_va(shm, 0);
    838			if (IS_ERR(arg)) {
    839				pr_err("%s: tee_shm_get_va %p failed\n",
    840				       __func__, shm);
    841				break;
    842			}
    843		}
    844
    845		handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
    846		break;
    847	default:
    848		pr_warn("Unknown RPC func 0x%x\n",
    849			(u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
    850		break;
    851	}
    852
    853	param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
    854}
    855
    856/**
    857 * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
    858 * @ctx:	calling context
    859 * @shm:	shared memory holding the message to pass to secure world
    860 * @offs:	offset of the message in @shm
    861 *
    862 * Does and SMC to OP-TEE in secure world and handles eventual resulting
    863 * Remote Procedure Calls (RPC) from OP-TEE.
    864 *
    865 * Returns return code from secure world, 0 is OK
    866 */
    867static int optee_smc_do_call_with_arg(struct tee_context *ctx,
    868				      struct tee_shm *shm, u_int offs)
    869{
    870	struct optee *optee = tee_get_drvdata(ctx->teedev);
    871	struct optee_call_waiter w;
    872	struct optee_rpc_param param = { };
    873	struct optee_call_ctx call_ctx = { };
    874	struct optee_msg_arg *rpc_arg = NULL;
    875	int rc;
    876
    877	if (optee->rpc_param_count) {
    878		struct optee_msg_arg *arg;
    879		unsigned int rpc_arg_offs;
    880
    881		arg = tee_shm_get_va(shm, offs);
    882		if (IS_ERR(arg))
    883			return PTR_ERR(arg);
    884
    885		rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
    886		rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);
    887		if (IS_ERR(arg))
    888			return PTR_ERR(arg);
    889	}
    890
    891	if  (rpc_arg && tee_shm_is_dynamic(shm)) {
    892		param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
    893		reg_pair_from_64(&param.a1, &param.a2, (u_long)shm);
    894		param.a3 = offs;
    895	} else {
    896		phys_addr_t parg;
    897
    898		rc = tee_shm_get_pa(shm, offs, &parg);
    899		if (rc)
    900			return rc;
    901
    902		if (rpc_arg)
    903			param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
    904		else
    905			param.a0 = OPTEE_SMC_CALL_WITH_ARG;
    906		reg_pair_from_64(&param.a1, &param.a2, parg);
    907	}
    908	/* Initialize waiter */
    909	optee_cq_wait_init(&optee->call_queue, &w);
    910	while (true) {
    911		struct arm_smccc_res res;
    912
    913		trace_optee_invoke_fn_begin(&param);
    914		optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
    915				     param.a4, param.a5, param.a6, param.a7,
    916				     &res);
    917		trace_optee_invoke_fn_end(&param, &res);
    918
    919		if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
    920			/*
    921			 * Out of threads in secure world, wait for a thread
    922			 * become available.
    923			 */
    924			optee_cq_wait_for_completion(&optee->call_queue, &w);
    925		} else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
    926			cond_resched();
    927			param.a0 = res.a0;
    928			param.a1 = res.a1;
    929			param.a2 = res.a2;
    930			param.a3 = res.a3;
    931			optee_handle_rpc(ctx, rpc_arg, &param, &call_ctx);
    932		} else {
    933			rc = res.a0;
    934			break;
    935		}
    936	}
    937
    938	optee_rpc_finalize_call(&call_ctx);
    939	/*
    940	 * We're done with our thread in secure world, if there's any
    941	 * thread waiters wake up one.
    942	 */
    943	optee_cq_wait_final(&optee->call_queue, &w);
    944
    945	return rc;
    946}
    947
    948static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
    949{
    950	struct optee_shm_arg_entry *entry;
    951	struct optee_msg_arg *msg_arg;
    952	struct tee_shm *shm;
    953	u_int offs;
    954
    955	msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
    956	if (IS_ERR(msg_arg))
    957		return PTR_ERR(msg_arg);
    958
    959	msg_arg->cmd = cmd;
    960	optee_smc_do_call_with_arg(ctx, shm, offs);
    961
    962	optee_free_msg_arg(ctx, entry, offs);
    963	return 0;
    964}
    965
    966static int optee_smc_do_bottom_half(struct tee_context *ctx)
    967{
    968	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
    969}
    970
    971static int optee_smc_stop_async_notif(struct tee_context *ctx)
    972{
    973	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
    974}
    975
    976/*
    977 * 5. Asynchronous notification
    978 */
    979
    980static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
    981				 bool *value_pending)
    982{
    983	struct arm_smccc_res res;
    984
    985	invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
    986
    987	if (res.a0)
    988		return 0;
    989	*value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
    990	*value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
    991	return res.a1;
    992}
    993
    994static irqreturn_t notif_irq_handler(int irq, void *dev_id)
    995{
    996	struct optee *optee = dev_id;
    997	bool do_bottom_half = false;
    998	bool value_valid;
    999	bool value_pending;
   1000	u32 value;
   1001
   1002	do {
   1003		value = get_async_notif_value(optee->smc.invoke_fn,
   1004					      &value_valid, &value_pending);
   1005		if (!value_valid)
   1006			break;
   1007
   1008		if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
   1009			do_bottom_half = true;
   1010		else
   1011			optee_notif_send(optee, value);
   1012	} while (value_pending);
   1013
   1014	if (do_bottom_half)
   1015		return IRQ_WAKE_THREAD;
   1016	return IRQ_HANDLED;
   1017}
   1018
   1019static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
   1020{
   1021	struct optee *optee = dev_id;
   1022
   1023	optee_smc_do_bottom_half(optee->ctx);
   1024
   1025	return IRQ_HANDLED;
   1026}
   1027
   1028static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
   1029{
   1030	int rc;
   1031
   1032	rc = request_threaded_irq(irq, notif_irq_handler,
   1033				  notif_irq_thread_fn,
   1034				  0, "optee_notification", optee);
   1035	if (rc)
   1036		return rc;
   1037
   1038	optee->smc.notif_irq = irq;
   1039
   1040	return 0;
   1041}
   1042
   1043static void optee_smc_notif_uninit_irq(struct optee *optee)
   1044{
   1045	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
   1046		optee_smc_stop_async_notif(optee->ctx);
   1047		if (optee->smc.notif_irq) {
   1048			free_irq(optee->smc.notif_irq, optee);
   1049			irq_dispose_mapping(optee->smc.notif_irq);
   1050		}
   1051	}
   1052}
   1053
   1054/*
   1055 * 6. Driver initialization
   1056 *
   1057 * During driver initialization is secure world probed to find out which
   1058 * features it supports so the driver can be initialized with a matching
   1059 * configuration. This involves for instance support for dynamic shared
   1060 * memory instead of a static memory carvout.
   1061 */
   1062
   1063static void optee_get_version(struct tee_device *teedev,
   1064			      struct tee_ioctl_version_data *vers)
   1065{
   1066	struct tee_ioctl_version_data v = {
   1067		.impl_id = TEE_IMPL_ID_OPTEE,
   1068		.impl_caps = TEE_OPTEE_CAP_TZ,
   1069		.gen_caps = TEE_GEN_CAP_GP,
   1070	};
   1071	struct optee *optee = tee_get_drvdata(teedev);
   1072
   1073	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
   1074		v.gen_caps |= TEE_GEN_CAP_REG_MEM;
   1075	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
   1076		v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
   1077	*vers = v;
   1078}
   1079
   1080static int optee_smc_open(struct tee_context *ctx)
   1081{
   1082	struct optee *optee = tee_get_drvdata(ctx->teedev);
   1083	u32 sec_caps = optee->smc.sec_caps;
   1084
   1085	return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
   1086}
   1087
   1088static const struct tee_driver_ops optee_clnt_ops = {
   1089	.get_version = optee_get_version,
   1090	.open = optee_smc_open,
   1091	.release = optee_release,
   1092	.open_session = optee_open_session,
   1093	.close_session = optee_close_session,
   1094	.invoke_func = optee_invoke_func,
   1095	.cancel_req = optee_cancel_req,
   1096	.shm_register = optee_shm_register,
   1097	.shm_unregister = optee_shm_unregister,
   1098};
   1099
   1100static const struct tee_desc optee_clnt_desc = {
   1101	.name = DRIVER_NAME "-clnt",
   1102	.ops = &optee_clnt_ops,
   1103	.owner = THIS_MODULE,
   1104};
   1105
   1106static const struct tee_driver_ops optee_supp_ops = {
   1107	.get_version = optee_get_version,
   1108	.open = optee_smc_open,
   1109	.release = optee_release_supp,
   1110	.supp_recv = optee_supp_recv,
   1111	.supp_send = optee_supp_send,
   1112	.shm_register = optee_shm_register_supp,
   1113	.shm_unregister = optee_shm_unregister_supp,
   1114};
   1115
   1116static const struct tee_desc optee_supp_desc = {
   1117	.name = DRIVER_NAME "-supp",
   1118	.ops = &optee_supp_ops,
   1119	.owner = THIS_MODULE,
   1120	.flags = TEE_DESC_PRIVILEGED,
   1121};
   1122
   1123static const struct optee_ops optee_ops = {
   1124	.do_call_with_arg = optee_smc_do_call_with_arg,
   1125	.to_msg_param = optee_to_msg_param,
   1126	.from_msg_param = optee_from_msg_param,
   1127};
   1128
   1129static int enable_async_notif(optee_invoke_fn *invoke_fn)
   1130{
   1131	struct arm_smccc_res res;
   1132
   1133	invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
   1134
   1135	if (res.a0)
   1136		return -EINVAL;
   1137	return 0;
   1138}
   1139
   1140static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
   1141{
   1142	struct arm_smccc_res res;
   1143
   1144	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
   1145
   1146	if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
   1147	    res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
   1148		return true;
   1149	return false;
   1150}
   1151
   1152static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
   1153{
   1154	union {
   1155		struct arm_smccc_res smccc;
   1156		struct optee_smc_call_get_os_revision_result result;
   1157	} res = {
   1158		.result = {
   1159			.build_id = 0
   1160		}
   1161	};
   1162
   1163	invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
   1164		  &res.smccc);
   1165
   1166	if (res.result.build_id)
   1167		pr_info("revision %lu.%lu (%08lx)", res.result.major,
   1168			res.result.minor, res.result.build_id);
   1169	else
   1170		pr_info("revision %lu.%lu", res.result.major, res.result.minor);
   1171}
   1172
   1173static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
   1174{
   1175	union {
   1176		struct arm_smccc_res smccc;
   1177		struct optee_smc_calls_revision_result result;
   1178	} res;
   1179
   1180	invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
   1181
   1182	if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
   1183	    (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
   1184		return true;
   1185	return false;
   1186}
   1187
   1188static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
   1189					    u32 *sec_caps, u32 *max_notif_value,
   1190					    unsigned int *rpc_param_count)
   1191{
   1192	union {
   1193		struct arm_smccc_res smccc;
   1194		struct optee_smc_exchange_capabilities_result result;
   1195	} res;
   1196	u32 a1 = 0;
   1197
   1198	/*
   1199	 * TODO This isn't enough to tell if it's UP system (from kernel
   1200	 * point of view) or not, is_smp() returns the information
   1201	 * needed, but can't be called directly from here.
   1202	 */
   1203	if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
   1204		a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
   1205
   1206	invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
   1207		  &res.smccc);
   1208
   1209	if (res.result.status != OPTEE_SMC_RETURN_OK)
   1210		return false;
   1211
   1212	*sec_caps = res.result.capabilities;
   1213	if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
   1214		*max_notif_value = res.result.max_notif_value;
   1215	else
   1216		*max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
   1217	if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
   1218		*rpc_param_count = (u8)res.result.data;
   1219	else
   1220		*rpc_param_count = 0;
   1221
   1222	return true;
   1223}
   1224
   1225static struct tee_shm_pool *
   1226optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
   1227{
   1228	union {
   1229		struct arm_smccc_res smccc;
   1230		struct optee_smc_get_shm_config_result result;
   1231	} res;
   1232	unsigned long vaddr;
   1233	phys_addr_t paddr;
   1234	size_t size;
   1235	phys_addr_t begin;
   1236	phys_addr_t end;
   1237	void *va;
   1238	void *rc;
   1239
   1240	invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
   1241	if (res.result.status != OPTEE_SMC_RETURN_OK) {
   1242		pr_err("static shm service not available\n");
   1243		return ERR_PTR(-ENOENT);
   1244	}
   1245
   1246	if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
   1247		pr_err("only normal cached shared memory supported\n");
   1248		return ERR_PTR(-EINVAL);
   1249	}
   1250
   1251	begin = roundup(res.result.start, PAGE_SIZE);
   1252	end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
   1253	paddr = begin;
   1254	size = end - begin;
   1255
   1256	va = memremap(paddr, size, MEMREMAP_WB);
   1257	if (!va) {
   1258		pr_err("shared memory ioremap failed\n");
   1259		return ERR_PTR(-EINVAL);
   1260	}
   1261	vaddr = (unsigned long)va;
   1262
   1263	rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
   1264					OPTEE_MIN_STATIC_POOL_ALIGN);
   1265	if (IS_ERR(rc))
   1266		memunmap(va);
   1267	else
   1268		*memremaped_shm = va;
   1269
   1270	return rc;
   1271}
   1272
   1273/* Simple wrapper functions to be able to use a function pointer */
   1274static void optee_smccc_smc(unsigned long a0, unsigned long a1,
   1275			    unsigned long a2, unsigned long a3,
   1276			    unsigned long a4, unsigned long a5,
   1277			    unsigned long a6, unsigned long a7,
   1278			    struct arm_smccc_res *res)
   1279{
   1280	arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
   1281}
   1282
   1283static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
   1284			    unsigned long a2, unsigned long a3,
   1285			    unsigned long a4, unsigned long a5,
   1286			    unsigned long a6, unsigned long a7,
   1287			    struct arm_smccc_res *res)
   1288{
   1289	arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
   1290}
   1291
   1292static optee_invoke_fn *get_invoke_func(struct device *dev)
   1293{
   1294	const char *method;
   1295
   1296	pr_info("probing for conduit method.\n");
   1297
   1298	if (device_property_read_string(dev, "method", &method)) {
   1299		pr_warn("missing \"method\" property\n");
   1300		return ERR_PTR(-ENXIO);
   1301	}
   1302
   1303	if (!strcmp("hvc", method))
   1304		return optee_smccc_hvc;
   1305	else if (!strcmp("smc", method))
   1306		return optee_smccc_smc;
   1307
   1308	pr_warn("invalid \"method\" property: %s\n", method);
   1309	return ERR_PTR(-EINVAL);
   1310}
   1311
   1312/* optee_remove - Device Removal Routine
   1313 * @pdev: platform device information struct
   1314 *
   1315 * optee_remove is called by platform subsystem to alert the driver
   1316 * that it should release the device
   1317 */
   1318static int optee_smc_remove(struct platform_device *pdev)
   1319{
   1320	struct optee *optee = platform_get_drvdata(pdev);
   1321
   1322	/*
   1323	 * Ask OP-TEE to free all cached shared memory objects to decrease
   1324	 * reference counters and also avoid wild pointers in secure world
   1325	 * into the old shared memory range.
   1326	 */
   1327	if (!optee->rpc_param_count)
   1328		optee_disable_shm_cache(optee);
   1329
   1330	optee_smc_notif_uninit_irq(optee);
   1331
   1332	optee_remove_common(optee);
   1333
   1334	if (optee->smc.memremaped_shm)
   1335		memunmap(optee->smc.memremaped_shm);
   1336
   1337	kfree(optee);
   1338
   1339	return 0;
   1340}
   1341
   1342/* optee_shutdown - Device Removal Routine
   1343 * @pdev: platform device information struct
   1344 *
   1345 * platform_shutdown is called by the platform subsystem to alert
   1346 * the driver that a shutdown, reboot, or kexec is happening and
   1347 * device must be disabled.
   1348 */
   1349static void optee_shutdown(struct platform_device *pdev)
   1350{
   1351	struct optee *optee = platform_get_drvdata(pdev);
   1352
   1353	if (!optee->rpc_param_count)
   1354		optee_disable_shm_cache(optee);
   1355}
   1356
   1357static int optee_probe(struct platform_device *pdev)
   1358{
   1359	optee_invoke_fn *invoke_fn;
   1360	struct tee_shm_pool *pool = ERR_PTR(-EINVAL);
   1361	struct optee *optee = NULL;
   1362	void *memremaped_shm = NULL;
   1363	unsigned int rpc_param_count;
   1364	struct tee_device *teedev;
   1365	struct tee_context *ctx;
   1366	u32 max_notif_value;
   1367	u32 arg_cache_flags;
   1368	u32 sec_caps;
   1369	int rc;
   1370
   1371	invoke_fn = get_invoke_func(&pdev->dev);
   1372	if (IS_ERR(invoke_fn))
   1373		return PTR_ERR(invoke_fn);
   1374
   1375	if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
   1376		pr_warn("api uid mismatch\n");
   1377		return -EINVAL;
   1378	}
   1379
   1380	optee_msg_get_os_revision(invoke_fn);
   1381
   1382	if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
   1383		pr_warn("api revision mismatch\n");
   1384		return -EINVAL;
   1385	}
   1386
   1387	if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,
   1388					     &max_notif_value,
   1389					     &rpc_param_count)) {
   1390		pr_warn("capabilities mismatch\n");
   1391		return -EINVAL;
   1392	}
   1393
   1394	/*
   1395	 * Try to use dynamic shared memory if possible
   1396	 */
   1397	if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
   1398		/*
   1399		 * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
   1400		 * optee_get_msg_arg() to pre-register (by having
   1401		 * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
   1402		 * an argument struct.
   1403		 *
   1404		 * With the page is pre-registered we can use a non-zero
   1405		 * offset for argument struct, this is indicated with
   1406		 * OPTEE_SHM_ARG_SHARED.
   1407		 *
   1408		 * This means that optee_smc_do_call_with_arg() will use
   1409		 * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
   1410		 */
   1411		if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
   1412			arg_cache_flags = OPTEE_SHM_ARG_SHARED;
   1413		else
   1414			arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
   1415
   1416		pool = optee_shm_pool_alloc_pages();
   1417	}
   1418
   1419	/*
   1420	 * If dynamic shared memory is not available or failed - try static one
   1421	 */
   1422	if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
   1423		/*
   1424		 * The static memory pool can use non-zero page offsets so
   1425		 * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
   1426		 *
   1427		 * optee_get_msg_arg() should not pre-register the
   1428		 * allocated page used to pass an argument struct, this is
   1429		 * indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
   1430		 *
   1431		 * This means that optee_smc_do_call_with_arg() will use
   1432		 * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
   1433		 * OPTEE_SMC_CALL_WITH_RPC_ARG.
   1434		 */
   1435		arg_cache_flags = OPTEE_SHM_ARG_SHARED |
   1436				  OPTEE_SHM_ARG_ALLOC_PRIV;
   1437		pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm);
   1438	}
   1439
   1440	if (IS_ERR(pool))
   1441		return PTR_ERR(pool);
   1442
   1443	optee = kzalloc(sizeof(*optee), GFP_KERNEL);
   1444	if (!optee) {
   1445		rc = -ENOMEM;
   1446		goto err_free_pool;
   1447	}
   1448
   1449	optee->ops = &optee_ops;
   1450	optee->smc.invoke_fn = invoke_fn;
   1451	optee->smc.sec_caps = sec_caps;
   1452	optee->rpc_param_count = rpc_param_count;
   1453
   1454	teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee);
   1455	if (IS_ERR(teedev)) {
   1456		rc = PTR_ERR(teedev);
   1457		goto err_free_optee;
   1458	}
   1459	optee->teedev = teedev;
   1460
   1461	teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee);
   1462	if (IS_ERR(teedev)) {
   1463		rc = PTR_ERR(teedev);
   1464		goto err_unreg_teedev;
   1465	}
   1466	optee->supp_teedev = teedev;
   1467
   1468	rc = tee_device_register(optee->teedev);
   1469	if (rc)
   1470		goto err_unreg_supp_teedev;
   1471
   1472	rc = tee_device_register(optee->supp_teedev);
   1473	if (rc)
   1474		goto err_unreg_supp_teedev;
   1475
   1476	mutex_init(&optee->call_queue.mutex);
   1477	INIT_LIST_HEAD(&optee->call_queue.waiters);
   1478	optee_supp_init(&optee->supp);
   1479	optee->smc.memremaped_shm = memremaped_shm;
   1480	optee->pool = pool;
   1481	optee_shm_arg_cache_init(optee, arg_cache_flags);
   1482
   1483	platform_set_drvdata(pdev, optee);
   1484	ctx = teedev_open(optee->teedev);
   1485	if (IS_ERR(ctx)) {
   1486		rc = PTR_ERR(ctx);
   1487		goto err_supp_uninit;
   1488	}
   1489	optee->ctx = ctx;
   1490	rc = optee_notif_init(optee, max_notif_value);
   1491	if (rc)
   1492		goto err_close_ctx;
   1493
   1494	if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
   1495		unsigned int irq;
   1496
   1497		rc = platform_get_irq(pdev, 0);
   1498		if (rc < 0) {
   1499			pr_err("platform_get_irq: ret %d\n", rc);
   1500			goto err_notif_uninit;
   1501		}
   1502		irq = rc;
   1503
   1504		rc = optee_smc_notif_init_irq(optee, irq);
   1505		if (rc) {
   1506			irq_dispose_mapping(irq);
   1507			goto err_notif_uninit;
   1508		}
   1509		enable_async_notif(optee->smc.invoke_fn);
   1510		pr_info("Asynchronous notifications enabled\n");
   1511	}
   1512
   1513	/*
   1514	 * Ensure that there are no pre-existing shm objects before enabling
   1515	 * the shm cache so that there's no chance of receiving an invalid
   1516	 * address during shutdown. This could occur, for example, if we're
   1517	 * kexec booting from an older kernel that did not properly cleanup the
   1518	 * shm cache.
   1519	 */
   1520	optee_disable_unmapped_shm_cache(optee);
   1521
   1522	/*
   1523	 * Only enable the shm cache in case we're not able to pass the RPC
   1524	 * arg struct right after the normal arg struct.
   1525	 */
   1526	if (!optee->rpc_param_count)
   1527		optee_enable_shm_cache(optee);
   1528
   1529	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
   1530		pr_info("dynamic shared memory is enabled\n");
   1531
   1532	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
   1533	if (rc)
   1534		goto err_disable_shm_cache;
   1535
   1536	pr_info("initialized driver\n");
   1537	return 0;
   1538
   1539err_disable_shm_cache:
   1540	if (!optee->rpc_param_count)
   1541		optee_disable_shm_cache(optee);
   1542	optee_smc_notif_uninit_irq(optee);
   1543	optee_unregister_devices();
   1544err_notif_uninit:
   1545	optee_notif_uninit(optee);
   1546err_close_ctx:
   1547	teedev_close_context(ctx);
   1548err_supp_uninit:
   1549	optee_shm_arg_cache_uninit(optee);
   1550	optee_supp_uninit(&optee->supp);
   1551	mutex_destroy(&optee->call_queue.mutex);
   1552err_unreg_supp_teedev:
   1553	tee_device_unregister(optee->supp_teedev);
   1554err_unreg_teedev:
   1555	tee_device_unregister(optee->teedev);
   1556err_free_optee:
   1557	kfree(optee);
   1558err_free_pool:
   1559	tee_shm_pool_free(pool);
   1560	if (memremaped_shm)
   1561		memunmap(memremaped_shm);
   1562	return rc;
   1563}
   1564
   1565static const struct of_device_id optee_dt_match[] = {
   1566	{ .compatible = "linaro,optee-tz" },
   1567	{},
   1568};
   1569MODULE_DEVICE_TABLE(of, optee_dt_match);
   1570
   1571static struct platform_driver optee_driver = {
   1572	.probe  = optee_probe,
   1573	.remove = optee_smc_remove,
   1574	.shutdown = optee_shutdown,
   1575	.driver = {
   1576		.name = "optee",
   1577		.of_match_table = optee_dt_match,
   1578	},
   1579};
   1580
   1581int optee_smc_abi_register(void)
   1582{
   1583	return platform_driver_register(&optee_driver);
   1584}
   1585
   1586void optee_smc_abi_unregister(void)
   1587{
   1588	platform_driver_unregister(&optee_driver);
   1589}