cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

ccp-dev-v5.c (30654B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * AMD Cryptographic Coprocessor (CCP) driver
      4 *
      5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
      6 *
      7 * Author: Gary R Hook <gary.hook@amd.com>
      8 */
      9
     10#include <linux/kernel.h>
     11#include <linux/kthread.h>
     12#include <linux/dma-mapping.h>
     13#include <linux/interrupt.h>
     14#include <linux/compiler.h>
     15#include <linux/ccp.h>
     16
     17#include "ccp-dev.h"
     18
     19/* Allocate the requested number of contiguous LSB slots
     20 * from the LSB bitmap. Look in the private range for this
     21 * queue first; failing that, check the public area.
     22 * If no space is available, wait around.
     23 * Return: first slot number
     24 */
     25static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
     26{
     27	struct ccp_device *ccp;
     28	int start;
     29
     30	/* First look at the map for the queue */
     31	if (cmd_q->lsb >= 0) {
     32		start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
     33							LSB_SIZE,
     34							0, count, 0);
     35		if (start < LSB_SIZE) {
     36			bitmap_set(cmd_q->lsbmap, start, count);
     37			return start + cmd_q->lsb * LSB_SIZE;
     38		}
     39	}
     40
     41	/* No joy; try to get an entry from the shared blocks */
     42	ccp = cmd_q->ccp;
     43	for (;;) {
     44		mutex_lock(&ccp->sb_mutex);
     45
     46		start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
     47							MAX_LSB_CNT * LSB_SIZE,
     48							0,
     49							count, 0);
     50		if (start <= MAX_LSB_CNT * LSB_SIZE) {
     51			bitmap_set(ccp->lsbmap, start, count);
     52
     53			mutex_unlock(&ccp->sb_mutex);
     54			return start;
     55		}
     56
     57		ccp->sb_avail = 0;
     58
     59		mutex_unlock(&ccp->sb_mutex);
     60
     61		/* Wait for KSB entries to become available */
     62		if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
     63			return 0;
     64	}
     65}
     66
     67/* Free a number of LSB slots from the bitmap, starting at
     68 * the indicated starting slot number.
     69 */
     70static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
     71			 unsigned int count)
     72{
     73	if (!start)
     74		return;
     75
     76	if (cmd_q->lsb == start) {
     77		/* An entry from the private LSB */
     78		bitmap_clear(cmd_q->lsbmap, start, count);
     79	} else {
     80		/* From the shared LSBs */
     81		struct ccp_device *ccp = cmd_q->ccp;
     82
     83		mutex_lock(&ccp->sb_mutex);
     84		bitmap_clear(ccp->lsbmap, start, count);
     85		ccp->sb_avail = 1;
     86		mutex_unlock(&ccp->sb_mutex);
     87		wake_up_interruptible_all(&ccp->sb_queue);
     88	}
     89}
     90
     91/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
     92union ccp_function {
     93	struct {
     94		u16 size:7;
     95		u16 encrypt:1;
     96		u16 mode:5;
     97		u16 type:2;
     98	} aes;
     99	struct {
    100		u16 size:7;
    101		u16 encrypt:1;
    102		u16 rsvd:5;
    103		u16 type:2;
    104	} aes_xts;
    105	struct {
    106		u16 size:7;
    107		u16 encrypt:1;
    108		u16 mode:5;
    109		u16 type:2;
    110	} des3;
    111	struct {
    112		u16 rsvd1:10;
    113		u16 type:4;
    114		u16 rsvd2:1;
    115	} sha;
    116	struct {
    117		u16 mode:3;
    118		u16 size:12;
    119	} rsa;
    120	struct {
    121		u16 byteswap:2;
    122		u16 bitwise:3;
    123		u16 reflect:2;
    124		u16 rsvd:8;
    125	} pt;
    126	struct  {
    127		u16 rsvd:13;
    128	} zlib;
    129	struct {
    130		u16 size:10;
    131		u16 type:2;
    132		u16 mode:3;
    133	} ecc;
    134	u16 raw;
    135};
    136
    137#define	CCP_AES_SIZE(p)		((p)->aes.size)
    138#define	CCP_AES_ENCRYPT(p)	((p)->aes.encrypt)
    139#define	CCP_AES_MODE(p)		((p)->aes.mode)
    140#define	CCP_AES_TYPE(p)		((p)->aes.type)
    141#define	CCP_XTS_SIZE(p)		((p)->aes_xts.size)
    142#define	CCP_XTS_TYPE(p)		((p)->aes_xts.type)
    143#define	CCP_XTS_ENCRYPT(p)	((p)->aes_xts.encrypt)
    144#define	CCP_DES3_SIZE(p)	((p)->des3.size)
    145#define	CCP_DES3_ENCRYPT(p)	((p)->des3.encrypt)
    146#define	CCP_DES3_MODE(p)	((p)->des3.mode)
    147#define	CCP_DES3_TYPE(p)	((p)->des3.type)
    148#define	CCP_SHA_TYPE(p)		((p)->sha.type)
    149#define	CCP_RSA_SIZE(p)		((p)->rsa.size)
    150#define	CCP_PT_BYTESWAP(p)	((p)->pt.byteswap)
    151#define	CCP_PT_BITWISE(p)	((p)->pt.bitwise)
    152#define	CCP_ECC_MODE(p)		((p)->ecc.mode)
    153#define	CCP_ECC_AFFINE(p)	((p)->ecc.one)
    154
    155/* Word 0 */
    156#define CCP5_CMD_DW0(p)		((p)->dw0)
    157#define CCP5_CMD_SOC(p)		(CCP5_CMD_DW0(p).soc)
    158#define CCP5_CMD_IOC(p)		(CCP5_CMD_DW0(p).ioc)
    159#define CCP5_CMD_INIT(p)	(CCP5_CMD_DW0(p).init)
    160#define CCP5_CMD_EOM(p)		(CCP5_CMD_DW0(p).eom)
    161#define CCP5_CMD_FUNCTION(p)	(CCP5_CMD_DW0(p).function)
    162#define CCP5_CMD_ENGINE(p)	(CCP5_CMD_DW0(p).engine)
    163#define CCP5_CMD_PROT(p)	(CCP5_CMD_DW0(p).prot)
    164
    165/* Word 1 */
    166#define CCP5_CMD_DW1(p)		((p)->length)
    167#define CCP5_CMD_LEN(p)		(CCP5_CMD_DW1(p))
    168
    169/* Word 2 */
    170#define CCP5_CMD_DW2(p)		((p)->src_lo)
    171#define CCP5_CMD_SRC_LO(p)	(CCP5_CMD_DW2(p))
    172
    173/* Word 3 */
    174#define CCP5_CMD_DW3(p)		((p)->dw3)
    175#define CCP5_CMD_SRC_MEM(p)	((p)->dw3.src_mem)
    176#define CCP5_CMD_SRC_HI(p)	((p)->dw3.src_hi)
    177#define CCP5_CMD_LSB_ID(p)	((p)->dw3.lsb_cxt_id)
    178#define CCP5_CMD_FIX_SRC(p)	((p)->dw3.fixed)
    179
    180/* Words 4/5 */
    181#define CCP5_CMD_DW4(p)		((p)->dw4)
    182#define CCP5_CMD_DST_LO(p)	(CCP5_CMD_DW4(p).dst_lo)
    183#define CCP5_CMD_DW5(p)		((p)->dw5.fields.dst_hi)
    184#define CCP5_CMD_DST_HI(p)	(CCP5_CMD_DW5(p))
    185#define CCP5_CMD_DST_MEM(p)	((p)->dw5.fields.dst_mem)
    186#define CCP5_CMD_FIX_DST(p)	((p)->dw5.fields.fixed)
    187#define CCP5_CMD_SHA_LO(p)	((p)->dw4.sha_len_lo)
    188#define CCP5_CMD_SHA_HI(p)	((p)->dw5.sha_len_hi)
    189
    190/* Word 6/7 */
    191#define CCP5_CMD_DW6(p)		((p)->key_lo)
    192#define CCP5_CMD_KEY_LO(p)	(CCP5_CMD_DW6(p))
    193#define CCP5_CMD_DW7(p)		((p)->dw7)
    194#define CCP5_CMD_KEY_HI(p)	((p)->dw7.key_hi)
    195#define CCP5_CMD_KEY_MEM(p)	((p)->dw7.key_mem)
    196
    197static inline u32 low_address(unsigned long addr)
    198{
    199	return (u64)addr & 0x0ffffffff;
    200}
    201
    202static inline u32 high_address(unsigned long addr)
    203{
    204	return ((u64)addr >> 32) & 0x00000ffff;
    205}
    206
    207static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
    208{
    209	unsigned int head_idx, n;
    210	u32 head_lo, queue_start;
    211
    212	queue_start = low_address(cmd_q->qdma_tail);
    213	head_lo = ioread32(cmd_q->reg_head_lo);
    214	head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
    215
    216	n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
    217
    218	return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
    219}
    220
    221static int ccp5_do_cmd(struct ccp5_desc *desc,
    222		       struct ccp_cmd_queue *cmd_q)
    223{
    224	__le32 *mP;
    225	u32 *dP;
    226	u32 tail;
    227	int	i;
    228	int ret = 0;
    229
    230	cmd_q->total_ops++;
    231
    232	if (CCP5_CMD_SOC(desc)) {
    233		CCP5_CMD_IOC(desc) = 1;
    234		CCP5_CMD_SOC(desc) = 0;
    235	}
    236	mutex_lock(&cmd_q->q_mutex);
    237
    238	mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx];
    239	dP = (u32 *)desc;
    240	for (i = 0; i < 8; i++)
    241		mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
    242
    243	cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
    244
    245	/* The data used by this command must be flushed to memory */
    246	wmb();
    247
    248	/* Write the new tail address back to the queue register */
    249	tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
    250	iowrite32(tail, cmd_q->reg_tail_lo);
    251
    252	/* Turn the queue back on using our cached control register */
    253	iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
    254	mutex_unlock(&cmd_q->q_mutex);
    255
    256	if (CCP5_CMD_IOC(desc)) {
    257		/* Wait for the job to complete */
    258		ret = wait_event_interruptible(cmd_q->int_queue,
    259					       cmd_q->int_rcvd);
    260		if (ret || cmd_q->cmd_error) {
    261			/* Log the error and flush the queue by
    262			 * moving the head pointer
    263			 */
    264			if (cmd_q->cmd_error)
    265				ccp_log_error(cmd_q->ccp,
    266					      cmd_q->cmd_error);
    267			iowrite32(tail, cmd_q->reg_head_lo);
    268			if (!ret)
    269				ret = -EIO;
    270		}
    271		cmd_q->int_rcvd = 0;
    272	}
    273
    274	return ret;
    275}
    276
    277static int ccp5_perform_aes(struct ccp_op *op)
    278{
    279	struct ccp5_desc desc;
    280	union ccp_function function;
    281	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
    282
    283	op->cmd_q->total_aes_ops++;
    284
    285	/* Zero out all the fields of the command desc */
    286	memset(&desc, 0, Q_DESC_SIZE);
    287
    288	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
    289
    290	CCP5_CMD_SOC(&desc) = op->soc;
    291	CCP5_CMD_IOC(&desc) = 1;
    292	CCP5_CMD_INIT(&desc) = op->init;
    293	CCP5_CMD_EOM(&desc) = op->eom;
    294	CCP5_CMD_PROT(&desc) = 0;
    295
    296	function.raw = 0;
    297	CCP_AES_ENCRYPT(&function) = op->u.aes.action;
    298	CCP_AES_MODE(&function) = op->u.aes.mode;
    299	CCP_AES_TYPE(&function) = op->u.aes.type;
    300	CCP_AES_SIZE(&function) = op->u.aes.size;
    301
    302	CCP5_CMD_FUNCTION(&desc) = function.raw;
    303
    304	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
    305
    306	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    307	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    308	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    309
    310	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
    311	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
    312	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    313
    314	CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
    315	CCP5_CMD_KEY_HI(&desc) = 0;
    316	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
    317	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
    318
    319	return ccp5_do_cmd(&desc, op->cmd_q);
    320}
    321
    322static int ccp5_perform_xts_aes(struct ccp_op *op)
    323{
    324	struct ccp5_desc desc;
    325	union ccp_function function;
    326	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
    327
    328	op->cmd_q->total_xts_aes_ops++;
    329
    330	/* Zero out all the fields of the command desc */
    331	memset(&desc, 0, Q_DESC_SIZE);
    332
    333	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
    334
    335	CCP5_CMD_SOC(&desc) = op->soc;
    336	CCP5_CMD_IOC(&desc) = 1;
    337	CCP5_CMD_INIT(&desc) = op->init;
    338	CCP5_CMD_EOM(&desc) = op->eom;
    339	CCP5_CMD_PROT(&desc) = 0;
    340
    341	function.raw = 0;
    342	CCP_XTS_TYPE(&function) = op->u.xts.type;
    343	CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
    344	CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
    345	CCP5_CMD_FUNCTION(&desc) = function.raw;
    346
    347	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
    348
    349	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    350	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    351	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    352
    353	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
    354	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
    355	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    356
    357	CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
    358	CCP5_CMD_KEY_HI(&desc) =  0;
    359	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
    360	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
    361
    362	return ccp5_do_cmd(&desc, op->cmd_q);
    363}
    364
    365static int ccp5_perform_sha(struct ccp_op *op)
    366{
    367	struct ccp5_desc desc;
    368	union ccp_function function;
    369
    370	op->cmd_q->total_sha_ops++;
    371
    372	/* Zero out all the fields of the command desc */
    373	memset(&desc, 0, Q_DESC_SIZE);
    374
    375	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
    376
    377	CCP5_CMD_SOC(&desc) = op->soc;
    378	CCP5_CMD_IOC(&desc) = 1;
    379	CCP5_CMD_INIT(&desc) = 1;
    380	CCP5_CMD_EOM(&desc) = op->eom;
    381	CCP5_CMD_PROT(&desc) = 0;
    382
    383	function.raw = 0;
    384	CCP_SHA_TYPE(&function) = op->u.sha.type;
    385	CCP5_CMD_FUNCTION(&desc) = function.raw;
    386
    387	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
    388
    389	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    390	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    391	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    392
    393	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
    394
    395	if (op->eom) {
    396		CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
    397		CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
    398	} else {
    399		CCP5_CMD_SHA_LO(&desc) = 0;
    400		CCP5_CMD_SHA_HI(&desc) = 0;
    401	}
    402
    403	return ccp5_do_cmd(&desc, op->cmd_q);
    404}
    405
    406static int ccp5_perform_des3(struct ccp_op *op)
    407{
    408	struct ccp5_desc desc;
    409	union ccp_function function;
    410	u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
    411
    412	op->cmd_q->total_3des_ops++;
    413
    414	/* Zero out all the fields of the command desc */
    415	memset(&desc, 0, sizeof(struct ccp5_desc));
    416
    417	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3;
    418
    419	CCP5_CMD_SOC(&desc) = op->soc;
    420	CCP5_CMD_IOC(&desc) = 1;
    421	CCP5_CMD_INIT(&desc) = op->init;
    422	CCP5_CMD_EOM(&desc) = op->eom;
    423	CCP5_CMD_PROT(&desc) = 0;
    424
    425	function.raw = 0;
    426	CCP_DES3_ENCRYPT(&function) = op->u.des3.action;
    427	CCP_DES3_MODE(&function) = op->u.des3.mode;
    428	CCP_DES3_TYPE(&function) = op->u.des3.type;
    429	CCP5_CMD_FUNCTION(&desc) = function.raw;
    430
    431	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
    432
    433	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    434	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    435	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    436
    437	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
    438	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
    439	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    440
    441	CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
    442	CCP5_CMD_KEY_HI(&desc) = 0;
    443	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
    444	CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
    445
    446	return ccp5_do_cmd(&desc, op->cmd_q);
    447}
    448
    449static int ccp5_perform_rsa(struct ccp_op *op)
    450{
    451	struct ccp5_desc desc;
    452	union ccp_function function;
    453
    454	op->cmd_q->total_rsa_ops++;
    455
    456	/* Zero out all the fields of the command desc */
    457	memset(&desc, 0, Q_DESC_SIZE);
    458
    459	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
    460
    461	CCP5_CMD_SOC(&desc) = op->soc;
    462	CCP5_CMD_IOC(&desc) = 1;
    463	CCP5_CMD_INIT(&desc) = 0;
    464	CCP5_CMD_EOM(&desc) = 1;
    465	CCP5_CMD_PROT(&desc) = 0;
    466
    467	function.raw = 0;
    468	CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
    469	CCP5_CMD_FUNCTION(&desc) = function.raw;
    470
    471	CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
    472
    473	/* Source is from external memory */
    474	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    475	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    476	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    477
    478	/* Destination is in external memory */
    479	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
    480	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
    481	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    482
    483	/* Key (Exponent) is in external memory */
    484	CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
    485	CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
    486	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    487
    488	return ccp5_do_cmd(&desc, op->cmd_q);
    489}
    490
    491static int ccp5_perform_passthru(struct ccp_op *op)
    492{
    493	struct ccp5_desc desc;
    494	union ccp_function function;
    495	struct ccp_dma_info *saddr = &op->src.u.dma;
    496	struct ccp_dma_info *daddr = &op->dst.u.dma;
    497
    498
    499	op->cmd_q->total_pt_ops++;
    500
    501	memset(&desc, 0, Q_DESC_SIZE);
    502
    503	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
    504
    505	CCP5_CMD_SOC(&desc) = 0;
    506	CCP5_CMD_IOC(&desc) = 1;
    507	CCP5_CMD_INIT(&desc) = 0;
    508	CCP5_CMD_EOM(&desc) = op->eom;
    509	CCP5_CMD_PROT(&desc) = 0;
    510
    511	function.raw = 0;
    512	CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
    513	CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
    514	CCP5_CMD_FUNCTION(&desc) = function.raw;
    515
    516	/* Length of source data is always 256 bytes */
    517	if (op->src.type == CCP_MEMTYPE_SYSTEM)
    518		CCP5_CMD_LEN(&desc) = saddr->length;
    519	else
    520		CCP5_CMD_LEN(&desc) = daddr->length;
    521
    522	if (op->src.type == CCP_MEMTYPE_SYSTEM) {
    523		CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    524		CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    525		CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    526
    527		if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
    528			CCP5_CMD_LSB_ID(&desc) = op->sb_key;
    529	} else {
    530		u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
    531
    532		CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
    533		CCP5_CMD_SRC_HI(&desc) = 0;
    534		CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
    535	}
    536
    537	if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
    538		CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
    539		CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
    540		CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    541	} else {
    542		u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
    543
    544		CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
    545		CCP5_CMD_DST_HI(&desc) = 0;
    546		CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
    547	}
    548
    549	return ccp5_do_cmd(&desc, op->cmd_q);
    550}
    551
    552static int ccp5_perform_ecc(struct ccp_op *op)
    553{
    554	struct ccp5_desc desc;
    555	union ccp_function function;
    556
    557	op->cmd_q->total_ecc_ops++;
    558
    559	/* Zero out all the fields of the command desc */
    560	memset(&desc, 0, Q_DESC_SIZE);
    561
    562	CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
    563
    564	CCP5_CMD_SOC(&desc) = 0;
    565	CCP5_CMD_IOC(&desc) = 1;
    566	CCP5_CMD_INIT(&desc) = 0;
    567	CCP5_CMD_EOM(&desc) = 1;
    568	CCP5_CMD_PROT(&desc) = 0;
    569
    570	function.raw = 0;
    571	function.ecc.mode = op->u.ecc.function;
    572	CCP5_CMD_FUNCTION(&desc) = function.raw;
    573
    574	CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
    575
    576	CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
    577	CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
    578	CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    579
    580	CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
    581	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
    582	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
    583
    584	return ccp5_do_cmd(&desc, op->cmd_q);
    585}
    586
    587static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
    588{
    589	int q_mask = 1 << cmd_q->id;
    590	int queues = 0;
    591	int j;
    592
    593	/* Build a bit mask to know which LSBs this queue has access to.
    594	 * Don't bother with segment 0 as it has special privileges.
    595	 */
    596	for (j = 1; j < MAX_LSB_CNT; j++) {
    597		if (status & q_mask)
    598			bitmap_set(cmd_q->lsbmask, j, 1);
    599		status >>= LSB_REGION_WIDTH;
    600	}
    601	queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
    602	dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
    603		 cmd_q->id, queues);
    604
    605	return queues ? 0 : -EINVAL;
    606}
    607
    608static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
    609					int lsb_cnt, int n_lsbs,
    610					unsigned long *lsb_pub)
    611{
    612	DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
    613	int bitno;
    614	int qlsb_wgt;
    615	int i;
    616
    617	/* For each queue:
    618	 * If the count of potential LSBs available to a queue matches the
    619	 * ordinal given to us in lsb_cnt:
    620	 * Copy the mask of possible LSBs for this queue into "qlsb";
    621	 * For each bit in qlsb, see if the corresponding bit in the
    622	 * aggregation mask is set; if so, we have a match.
    623	 *     If we have a match, clear the bit in the aggregation to
    624	 *     mark it as no longer available.
    625	 *     If there is no match, clear the bit in qlsb and keep looking.
    626	 */
    627	for (i = 0; i < ccp->cmd_q_count; i++) {
    628		struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
    629
    630		qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
    631
    632		if (qlsb_wgt == lsb_cnt) {
    633			bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
    634
    635			bitno = find_first_bit(qlsb, MAX_LSB_CNT);
    636			while (bitno < MAX_LSB_CNT) {
    637				if (test_bit(bitno, lsb_pub)) {
    638					/* We found an available LSB
    639					 * that this queue can access
    640					 */
    641					cmd_q->lsb = bitno;
    642					bitmap_clear(lsb_pub, bitno, 1);
    643					dev_dbg(ccp->dev,
    644						 "Queue %d gets LSB %d\n",
    645						 i, bitno);
    646					break;
    647				}
    648				bitmap_clear(qlsb, bitno, 1);
    649				bitno = find_first_bit(qlsb, MAX_LSB_CNT);
    650			}
    651			if (bitno >= MAX_LSB_CNT)
    652				return -EINVAL;
    653			n_lsbs--;
    654		}
    655	}
    656	return n_lsbs;
    657}
    658
    659/* For each queue, from the most- to least-constrained:
    660 * find an LSB that can be assigned to the queue. If there are N queues that
    661 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
    662 * dedicated LSB. Remaining LSB regions become a shared resource.
    663 * If we have fewer LSBs than queues, all LSB regions become shared resources.
    664 */
    665static int ccp_assign_lsbs(struct ccp_device *ccp)
    666{
    667	DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
    668	DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
    669	int n_lsbs = 0;
    670	int bitno;
    671	int i, lsb_cnt;
    672	int rc = 0;
    673
    674	bitmap_zero(lsb_pub, MAX_LSB_CNT);
    675
    676	/* Create an aggregate bitmap to get a total count of available LSBs */
    677	for (i = 0; i < ccp->cmd_q_count; i++)
    678		bitmap_or(lsb_pub,
    679			  lsb_pub, ccp->cmd_q[i].lsbmask,
    680			  MAX_LSB_CNT);
    681
    682	n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
    683
    684	if (n_lsbs >= ccp->cmd_q_count) {
    685		/* We have enough LSBS to give every queue a private LSB.
    686		 * Brute force search to start with the queues that are more
    687		 * constrained in LSB choice. When an LSB is privately
    688		 * assigned, it is removed from the public mask.
    689		 * This is an ugly N squared algorithm with some optimization.
    690		 */
    691		for (lsb_cnt = 1;
    692		     n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
    693		     lsb_cnt++) {
    694			rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
    695							  lsb_pub);
    696			if (rc < 0)
    697				return -EINVAL;
    698			n_lsbs = rc;
    699		}
    700	}
    701
    702	rc = 0;
    703	/* What's left of the LSBs, according to the public mask, now become
    704	 * shared. Any zero bits in the lsb_pub mask represent an LSB region
    705	 * that can't be used as a shared resource, so mark the LSB slots for
    706	 * them as "in use".
    707	 */
    708	bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
    709
    710	bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
    711	while (bitno < MAX_LSB_CNT) {
    712		bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
    713		bitmap_set(qlsb, bitno, 1);
    714		bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
    715	}
    716
    717	return rc;
    718}
    719
    720static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
    721{
    722	unsigned int i;
    723
    724	for (i = 0; i < ccp->cmd_q_count; i++)
    725		iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
    726}
    727
    728static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
    729{
    730	unsigned int i;
    731
    732	for (i = 0; i < ccp->cmd_q_count; i++)
    733		iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
    734}
    735
    736static void ccp5_irq_bh(unsigned long data)
    737{
    738	struct ccp_device *ccp = (struct ccp_device *)data;
    739	u32 status;
    740	unsigned int i;
    741
    742	for (i = 0; i < ccp->cmd_q_count; i++) {
    743		struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
    744
    745		status = ioread32(cmd_q->reg_interrupt_status);
    746
    747		if (status) {
    748			cmd_q->int_status = status;
    749			cmd_q->q_status = ioread32(cmd_q->reg_status);
    750			cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
    751
    752			/* On error, only save the first error value */
    753			if ((status & INT_ERROR) && !cmd_q->cmd_error)
    754				cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
    755
    756			cmd_q->int_rcvd = 1;
    757
    758			/* Acknowledge the interrupt and wake the kthread */
    759			iowrite32(status, cmd_q->reg_interrupt_status);
    760			wake_up_interruptible(&cmd_q->int_queue);
    761		}
    762	}
    763	ccp5_enable_queue_interrupts(ccp);
    764}
    765
    766static irqreturn_t ccp5_irq_handler(int irq, void *data)
    767{
    768	struct ccp_device *ccp = (struct ccp_device *)data;
    769
    770	ccp5_disable_queue_interrupts(ccp);
    771	ccp->total_interrupts++;
    772	if (ccp->use_tasklet)
    773		tasklet_schedule(&ccp->irq_tasklet);
    774	else
    775		ccp5_irq_bh((unsigned long)ccp);
    776	return IRQ_HANDLED;
    777}
    778
    779static int ccp5_init(struct ccp_device *ccp)
    780{
    781	struct device *dev = ccp->dev;
    782	struct ccp_cmd_queue *cmd_q;
    783	struct dma_pool *dma_pool;
    784	char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
    785	unsigned int qmr, i;
    786	u64 status;
    787	u32 status_lo, status_hi;
    788	int ret;
    789
    790	/* Find available queues */
    791	qmr = ioread32(ccp->io_regs + Q_MASK_REG);
    792	/*
    793	 * Check for a access to the registers.  If this read returns
    794	 * 0xffffffff, it's likely that the system is running a broken
    795	 * BIOS which disallows access to the device. Stop here and fail
    796	 * the initialization (but not the load, as the PSP could get
    797	 * properly initialized).
    798	 */
    799	if (qmr == 0xffffffff) {
    800		dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n");
    801		return 1;
    802	}
    803
    804	for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
    805		if (!(qmr & (1 << i)))
    806			continue;
    807
    808		/* Allocate a dma pool for this queue */
    809		snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
    810			 ccp->name, i);
    811		dma_pool = dma_pool_create(dma_pool_name, dev,
    812					   CCP_DMAPOOL_MAX_SIZE,
    813					   CCP_DMAPOOL_ALIGN, 0);
    814		if (!dma_pool) {
    815			dev_err(dev, "unable to allocate dma pool\n");
    816			ret = -ENOMEM;
    817			goto e_pool;
    818		}
    819
    820		cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
    821		ccp->cmd_q_count++;
    822
    823		cmd_q->ccp = ccp;
    824		cmd_q->id = i;
    825		cmd_q->dma_pool = dma_pool;
    826		mutex_init(&cmd_q->q_mutex);
    827
    828		/* Page alignment satisfies our needs for N <= 128 */
    829		BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
    830		cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
    831		cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,
    832						   &cmd_q->qbase_dma,
    833						   GFP_KERNEL);
    834		if (!cmd_q->qbase) {
    835			dev_err(dev, "unable to allocate command queue\n");
    836			ret = -ENOMEM;
    837			goto e_pool;
    838		}
    839
    840		cmd_q->qidx = 0;
    841		/* Preset some register values and masks that are queue
    842		 * number dependent
    843		 */
    844		cmd_q->reg_control = ccp->io_regs +
    845				     CMD5_Q_STATUS_INCR * (i + 1);
    846		cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
    847		cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
    848		cmd_q->reg_int_enable = cmd_q->reg_control +
    849					CMD5_Q_INT_ENABLE_BASE;
    850		cmd_q->reg_interrupt_status = cmd_q->reg_control +
    851					      CMD5_Q_INTERRUPT_STATUS_BASE;
    852		cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
    853		cmd_q->reg_int_status = cmd_q->reg_control +
    854					CMD5_Q_INT_STATUS_BASE;
    855		cmd_q->reg_dma_status = cmd_q->reg_control +
    856					CMD5_Q_DMA_STATUS_BASE;
    857		cmd_q->reg_dma_read_status = cmd_q->reg_control +
    858					     CMD5_Q_DMA_READ_STATUS_BASE;
    859		cmd_q->reg_dma_write_status = cmd_q->reg_control +
    860					      CMD5_Q_DMA_WRITE_STATUS_BASE;
    861
    862		init_waitqueue_head(&cmd_q->int_queue);
    863
    864		dev_dbg(dev, "queue #%u available\n", i);
    865	}
    866
    867	if (ccp->cmd_q_count == 0) {
    868		dev_notice(dev, "no command queues available\n");
    869		ret = 1;
    870		goto e_pool;
    871	}
    872
    873	/* Turn off the queues and disable interrupts until ready */
    874	ccp5_disable_queue_interrupts(ccp);
    875	for (i = 0; i < ccp->cmd_q_count; i++) {
    876		cmd_q = &ccp->cmd_q[i];
    877
    878		cmd_q->qcontrol = 0; /* Start with nothing */
    879		iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
    880
    881		ioread32(cmd_q->reg_int_status);
    882		ioread32(cmd_q->reg_status);
    883
    884		/* Clear the interrupt status */
    885		iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
    886	}
    887
    888	dev_dbg(dev, "Requesting an IRQ...\n");
    889	/* Request an irq */
    890	ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
    891	if (ret) {
    892		dev_err(dev, "unable to allocate an IRQ\n");
    893		goto e_pool;
    894	}
    895	/* Initialize the ISR tasklet */
    896	if (ccp->use_tasklet)
    897		tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
    898			     (unsigned long)ccp);
    899
    900	dev_dbg(dev, "Loading LSB map...\n");
    901	/* Copy the private LSB mask to the public registers */
    902	status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
    903	status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
    904	iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
    905	iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
    906	status = ((u64)status_hi<<30) | (u64)status_lo;
    907
    908	dev_dbg(dev, "Configuring virtual queues...\n");
    909	/* Configure size of each virtual queue accessible to host */
    910	for (i = 0; i < ccp->cmd_q_count; i++) {
    911		u32 dma_addr_lo;
    912		u32 dma_addr_hi;
    913
    914		cmd_q = &ccp->cmd_q[i];
    915
    916		cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
    917		cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
    918
    919		cmd_q->qdma_tail = cmd_q->qbase_dma;
    920		dma_addr_lo = low_address(cmd_q->qdma_tail);
    921		iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
    922		iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
    923
    924		dma_addr_hi = high_address(cmd_q->qdma_tail);
    925		cmd_q->qcontrol |= (dma_addr_hi << 16);
    926		iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
    927
    928		/* Find the LSB regions accessible to the queue */
    929		ccp_find_lsb_regions(cmd_q, status);
    930		cmd_q->lsb = -1; /* Unassigned value */
    931	}
    932
    933	dev_dbg(dev, "Assigning LSBs...\n");
    934	ret = ccp_assign_lsbs(ccp);
    935	if (ret) {
    936		dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
    937		goto e_irq;
    938	}
    939
    940	/* Optimization: pre-allocate LSB slots for each queue */
    941	for (i = 0; i < ccp->cmd_q_count; i++) {
    942		ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
    943		ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
    944	}
    945
    946	dev_dbg(dev, "Starting threads...\n");
    947	/* Create a kthread for each queue */
    948	for (i = 0; i < ccp->cmd_q_count; i++) {
    949		struct task_struct *kthread;
    950
    951		cmd_q = &ccp->cmd_q[i];
    952
    953		kthread = kthread_run(ccp_cmd_queue_thread, cmd_q,
    954				      "%s-q%u", ccp->name, cmd_q->id);
    955		if (IS_ERR(kthread)) {
    956			dev_err(dev, "error creating queue thread (%ld)\n",
    957				PTR_ERR(kthread));
    958			ret = PTR_ERR(kthread);
    959			goto e_kthread;
    960		}
    961
    962		cmd_q->kthread = kthread;
    963	}
    964
    965	dev_dbg(dev, "Enabling interrupts...\n");
    966	ccp5_enable_queue_interrupts(ccp);
    967
    968	dev_dbg(dev, "Registering device...\n");
    969	/* Put this on the unit list to make it available */
    970	ccp_add_device(ccp);
    971
    972	ret = ccp_register_rng(ccp);
    973	if (ret)
    974		goto e_kthread;
    975
    976	/* Register the DMA engine support */
    977	ret = ccp_dmaengine_register(ccp);
    978	if (ret)
    979		goto e_hwrng;
    980
    981#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
    982	/* Set up debugfs entries */
    983	ccp5_debugfs_setup(ccp);
    984#endif
    985
    986	return 0;
    987
    988e_hwrng:
    989	ccp_unregister_rng(ccp);
    990
    991e_kthread:
    992	for (i = 0; i < ccp->cmd_q_count; i++)
    993		if (ccp->cmd_q[i].kthread)
    994			kthread_stop(ccp->cmd_q[i].kthread);
    995
    996e_irq:
    997	sp_free_ccp_irq(ccp->sp, ccp);
    998
    999e_pool:
   1000	for (i = 0; i < ccp->cmd_q_count; i++)
   1001		dma_pool_destroy(ccp->cmd_q[i].dma_pool);
   1002
   1003	return ret;
   1004}
   1005
   1006static void ccp5_destroy(struct ccp_device *ccp)
   1007{
   1008	struct ccp_cmd_queue *cmd_q;
   1009	struct ccp_cmd *cmd;
   1010	unsigned int i;
   1011
   1012	/* Unregister the DMA engine */
   1013	ccp_dmaengine_unregister(ccp);
   1014
   1015	/* Unregister the RNG */
   1016	ccp_unregister_rng(ccp);
   1017
   1018	/* Remove this device from the list of available units first */
   1019	ccp_del_device(ccp);
   1020
   1021#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
   1022	/* We're in the process of tearing down the entire driver;
   1023	 * when all the devices are gone clean up debugfs
   1024	 */
   1025	if (ccp_present())
   1026		ccp5_debugfs_destroy();
   1027#endif
   1028
   1029	/* Disable and clear interrupts */
   1030	ccp5_disable_queue_interrupts(ccp);
   1031	for (i = 0; i < ccp->cmd_q_count; i++) {
   1032		cmd_q = &ccp->cmd_q[i];
   1033
   1034		/* Turn off the run bit */
   1035		iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
   1036
   1037		/* Clear the interrupt status */
   1038		iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
   1039		ioread32(cmd_q->reg_int_status);
   1040		ioread32(cmd_q->reg_status);
   1041	}
   1042
   1043	/* Stop the queue kthreads */
   1044	for (i = 0; i < ccp->cmd_q_count; i++)
   1045		if (ccp->cmd_q[i].kthread)
   1046			kthread_stop(ccp->cmd_q[i].kthread);
   1047
   1048	sp_free_ccp_irq(ccp->sp, ccp);
   1049
   1050	/* Flush the cmd and backlog queue */
   1051	while (!list_empty(&ccp->cmd)) {
   1052		/* Invoke the callback directly with an error code */
   1053		cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
   1054		list_del(&cmd->entry);
   1055		cmd->callback(cmd->data, -ENODEV);
   1056	}
   1057	while (!list_empty(&ccp->backlog)) {
   1058		/* Invoke the callback directly with an error code */
   1059		cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
   1060		list_del(&cmd->entry);
   1061		cmd->callback(cmd->data, -ENODEV);
   1062	}
   1063}
   1064
   1065static void ccp5_config(struct ccp_device *ccp)
   1066{
   1067	/* Public side */
   1068	iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
   1069}
   1070
   1071static void ccp5other_config(struct ccp_device *ccp)
   1072{
   1073	int i;
   1074	u32 rnd;
   1075
   1076	/* We own all of the queues on the NTB CCP */
   1077
   1078	iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
   1079	iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
   1080	for (i = 0; i < 12; i++) {
   1081		rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
   1082		iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
   1083	}
   1084
   1085	iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
   1086	iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
   1087	iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
   1088
   1089	iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
   1090	iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
   1091
   1092	iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
   1093
   1094	ccp5_config(ccp);
   1095}
   1096
   1097/* Version 5 adds some function, but is essentially the same as v5 */
   1098static const struct ccp_actions ccp5_actions = {
   1099	.aes = ccp5_perform_aes,
   1100	.xts_aes = ccp5_perform_xts_aes,
   1101	.sha = ccp5_perform_sha,
   1102	.des3 = ccp5_perform_des3,
   1103	.rsa = ccp5_perform_rsa,
   1104	.passthru = ccp5_perform_passthru,
   1105	.ecc = ccp5_perform_ecc,
   1106	.sballoc = ccp_lsb_alloc,
   1107	.sbfree = ccp_lsb_free,
   1108	.init = ccp5_init,
   1109	.destroy = ccp5_destroy,
   1110	.get_free_slots = ccp5_get_free_slots,
   1111};
   1112
   1113const struct ccp_vdata ccpv5a = {
   1114	.version = CCP_VERSION(5, 0),
   1115	.setup = ccp5_config,
   1116	.perform = &ccp5_actions,
   1117	.offset = 0x0,
   1118	.rsamax = CCP5_RSA_MAX_WIDTH,
   1119};
   1120
   1121const struct ccp_vdata ccpv5b = {
   1122	.version = CCP_VERSION(5, 0),
   1123	.dma_chan_attr = DMA_PRIVATE,
   1124	.setup = ccp5other_config,
   1125	.perform = &ccp5_actions,
   1126	.offset = 0x0,
   1127	.rsamax = CCP5_RSA_MAX_WIDTH,
   1128};