cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

caamalg.c (98615B)


      1// SPDX-License-Identifier: GPL-2.0+
      2/*
      3 * caam - Freescale FSL CAAM support for crypto API
      4 *
      5 * Copyright 2008-2011 Freescale Semiconductor, Inc.
      6 * Copyright 2016-2019 NXP
      7 *
      8 * Based on talitos crypto API driver.
      9 *
     10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
     11 *
     12 * ---------------                     ---------------
     13 * | JobDesc #1  |-------------------->|  ShareDesc  |
     14 * | *(packet 1) |                     |   (PDB)     |
     15 * ---------------      |------------->|  (hashKey)  |
     16 *       .              |              | (cipherKey) |
     17 *       .              |    |-------->| (operation) |
     18 * ---------------      |    |         ---------------
     19 * | JobDesc #2  |------|    |
     20 * | *(packet 2) |           |
     21 * ---------------           |
     22 *       .                   |
     23 *       .                   |
     24 * ---------------           |
     25 * | JobDesc #3  |------------
     26 * | *(packet 3) |
     27 * ---------------
     28 *
     29 * The SharedDesc never changes for a connection unless rekeyed, but
     30 * each packet will likely be in a different place. So all we need
     31 * to know to process the packet is where the input is, where the
     32 * output goes, and what context we want to process with. Context is
     33 * in the SharedDesc, packet references in the JobDesc.
     34 *
     35 * So, a job desc looks like:
     36 *
     37 * ---------------------
     38 * | Header            |
     39 * | ShareDesc Pointer |
     40 * | SEQ_OUT_PTR       |
     41 * | (output buffer)   |
     42 * | (output length)   |
     43 * | SEQ_IN_PTR        |
     44 * | (input buffer)    |
     45 * | (input length)    |
     46 * ---------------------
     47 */
     48
     49#include "compat.h"
     50
     51#include "regs.h"
     52#include "intern.h"
     53#include "desc_constr.h"
     54#include "jr.h"
     55#include "error.h"
     56#include "sg_sw_sec4.h"
     57#include "key_gen.h"
     58#include "caamalg_desc.h"
     59#include <crypto/engine.h>
     60#include <crypto/xts.h>
     61#include <asm/unaligned.h>
     62
     63/*
     64 * crypto alg
     65 */
     66#define CAAM_CRA_PRIORITY		3000
     67/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
     68#define CAAM_MAX_KEY_SIZE		(AES_MAX_KEY_SIZE + \
     69					 CTR_RFC3686_NONCE_SIZE + \
     70					 SHA512_DIGEST_SIZE * 2)
     71
     72#define AEAD_DESC_JOB_IO_LEN		(DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
     73#define GCM_DESC_JOB_IO_LEN		(AEAD_DESC_JOB_IO_LEN + \
     74					 CAAM_CMD_SZ * 4)
     75#define AUTHENC_DESC_JOB_IO_LEN		(AEAD_DESC_JOB_IO_LEN + \
     76					 CAAM_CMD_SZ * 5)
     77
     78#define CHACHAPOLY_DESC_JOB_IO_LEN	(AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
     79
     80#define DESC_MAX_USED_BYTES		(CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
     81#define DESC_MAX_USED_LEN		(DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
     82
     83struct caam_alg_entry {
     84	int class1_alg_type;
     85	int class2_alg_type;
     86	bool rfc3686;
     87	bool geniv;
     88	bool nodkp;
     89};
     90
     91struct caam_aead_alg {
     92	struct aead_alg aead;
     93	struct caam_alg_entry caam;
     94	bool registered;
     95};
     96
     97struct caam_skcipher_alg {
     98	struct skcipher_alg skcipher;
     99	struct caam_alg_entry caam;
    100	bool registered;
    101};
    102
    103/*
    104 * per-session context
    105 */
    106struct caam_ctx {
    107	struct crypto_engine_ctx enginectx;
    108	u32 sh_desc_enc[DESC_MAX_USED_LEN];
    109	u32 sh_desc_dec[DESC_MAX_USED_LEN];
    110	u8 key[CAAM_MAX_KEY_SIZE];
    111	dma_addr_t sh_desc_enc_dma;
    112	dma_addr_t sh_desc_dec_dma;
    113	dma_addr_t key_dma;
    114	enum dma_data_direction dir;
    115	struct device *jrdev;
    116	struct alginfo adata;
    117	struct alginfo cdata;
    118	unsigned int authsize;
    119	bool xts_key_fallback;
    120	struct crypto_skcipher *fallback;
    121};
    122
    123struct caam_skcipher_req_ctx {
    124	struct skcipher_edesc *edesc;
    125	struct skcipher_request fallback_req;
    126};
    127
    128struct caam_aead_req_ctx {
    129	struct aead_edesc *edesc;
    130};
    131
    132static int aead_null_set_sh_desc(struct crypto_aead *aead)
    133{
    134	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    135	struct device *jrdev = ctx->jrdev;
    136	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
    137	u32 *desc;
    138	int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
    139			ctx->adata.keylen_pad;
    140
    141	/*
    142	 * Job Descriptor and Shared Descriptors
    143	 * must all fit into the 64-word Descriptor h/w Buffer
    144	 */
    145	if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
    146		ctx->adata.key_inline = true;
    147		ctx->adata.key_virt = ctx->key;
    148	} else {
    149		ctx->adata.key_inline = false;
    150		ctx->adata.key_dma = ctx->key_dma;
    151	}
    152
    153	/* aead_encrypt shared descriptor */
    154	desc = ctx->sh_desc_enc;
    155	cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
    156				    ctrlpriv->era);
    157	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    158				   desc_bytes(desc), ctx->dir);
    159
    160	/*
    161	 * Job Descriptor and Shared Descriptors
    162	 * must all fit into the 64-word Descriptor h/w Buffer
    163	 */
    164	if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
    165		ctx->adata.key_inline = true;
    166		ctx->adata.key_virt = ctx->key;
    167	} else {
    168		ctx->adata.key_inline = false;
    169		ctx->adata.key_dma = ctx->key_dma;
    170	}
    171
    172	/* aead_decrypt shared descriptor */
    173	desc = ctx->sh_desc_dec;
    174	cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
    175				    ctrlpriv->era);
    176	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    177				   desc_bytes(desc), ctx->dir);
    178
    179	return 0;
    180}
    181
    182static int aead_set_sh_desc(struct crypto_aead *aead)
    183{
    184	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
    185						 struct caam_aead_alg, aead);
    186	unsigned int ivsize = crypto_aead_ivsize(aead);
    187	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    188	struct device *jrdev = ctx->jrdev;
    189	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
    190	u32 ctx1_iv_off = 0;
    191	u32 *desc, *nonce = NULL;
    192	u32 inl_mask;
    193	unsigned int data_len[2];
    194	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
    195			       OP_ALG_AAI_CTR_MOD128);
    196	const bool is_rfc3686 = alg->caam.rfc3686;
    197
    198	if (!ctx->authsize)
    199		return 0;
    200
    201	/* NULL encryption / decryption */
    202	if (!ctx->cdata.keylen)
    203		return aead_null_set_sh_desc(aead);
    204
    205	/*
    206	 * AES-CTR needs to load IV in CONTEXT1 reg
    207	 * at an offset of 128bits (16bytes)
    208	 * CONTEXT1[255:128] = IV
    209	 */
    210	if (ctr_mode)
    211		ctx1_iv_off = 16;
    212
    213	/*
    214	 * RFC3686 specific:
    215	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
    216	 */
    217	if (is_rfc3686) {
    218		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
    219		nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
    220				ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
    221	}
    222
    223	/*
    224	 * In case |user key| > |derived key|, using DKP<imm,imm>
    225	 * would result in invalid opcodes (last bytes of user key) in
    226	 * the resulting descriptor. Use DKP<ptr,imm> instead => both
    227	 * virtual and dma key addresses are needed.
    228	 */
    229	ctx->adata.key_virt = ctx->key;
    230	ctx->adata.key_dma = ctx->key_dma;
    231
    232	ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
    233	ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
    234
    235	data_len[0] = ctx->adata.keylen_pad;
    236	data_len[1] = ctx->cdata.keylen;
    237
    238	if (alg->caam.geniv)
    239		goto skip_enc;
    240
    241	/*
    242	 * Job Descriptor and Shared Descriptors
    243	 * must all fit into the 64-word Descriptor h/w Buffer
    244	 */
    245	if (desc_inline_query(DESC_AEAD_ENC_LEN +
    246			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
    247			      AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
    248			      ARRAY_SIZE(data_len)) < 0)
    249		return -EINVAL;
    250
    251	ctx->adata.key_inline = !!(inl_mask & 1);
    252	ctx->cdata.key_inline = !!(inl_mask & 2);
    253
    254	/* aead_encrypt shared descriptor */
    255	desc = ctx->sh_desc_enc;
    256	cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
    257			       ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
    258			       false, ctrlpriv->era);
    259	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    260				   desc_bytes(desc), ctx->dir);
    261
    262skip_enc:
    263	/*
    264	 * Job Descriptor and Shared Descriptors
    265	 * must all fit into the 64-word Descriptor h/w Buffer
    266	 */
    267	if (desc_inline_query(DESC_AEAD_DEC_LEN +
    268			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
    269			      AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
    270			      ARRAY_SIZE(data_len)) < 0)
    271		return -EINVAL;
    272
    273	ctx->adata.key_inline = !!(inl_mask & 1);
    274	ctx->cdata.key_inline = !!(inl_mask & 2);
    275
    276	/* aead_decrypt shared descriptor */
    277	desc = ctx->sh_desc_dec;
    278	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
    279			       ctx->authsize, alg->caam.geniv, is_rfc3686,
    280			       nonce, ctx1_iv_off, false, ctrlpriv->era);
    281	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    282				   desc_bytes(desc), ctx->dir);
    283
    284	if (!alg->caam.geniv)
    285		goto skip_givenc;
    286
    287	/*
    288	 * Job Descriptor and Shared Descriptors
    289	 * must all fit into the 64-word Descriptor h/w Buffer
    290	 */
    291	if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
    292			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
    293			      AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
    294			      ARRAY_SIZE(data_len)) < 0)
    295		return -EINVAL;
    296
    297	ctx->adata.key_inline = !!(inl_mask & 1);
    298	ctx->cdata.key_inline = !!(inl_mask & 2);
    299
    300	/* aead_givencrypt shared descriptor */
    301	desc = ctx->sh_desc_enc;
    302	cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
    303				  ctx->authsize, is_rfc3686, nonce,
    304				  ctx1_iv_off, false, ctrlpriv->era);
    305	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    306				   desc_bytes(desc), ctx->dir);
    307
    308skip_givenc:
    309	return 0;
    310}
    311
    312static int aead_setauthsize(struct crypto_aead *authenc,
    313				    unsigned int authsize)
    314{
    315	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
    316
    317	ctx->authsize = authsize;
    318	aead_set_sh_desc(authenc);
    319
    320	return 0;
    321}
    322
    323static int gcm_set_sh_desc(struct crypto_aead *aead)
    324{
    325	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    326	struct device *jrdev = ctx->jrdev;
    327	unsigned int ivsize = crypto_aead_ivsize(aead);
    328	u32 *desc;
    329	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
    330			ctx->cdata.keylen;
    331
    332	if (!ctx->cdata.keylen || !ctx->authsize)
    333		return 0;
    334
    335	/*
    336	 * AES GCM encrypt shared descriptor
    337	 * Job Descriptor and Shared Descriptor
    338	 * must fit into the 64-word Descriptor h/w Buffer
    339	 */
    340	if (rem_bytes >= DESC_GCM_ENC_LEN) {
    341		ctx->cdata.key_inline = true;
    342		ctx->cdata.key_virt = ctx->key;
    343	} else {
    344		ctx->cdata.key_inline = false;
    345		ctx->cdata.key_dma = ctx->key_dma;
    346	}
    347
    348	desc = ctx->sh_desc_enc;
    349	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
    350	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    351				   desc_bytes(desc), ctx->dir);
    352
    353	/*
    354	 * Job Descriptor and Shared Descriptors
    355	 * must all fit into the 64-word Descriptor h/w Buffer
    356	 */
    357	if (rem_bytes >= DESC_GCM_DEC_LEN) {
    358		ctx->cdata.key_inline = true;
    359		ctx->cdata.key_virt = ctx->key;
    360	} else {
    361		ctx->cdata.key_inline = false;
    362		ctx->cdata.key_dma = ctx->key_dma;
    363	}
    364
    365	desc = ctx->sh_desc_dec;
    366	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
    367	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    368				   desc_bytes(desc), ctx->dir);
    369
    370	return 0;
    371}
    372
    373static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
    374{
    375	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
    376	int err;
    377
    378	err = crypto_gcm_check_authsize(authsize);
    379	if (err)
    380		return err;
    381
    382	ctx->authsize = authsize;
    383	gcm_set_sh_desc(authenc);
    384
    385	return 0;
    386}
    387
    388static int rfc4106_set_sh_desc(struct crypto_aead *aead)
    389{
    390	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    391	struct device *jrdev = ctx->jrdev;
    392	unsigned int ivsize = crypto_aead_ivsize(aead);
    393	u32 *desc;
    394	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
    395			ctx->cdata.keylen;
    396
    397	if (!ctx->cdata.keylen || !ctx->authsize)
    398		return 0;
    399
    400	/*
    401	 * RFC4106 encrypt shared descriptor
    402	 * Job Descriptor and Shared Descriptor
    403	 * must fit into the 64-word Descriptor h/w Buffer
    404	 */
    405	if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
    406		ctx->cdata.key_inline = true;
    407		ctx->cdata.key_virt = ctx->key;
    408	} else {
    409		ctx->cdata.key_inline = false;
    410		ctx->cdata.key_dma = ctx->key_dma;
    411	}
    412
    413	desc = ctx->sh_desc_enc;
    414	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
    415				  false);
    416	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    417				   desc_bytes(desc), ctx->dir);
    418
    419	/*
    420	 * Job Descriptor and Shared Descriptors
    421	 * must all fit into the 64-word Descriptor h/w Buffer
    422	 */
    423	if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
    424		ctx->cdata.key_inline = true;
    425		ctx->cdata.key_virt = ctx->key;
    426	} else {
    427		ctx->cdata.key_inline = false;
    428		ctx->cdata.key_dma = ctx->key_dma;
    429	}
    430
    431	desc = ctx->sh_desc_dec;
    432	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
    433				  false);
    434	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    435				   desc_bytes(desc), ctx->dir);
    436
    437	return 0;
    438}
    439
    440static int rfc4106_setauthsize(struct crypto_aead *authenc,
    441			       unsigned int authsize)
    442{
    443	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
    444	int err;
    445
    446	err = crypto_rfc4106_check_authsize(authsize);
    447	if (err)
    448		return err;
    449
    450	ctx->authsize = authsize;
    451	rfc4106_set_sh_desc(authenc);
    452
    453	return 0;
    454}
    455
    456static int rfc4543_set_sh_desc(struct crypto_aead *aead)
    457{
    458	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    459	struct device *jrdev = ctx->jrdev;
    460	unsigned int ivsize = crypto_aead_ivsize(aead);
    461	u32 *desc;
    462	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
    463			ctx->cdata.keylen;
    464
    465	if (!ctx->cdata.keylen || !ctx->authsize)
    466		return 0;
    467
    468	/*
    469	 * RFC4543 encrypt shared descriptor
    470	 * Job Descriptor and Shared Descriptor
    471	 * must fit into the 64-word Descriptor h/w Buffer
    472	 */
    473	if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
    474		ctx->cdata.key_inline = true;
    475		ctx->cdata.key_virt = ctx->key;
    476	} else {
    477		ctx->cdata.key_inline = false;
    478		ctx->cdata.key_dma = ctx->key_dma;
    479	}
    480
    481	desc = ctx->sh_desc_enc;
    482	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
    483				  false);
    484	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    485				   desc_bytes(desc), ctx->dir);
    486
    487	/*
    488	 * Job Descriptor and Shared Descriptors
    489	 * must all fit into the 64-word Descriptor h/w Buffer
    490	 */
    491	if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
    492		ctx->cdata.key_inline = true;
    493		ctx->cdata.key_virt = ctx->key;
    494	} else {
    495		ctx->cdata.key_inline = false;
    496		ctx->cdata.key_dma = ctx->key_dma;
    497	}
    498
    499	desc = ctx->sh_desc_dec;
    500	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
    501				  false);
    502	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    503				   desc_bytes(desc), ctx->dir);
    504
    505	return 0;
    506}
    507
    508static int rfc4543_setauthsize(struct crypto_aead *authenc,
    509			       unsigned int authsize)
    510{
    511	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
    512
    513	if (authsize != 16)
    514		return -EINVAL;
    515
    516	ctx->authsize = authsize;
    517	rfc4543_set_sh_desc(authenc);
    518
    519	return 0;
    520}
    521
    522static int chachapoly_set_sh_desc(struct crypto_aead *aead)
    523{
    524	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    525	struct device *jrdev = ctx->jrdev;
    526	unsigned int ivsize = crypto_aead_ivsize(aead);
    527	u32 *desc;
    528
    529	if (!ctx->cdata.keylen || !ctx->authsize)
    530		return 0;
    531
    532	desc = ctx->sh_desc_enc;
    533	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
    534			       ctx->authsize, true, false);
    535	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    536				   desc_bytes(desc), ctx->dir);
    537
    538	desc = ctx->sh_desc_dec;
    539	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
    540			       ctx->authsize, false, false);
    541	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    542				   desc_bytes(desc), ctx->dir);
    543
    544	return 0;
    545}
    546
    547static int chachapoly_setauthsize(struct crypto_aead *aead,
    548				  unsigned int authsize)
    549{
    550	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    551
    552	if (authsize != POLY1305_DIGEST_SIZE)
    553		return -EINVAL;
    554
    555	ctx->authsize = authsize;
    556	return chachapoly_set_sh_desc(aead);
    557}
    558
    559static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
    560			     unsigned int keylen)
    561{
    562	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    563	unsigned int ivsize = crypto_aead_ivsize(aead);
    564	unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
    565
    566	if (keylen != CHACHA_KEY_SIZE + saltlen)
    567		return -EINVAL;
    568
    569	ctx->cdata.key_virt = key;
    570	ctx->cdata.keylen = keylen - saltlen;
    571
    572	return chachapoly_set_sh_desc(aead);
    573}
    574
    575static int aead_setkey(struct crypto_aead *aead,
    576			       const u8 *key, unsigned int keylen)
    577{
    578	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    579	struct device *jrdev = ctx->jrdev;
    580	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
    581	struct crypto_authenc_keys keys;
    582	int ret = 0;
    583
    584	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
    585		goto badkey;
    586
    587	dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
    588	       keys.authkeylen + keys.enckeylen, keys.enckeylen,
    589	       keys.authkeylen);
    590	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
    591			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
    592
    593	/*
    594	 * If DKP is supported, use it in the shared descriptor to generate
    595	 * the split key.
    596	 */
    597	if (ctrlpriv->era >= 6) {
    598		ctx->adata.keylen = keys.authkeylen;
    599		ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
    600						      OP_ALG_ALGSEL_MASK);
    601
    602		if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
    603			goto badkey;
    604
    605		memcpy(ctx->key, keys.authkey, keys.authkeylen);
    606		memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
    607		       keys.enckeylen);
    608		dma_sync_single_for_device(jrdev, ctx->key_dma,
    609					   ctx->adata.keylen_pad +
    610					   keys.enckeylen, ctx->dir);
    611		goto skip_split_key;
    612	}
    613
    614	ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
    615			    keys.authkeylen, CAAM_MAX_KEY_SIZE -
    616			    keys.enckeylen);
    617	if (ret) {
    618		goto badkey;
    619	}
    620
    621	/* postpend encryption key to auth split key */
    622	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
    623	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
    624				   keys.enckeylen, ctx->dir);
    625
    626	print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
    627			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
    628			     ctx->adata.keylen_pad + keys.enckeylen, 1);
    629
    630skip_split_key:
    631	ctx->cdata.keylen = keys.enckeylen;
    632	memzero_explicit(&keys, sizeof(keys));
    633	return aead_set_sh_desc(aead);
    634badkey:
    635	memzero_explicit(&keys, sizeof(keys));
    636	return -EINVAL;
    637}
    638
    639static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
    640			    unsigned int keylen)
    641{
    642	struct crypto_authenc_keys keys;
    643	int err;
    644
    645	err = crypto_authenc_extractkeys(&keys, key, keylen);
    646	if (unlikely(err))
    647		return err;
    648
    649	err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
    650	      aead_setkey(aead, key, keylen);
    651
    652	memzero_explicit(&keys, sizeof(keys));
    653	return err;
    654}
    655
    656static int gcm_setkey(struct crypto_aead *aead,
    657		      const u8 *key, unsigned int keylen)
    658{
    659	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    660	struct device *jrdev = ctx->jrdev;
    661	int err;
    662
    663	err = aes_check_keylen(keylen);
    664	if (err)
    665		return err;
    666
    667	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
    668			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
    669
    670	memcpy(ctx->key, key, keylen);
    671	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
    672	ctx->cdata.keylen = keylen;
    673
    674	return gcm_set_sh_desc(aead);
    675}
    676
    677static int rfc4106_setkey(struct crypto_aead *aead,
    678			  const u8 *key, unsigned int keylen)
    679{
    680	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    681	struct device *jrdev = ctx->jrdev;
    682	int err;
    683
    684	err = aes_check_keylen(keylen - 4);
    685	if (err)
    686		return err;
    687
    688	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
    689			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
    690
    691	memcpy(ctx->key, key, keylen);
    692
    693	/*
    694	 * The last four bytes of the key material are used as the salt value
    695	 * in the nonce. Update the AES key length.
    696	 */
    697	ctx->cdata.keylen = keylen - 4;
    698	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
    699				   ctx->dir);
    700	return rfc4106_set_sh_desc(aead);
    701}
    702
    703static int rfc4543_setkey(struct crypto_aead *aead,
    704			  const u8 *key, unsigned int keylen)
    705{
    706	struct caam_ctx *ctx = crypto_aead_ctx(aead);
    707	struct device *jrdev = ctx->jrdev;
    708	int err;
    709
    710	err = aes_check_keylen(keylen - 4);
    711	if (err)
    712		return err;
    713
    714	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
    715			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
    716
    717	memcpy(ctx->key, key, keylen);
    718
    719	/*
    720	 * The last four bytes of the key material are used as the salt value
    721	 * in the nonce. Update the AES key length.
    722	 */
    723	ctx->cdata.keylen = keylen - 4;
    724	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
    725				   ctx->dir);
    726	return rfc4543_set_sh_desc(aead);
    727}
    728
    729static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
    730			   unsigned int keylen, const u32 ctx1_iv_off)
    731{
    732	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
    733	struct caam_skcipher_alg *alg =
    734		container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
    735			     skcipher);
    736	struct device *jrdev = ctx->jrdev;
    737	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
    738	u32 *desc;
    739	const bool is_rfc3686 = alg->caam.rfc3686;
    740
    741	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
    742			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
    743
    744	ctx->cdata.keylen = keylen;
    745	ctx->cdata.key_virt = key;
    746	ctx->cdata.key_inline = true;
    747
    748	/* skcipher_encrypt shared descriptor */
    749	desc = ctx->sh_desc_enc;
    750	cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
    751				   ctx1_iv_off);
    752	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    753				   desc_bytes(desc), ctx->dir);
    754
    755	/* skcipher_decrypt shared descriptor */
    756	desc = ctx->sh_desc_dec;
    757	cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
    758				   ctx1_iv_off);
    759	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    760				   desc_bytes(desc), ctx->dir);
    761
    762	return 0;
    763}
    764
    765static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
    766			       const u8 *key, unsigned int keylen)
    767{
    768	int err;
    769
    770	err = aes_check_keylen(keylen);
    771	if (err)
    772		return err;
    773
    774	return skcipher_setkey(skcipher, key, keylen, 0);
    775}
    776
    777static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
    778				   const u8 *key, unsigned int keylen)
    779{
    780	u32 ctx1_iv_off;
    781	int err;
    782
    783	/*
    784	 * RFC3686 specific:
    785	 *	| CONTEXT1[255:128] = {NONCE, IV, COUNTER}
    786	 *	| *key = {KEY, NONCE}
    787	 */
    788	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
    789	keylen -= CTR_RFC3686_NONCE_SIZE;
    790
    791	err = aes_check_keylen(keylen);
    792	if (err)
    793		return err;
    794
    795	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
    796}
    797
    798static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
    799			       const u8 *key, unsigned int keylen)
    800{
    801	u32 ctx1_iv_off;
    802	int err;
    803
    804	/*
    805	 * AES-CTR needs to load IV in CONTEXT1 reg
    806	 * at an offset of 128bits (16bytes)
    807	 * CONTEXT1[255:128] = IV
    808	 */
    809	ctx1_iv_off = 16;
    810
    811	err = aes_check_keylen(keylen);
    812	if (err)
    813		return err;
    814
    815	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
    816}
    817
    818static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
    819			       const u8 *key, unsigned int keylen)
    820{
    821	return verify_skcipher_des_key(skcipher, key) ?:
    822	       skcipher_setkey(skcipher, key, keylen, 0);
    823}
    824
    825static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
    826				const u8 *key, unsigned int keylen)
    827{
    828	return verify_skcipher_des3_key(skcipher, key) ?:
    829	       skcipher_setkey(skcipher, key, keylen, 0);
    830}
    831
    832static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
    833			       unsigned int keylen)
    834{
    835	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
    836	struct device *jrdev = ctx->jrdev;
    837	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
    838	u32 *desc;
    839	int err;
    840
    841	err = xts_verify_key(skcipher, key, keylen);
    842	if (err) {
    843		dev_dbg(jrdev, "key size mismatch\n");
    844		return err;
    845	}
    846
    847	if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
    848		ctx->xts_key_fallback = true;
    849
    850	if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
    851		err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
    852		if (err)
    853			return err;
    854	}
    855
    856	ctx->cdata.keylen = keylen;
    857	ctx->cdata.key_virt = key;
    858	ctx->cdata.key_inline = true;
    859
    860	/* xts_skcipher_encrypt shared descriptor */
    861	desc = ctx->sh_desc_enc;
    862	cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
    863	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
    864				   desc_bytes(desc), ctx->dir);
    865
    866	/* xts_skcipher_decrypt shared descriptor */
    867	desc = ctx->sh_desc_dec;
    868	cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
    869	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
    870				   desc_bytes(desc), ctx->dir);
    871
    872	return 0;
    873}
    874
    875/*
    876 * aead_edesc - s/w-extended aead descriptor
    877 * @src_nents: number of segments in input s/w scatterlist
    878 * @dst_nents: number of segments in output s/w scatterlist
    879 * @mapped_src_nents: number of segments in input h/w link table
    880 * @mapped_dst_nents: number of segments in output h/w link table
    881 * @sec4_sg_bytes: length of dma mapped sec4_sg space
    882 * @bklog: stored to determine if the request needs backlog
    883 * @sec4_sg_dma: bus physical mapped address of h/w link table
    884 * @sec4_sg: pointer to h/w link table
    885 * @hw_desc: the h/w job descriptor followed by any referenced link tables
    886 */
    887struct aead_edesc {
    888	int src_nents;
    889	int dst_nents;
    890	int mapped_src_nents;
    891	int mapped_dst_nents;
    892	int sec4_sg_bytes;
    893	bool bklog;
    894	dma_addr_t sec4_sg_dma;
    895	struct sec4_sg_entry *sec4_sg;
    896	u32 hw_desc[];
    897};
    898
    899/*
    900 * skcipher_edesc - s/w-extended skcipher descriptor
    901 * @src_nents: number of segments in input s/w scatterlist
    902 * @dst_nents: number of segments in output s/w scatterlist
    903 * @mapped_src_nents: number of segments in input h/w link table
    904 * @mapped_dst_nents: number of segments in output h/w link table
    905 * @iv_dma: dma address of iv for checking continuity and link table
    906 * @sec4_sg_bytes: length of dma mapped sec4_sg space
    907 * @bklog: stored to determine if the request needs backlog
    908 * @sec4_sg_dma: bus physical mapped address of h/w link table
    909 * @sec4_sg: pointer to h/w link table
    910 * @hw_desc: the h/w job descriptor followed by any referenced link tables
    911 *	     and IV
    912 */
    913struct skcipher_edesc {
    914	int src_nents;
    915	int dst_nents;
    916	int mapped_src_nents;
    917	int mapped_dst_nents;
    918	dma_addr_t iv_dma;
    919	int sec4_sg_bytes;
    920	bool bklog;
    921	dma_addr_t sec4_sg_dma;
    922	struct sec4_sg_entry *sec4_sg;
    923	u32 hw_desc[];
    924};
    925
    926static void caam_unmap(struct device *dev, struct scatterlist *src,
    927		       struct scatterlist *dst, int src_nents,
    928		       int dst_nents,
    929		       dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
    930		       int sec4_sg_bytes)
    931{
    932	if (dst != src) {
    933		if (src_nents)
    934			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
    935		if (dst_nents)
    936			dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
    937	} else {
    938		dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
    939	}
    940
    941	if (iv_dma)
    942		dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
    943	if (sec4_sg_bytes)
    944		dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
    945				 DMA_TO_DEVICE);
    946}
    947
    948static void aead_unmap(struct device *dev,
    949		       struct aead_edesc *edesc,
    950		       struct aead_request *req)
    951{
    952	caam_unmap(dev, req->src, req->dst,
    953		   edesc->src_nents, edesc->dst_nents, 0, 0,
    954		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
    955}
    956
    957static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
    958			   struct skcipher_request *req)
    959{
    960	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
    961	int ivsize = crypto_skcipher_ivsize(skcipher);
    962
    963	caam_unmap(dev, req->src, req->dst,
    964		   edesc->src_nents, edesc->dst_nents,
    965		   edesc->iv_dma, ivsize,
    966		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
    967}
    968
    969static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
    970			    void *context)
    971{
    972	struct aead_request *req = context;
    973	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
    974	struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
    975	struct aead_edesc *edesc;
    976	int ecode = 0;
    977	bool has_bklog;
    978
    979	dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
    980
    981	edesc = rctx->edesc;
    982	has_bklog = edesc->bklog;
    983
    984	if (err)
    985		ecode = caam_jr_strstatus(jrdev, err);
    986
    987	aead_unmap(jrdev, edesc, req);
    988
    989	kfree(edesc);
    990
    991	/*
    992	 * If no backlog flag, the completion of the request is done
    993	 * by CAAM, not crypto engine.
    994	 */
    995	if (!has_bklog)
    996		aead_request_complete(req, ecode);
    997	else
    998		crypto_finalize_aead_request(jrp->engine, req, ecode);
    999}
   1000
   1001static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
   1002				void *context)
   1003{
   1004	struct skcipher_request *req = context;
   1005	struct skcipher_edesc *edesc;
   1006	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
   1007	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
   1008	struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
   1009	int ivsize = crypto_skcipher_ivsize(skcipher);
   1010	int ecode = 0;
   1011	bool has_bklog;
   1012
   1013	dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
   1014
   1015	edesc = rctx->edesc;
   1016	has_bklog = edesc->bklog;
   1017	if (err)
   1018		ecode = caam_jr_strstatus(jrdev, err);
   1019
   1020	skcipher_unmap(jrdev, edesc, req);
   1021
   1022	/*
   1023	 * The crypto API expects us to set the IV (req->iv) to the last
   1024	 * ciphertext block (CBC mode) or last counter (CTR mode).
   1025	 * This is used e.g. by the CTS mode.
   1026	 */
   1027	if (ivsize && !ecode) {
   1028		memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
   1029		       ivsize);
   1030
   1031		print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
   1032				     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
   1033				     ivsize, 1);
   1034	}
   1035
   1036	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
   1037		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
   1038		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
   1039
   1040	kfree(edesc);
   1041
   1042	/*
   1043	 * If no backlog flag, the completion of the request is done
   1044	 * by CAAM, not crypto engine.
   1045	 */
   1046	if (!has_bklog)
   1047		skcipher_request_complete(req, ecode);
   1048	else
   1049		crypto_finalize_skcipher_request(jrp->engine, req, ecode);
   1050}
   1051
   1052/*
   1053 * Fill in aead job descriptor
   1054 */
   1055static void init_aead_job(struct aead_request *req,
   1056			  struct aead_edesc *edesc,
   1057			  bool all_contig, bool encrypt)
   1058{
   1059	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1060	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1061	int authsize = ctx->authsize;
   1062	u32 *desc = edesc->hw_desc;
   1063	u32 out_options, in_options;
   1064	dma_addr_t dst_dma, src_dma;
   1065	int len, sec4_sg_index = 0;
   1066	dma_addr_t ptr;
   1067	u32 *sh_desc;
   1068
   1069	sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
   1070	ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
   1071
   1072	len = desc_len(sh_desc);
   1073	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
   1074
   1075	if (all_contig) {
   1076		src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
   1077						    0;
   1078		in_options = 0;
   1079	} else {
   1080		src_dma = edesc->sec4_sg_dma;
   1081		sec4_sg_index += edesc->mapped_src_nents;
   1082		in_options = LDST_SGF;
   1083	}
   1084
   1085	append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
   1086			  in_options);
   1087
   1088	dst_dma = src_dma;
   1089	out_options = in_options;
   1090
   1091	if (unlikely(req->src != req->dst)) {
   1092		if (!edesc->mapped_dst_nents) {
   1093			dst_dma = 0;
   1094			out_options = 0;
   1095		} else if (edesc->mapped_dst_nents == 1) {
   1096			dst_dma = sg_dma_address(req->dst);
   1097			out_options = 0;
   1098		} else {
   1099			dst_dma = edesc->sec4_sg_dma +
   1100				  sec4_sg_index *
   1101				  sizeof(struct sec4_sg_entry);
   1102			out_options = LDST_SGF;
   1103		}
   1104	}
   1105
   1106	if (encrypt)
   1107		append_seq_out_ptr(desc, dst_dma,
   1108				   req->assoclen + req->cryptlen + authsize,
   1109				   out_options);
   1110	else
   1111		append_seq_out_ptr(desc, dst_dma,
   1112				   req->assoclen + req->cryptlen - authsize,
   1113				   out_options);
   1114}
   1115
   1116static void init_gcm_job(struct aead_request *req,
   1117			 struct aead_edesc *edesc,
   1118			 bool all_contig, bool encrypt)
   1119{
   1120	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1121	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1122	unsigned int ivsize = crypto_aead_ivsize(aead);
   1123	u32 *desc = edesc->hw_desc;
   1124	bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
   1125	unsigned int last;
   1126
   1127	init_aead_job(req, edesc, all_contig, encrypt);
   1128	append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
   1129
   1130	/* BUG This should not be specific to generic GCM. */
   1131	last = 0;
   1132	if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
   1133		last = FIFOLD_TYPE_LAST1;
   1134
   1135	/* Read GCM IV */
   1136	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
   1137			 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
   1138	/* Append Salt */
   1139	if (!generic_gcm)
   1140		append_data(desc, ctx->key + ctx->cdata.keylen, 4);
   1141	/* Append IV */
   1142	append_data(desc, req->iv, ivsize);
   1143	/* End of blank commands */
   1144}
   1145
   1146static void init_chachapoly_job(struct aead_request *req,
   1147				struct aead_edesc *edesc, bool all_contig,
   1148				bool encrypt)
   1149{
   1150	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1151	unsigned int ivsize = crypto_aead_ivsize(aead);
   1152	unsigned int assoclen = req->assoclen;
   1153	u32 *desc = edesc->hw_desc;
   1154	u32 ctx_iv_off = 4;
   1155
   1156	init_aead_job(req, edesc, all_contig, encrypt);
   1157
   1158	if (ivsize != CHACHAPOLY_IV_SIZE) {
   1159		/* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
   1160		ctx_iv_off += 4;
   1161
   1162		/*
   1163		 * The associated data comes already with the IV but we need
   1164		 * to skip it when we authenticate or encrypt...
   1165		 */
   1166		assoclen -= ivsize;
   1167	}
   1168
   1169	append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
   1170
   1171	/*
   1172	 * For IPsec load the IV further in the same register.
   1173	 * For RFC7539 simply load the 12 bytes nonce in a single operation
   1174	 */
   1175	append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB |
   1176			   LDST_SRCDST_BYTE_CONTEXT |
   1177			   ctx_iv_off << LDST_OFFSET_SHIFT);
   1178}
   1179
   1180static void init_authenc_job(struct aead_request *req,
   1181			     struct aead_edesc *edesc,
   1182			     bool all_contig, bool encrypt)
   1183{
   1184	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1185	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
   1186						 struct caam_aead_alg, aead);
   1187	unsigned int ivsize = crypto_aead_ivsize(aead);
   1188	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1189	struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
   1190	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
   1191			       OP_ALG_AAI_CTR_MOD128);
   1192	const bool is_rfc3686 = alg->caam.rfc3686;
   1193	u32 *desc = edesc->hw_desc;
   1194	u32 ivoffset = 0;
   1195
   1196	/*
   1197	 * AES-CTR needs to load IV in CONTEXT1 reg
   1198	 * at an offset of 128bits (16bytes)
   1199	 * CONTEXT1[255:128] = IV
   1200	 */
   1201	if (ctr_mode)
   1202		ivoffset = 16;
   1203
   1204	/*
   1205	 * RFC3686 specific:
   1206	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
   1207	 */
   1208	if (is_rfc3686)
   1209		ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
   1210
   1211	init_aead_job(req, edesc, all_contig, encrypt);
   1212
   1213	/*
   1214	 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
   1215	 * having DPOVRD as destination.
   1216	 */
   1217	if (ctrlpriv->era < 3)
   1218		append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
   1219	else
   1220		append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
   1221
   1222	if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
   1223		append_load_as_imm(desc, req->iv, ivsize,
   1224				   LDST_CLASS_1_CCB |
   1225				   LDST_SRCDST_BYTE_CONTEXT |
   1226				   (ivoffset << LDST_OFFSET_SHIFT));
   1227}
   1228
   1229/*
   1230 * Fill in skcipher job descriptor
   1231 */
   1232static void init_skcipher_job(struct skcipher_request *req,
   1233			      struct skcipher_edesc *edesc,
   1234			      const bool encrypt)
   1235{
   1236	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
   1237	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
   1238	struct device *jrdev = ctx->jrdev;
   1239	int ivsize = crypto_skcipher_ivsize(skcipher);
   1240	u32 *desc = edesc->hw_desc;
   1241	u32 *sh_desc;
   1242	u32 in_options = 0, out_options = 0;
   1243	dma_addr_t src_dma, dst_dma, ptr;
   1244	int len, sec4_sg_index = 0;
   1245
   1246	print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
   1247			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
   1248	dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
   1249	       (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
   1250
   1251	caam_dump_sg("src    @" __stringify(__LINE__)": ",
   1252		     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
   1253		     edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
   1254
   1255	sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
   1256	ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
   1257
   1258	len = desc_len(sh_desc);
   1259	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
   1260
   1261	if (ivsize || edesc->mapped_src_nents > 1) {
   1262		src_dma = edesc->sec4_sg_dma;
   1263		sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
   1264		in_options = LDST_SGF;
   1265	} else {
   1266		src_dma = sg_dma_address(req->src);
   1267	}
   1268
   1269	append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options);
   1270
   1271	if (likely(req->src == req->dst)) {
   1272		dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
   1273		out_options = in_options;
   1274	} else if (!ivsize && edesc->mapped_dst_nents == 1) {
   1275		dst_dma = sg_dma_address(req->dst);
   1276	} else {
   1277		dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
   1278			  sizeof(struct sec4_sg_entry);
   1279		out_options = LDST_SGF;
   1280	}
   1281
   1282	append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
   1283}
   1284
   1285/*
   1286 * allocate and map the aead extended descriptor
   1287 */
   1288static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
   1289					   int desc_bytes, bool *all_contig_ptr,
   1290					   bool encrypt)
   1291{
   1292	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1293	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1294	struct device *jrdev = ctx->jrdev;
   1295	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
   1296	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
   1297		       GFP_KERNEL : GFP_ATOMIC;
   1298	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
   1299	int src_len, dst_len = 0;
   1300	struct aead_edesc *edesc;
   1301	int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
   1302	unsigned int authsize = ctx->authsize;
   1303
   1304	if (unlikely(req->dst != req->src)) {
   1305		src_len = req->assoclen + req->cryptlen;
   1306		dst_len = src_len + (encrypt ? authsize : (-authsize));
   1307
   1308		src_nents = sg_nents_for_len(req->src, src_len);
   1309		if (unlikely(src_nents < 0)) {
   1310			dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
   1311				src_len);
   1312			return ERR_PTR(src_nents);
   1313		}
   1314
   1315		dst_nents = sg_nents_for_len(req->dst, dst_len);
   1316		if (unlikely(dst_nents < 0)) {
   1317			dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
   1318				dst_len);
   1319			return ERR_PTR(dst_nents);
   1320		}
   1321	} else {
   1322		src_len = req->assoclen + req->cryptlen +
   1323			  (encrypt ? authsize : 0);
   1324
   1325		src_nents = sg_nents_for_len(req->src, src_len);
   1326		if (unlikely(src_nents < 0)) {
   1327			dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
   1328				src_len);
   1329			return ERR_PTR(src_nents);
   1330		}
   1331	}
   1332
   1333	if (likely(req->src == req->dst)) {
   1334		mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
   1335					      DMA_BIDIRECTIONAL);
   1336		if (unlikely(!mapped_src_nents)) {
   1337			dev_err(jrdev, "unable to map source\n");
   1338			return ERR_PTR(-ENOMEM);
   1339		}
   1340	} else {
   1341		/* Cover also the case of null (zero length) input data */
   1342		if (src_nents) {
   1343			mapped_src_nents = dma_map_sg(jrdev, req->src,
   1344						      src_nents, DMA_TO_DEVICE);
   1345			if (unlikely(!mapped_src_nents)) {
   1346				dev_err(jrdev, "unable to map source\n");
   1347				return ERR_PTR(-ENOMEM);
   1348			}
   1349		} else {
   1350			mapped_src_nents = 0;
   1351		}
   1352
   1353		/* Cover also the case of null (zero length) output data */
   1354		if (dst_nents) {
   1355			mapped_dst_nents = dma_map_sg(jrdev, req->dst,
   1356						      dst_nents,
   1357						      DMA_FROM_DEVICE);
   1358			if (unlikely(!mapped_dst_nents)) {
   1359				dev_err(jrdev, "unable to map destination\n");
   1360				dma_unmap_sg(jrdev, req->src, src_nents,
   1361					     DMA_TO_DEVICE);
   1362				return ERR_PTR(-ENOMEM);
   1363			}
   1364		} else {
   1365			mapped_dst_nents = 0;
   1366		}
   1367	}
   1368
   1369	/*
   1370	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
   1371	 * the end of the table by allocating more S/G entries.
   1372	 */
   1373	sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
   1374	if (mapped_dst_nents > 1)
   1375		sec4_sg_len += pad_sg_nents(mapped_dst_nents);
   1376	else
   1377		sec4_sg_len = pad_sg_nents(sec4_sg_len);
   1378
   1379	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
   1380
   1381	/* allocate space for base edesc and hw desc commands, link tables */
   1382	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
   1383			GFP_DMA | flags);
   1384	if (!edesc) {
   1385		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
   1386			   0, 0, 0);
   1387		return ERR_PTR(-ENOMEM);
   1388	}
   1389
   1390	edesc->src_nents = src_nents;
   1391	edesc->dst_nents = dst_nents;
   1392	edesc->mapped_src_nents = mapped_src_nents;
   1393	edesc->mapped_dst_nents = mapped_dst_nents;
   1394	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
   1395			 desc_bytes;
   1396
   1397	rctx->edesc = edesc;
   1398
   1399	*all_contig_ptr = !(mapped_src_nents > 1);
   1400
   1401	sec4_sg_index = 0;
   1402	if (mapped_src_nents > 1) {
   1403		sg_to_sec4_sg_last(req->src, src_len,
   1404				   edesc->sec4_sg + sec4_sg_index, 0);
   1405		sec4_sg_index += mapped_src_nents;
   1406	}
   1407	if (mapped_dst_nents > 1) {
   1408		sg_to_sec4_sg_last(req->dst, dst_len,
   1409				   edesc->sec4_sg + sec4_sg_index, 0);
   1410	}
   1411
   1412	if (!sec4_sg_bytes)
   1413		return edesc;
   1414
   1415	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
   1416					    sec4_sg_bytes, DMA_TO_DEVICE);
   1417	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
   1418		dev_err(jrdev, "unable to map S/G table\n");
   1419		aead_unmap(jrdev, edesc, req);
   1420		kfree(edesc);
   1421		return ERR_PTR(-ENOMEM);
   1422	}
   1423
   1424	edesc->sec4_sg_bytes = sec4_sg_bytes;
   1425
   1426	return edesc;
   1427}
   1428
   1429static int aead_enqueue_req(struct device *jrdev, struct aead_request *req)
   1430{
   1431	struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
   1432	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
   1433	struct aead_edesc *edesc = rctx->edesc;
   1434	u32 *desc = edesc->hw_desc;
   1435	int ret;
   1436
   1437	/*
   1438	 * Only the backlog request are sent to crypto-engine since the others
   1439	 * can be handled by CAAM, if free, especially since JR has up to 1024
   1440	 * entries (more than the 10 entries from crypto-engine).
   1441	 */
   1442	if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
   1443		ret = crypto_transfer_aead_request_to_engine(jrpriv->engine,
   1444							     req);
   1445	else
   1446		ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req);
   1447
   1448	if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
   1449		aead_unmap(jrdev, edesc, req);
   1450		kfree(rctx->edesc);
   1451	}
   1452
   1453	return ret;
   1454}
   1455
   1456static inline int chachapoly_crypt(struct aead_request *req, bool encrypt)
   1457{
   1458	struct aead_edesc *edesc;
   1459	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1460	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1461	struct device *jrdev = ctx->jrdev;
   1462	bool all_contig;
   1463	u32 *desc;
   1464
   1465	edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
   1466				 encrypt);
   1467	if (IS_ERR(edesc))
   1468		return PTR_ERR(edesc);
   1469
   1470	desc = edesc->hw_desc;
   1471
   1472	init_chachapoly_job(req, edesc, all_contig, encrypt);
   1473	print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
   1474			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
   1475			     1);
   1476
   1477	return aead_enqueue_req(jrdev, req);
   1478}
   1479
   1480static int chachapoly_encrypt(struct aead_request *req)
   1481{
   1482	return chachapoly_crypt(req, true);
   1483}
   1484
   1485static int chachapoly_decrypt(struct aead_request *req)
   1486{
   1487	return chachapoly_crypt(req, false);
   1488}
   1489
   1490static inline int aead_crypt(struct aead_request *req, bool encrypt)
   1491{
   1492	struct aead_edesc *edesc;
   1493	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1494	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1495	struct device *jrdev = ctx->jrdev;
   1496	bool all_contig;
   1497
   1498	/* allocate extended descriptor */
   1499	edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
   1500				 &all_contig, encrypt);
   1501	if (IS_ERR(edesc))
   1502		return PTR_ERR(edesc);
   1503
   1504	/* Create and submit job descriptor */
   1505	init_authenc_job(req, edesc, all_contig, encrypt);
   1506
   1507	print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
   1508			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
   1509			     desc_bytes(edesc->hw_desc), 1);
   1510
   1511	return aead_enqueue_req(jrdev, req);
   1512}
   1513
   1514static int aead_encrypt(struct aead_request *req)
   1515{
   1516	return aead_crypt(req, true);
   1517}
   1518
   1519static int aead_decrypt(struct aead_request *req)
   1520{
   1521	return aead_crypt(req, false);
   1522}
   1523
   1524static int aead_do_one_req(struct crypto_engine *engine, void *areq)
   1525{
   1526	struct aead_request *req = aead_request_cast(areq);
   1527	struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
   1528	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
   1529	u32 *desc = rctx->edesc->hw_desc;
   1530	int ret;
   1531
   1532	rctx->edesc->bklog = true;
   1533
   1534	ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
   1535
   1536	if (ret == -ENOSPC && engine->retry_support)
   1537		return ret;
   1538
   1539	if (ret != -EINPROGRESS) {
   1540		aead_unmap(ctx->jrdev, rctx->edesc, req);
   1541		kfree(rctx->edesc);
   1542	} else {
   1543		ret = 0;
   1544	}
   1545
   1546	return ret;
   1547}
   1548
   1549static inline int gcm_crypt(struct aead_request *req, bool encrypt)
   1550{
   1551	struct aead_edesc *edesc;
   1552	struct crypto_aead *aead = crypto_aead_reqtfm(req);
   1553	struct caam_ctx *ctx = crypto_aead_ctx(aead);
   1554	struct device *jrdev = ctx->jrdev;
   1555	bool all_contig;
   1556
   1557	/* allocate extended descriptor */
   1558	edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig,
   1559				 encrypt);
   1560	if (IS_ERR(edesc))
   1561		return PTR_ERR(edesc);
   1562
   1563	/* Create and submit job descriptor */
   1564	init_gcm_job(req, edesc, all_contig, encrypt);
   1565
   1566	print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
   1567			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
   1568			     desc_bytes(edesc->hw_desc), 1);
   1569
   1570	return aead_enqueue_req(jrdev, req);
   1571}
   1572
   1573static int gcm_encrypt(struct aead_request *req)
   1574{
   1575	return gcm_crypt(req, true);
   1576}
   1577
   1578static int gcm_decrypt(struct aead_request *req)
   1579{
   1580	return gcm_crypt(req, false);
   1581}
   1582
   1583static int ipsec_gcm_encrypt(struct aead_request *req)
   1584{
   1585	return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req);
   1586}
   1587
   1588static int ipsec_gcm_decrypt(struct aead_request *req)
   1589{
   1590	return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req);
   1591}
   1592
   1593/*
   1594 * allocate and map the skcipher extended descriptor for skcipher
   1595 */
   1596static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
   1597						   int desc_bytes)
   1598{
   1599	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
   1600	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
   1601	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
   1602	struct device *jrdev = ctx->jrdev;
   1603	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
   1604		       GFP_KERNEL : GFP_ATOMIC;
   1605	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
   1606	struct skcipher_edesc *edesc;
   1607	dma_addr_t iv_dma = 0;
   1608	u8 *iv;
   1609	int ivsize = crypto_skcipher_ivsize(skcipher);
   1610	int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
   1611
   1612	src_nents = sg_nents_for_len(req->src, req->cryptlen);
   1613	if (unlikely(src_nents < 0)) {
   1614		dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
   1615			req->cryptlen);
   1616		return ERR_PTR(src_nents);
   1617	}
   1618
   1619	if (req->dst != req->src) {
   1620		dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
   1621		if (unlikely(dst_nents < 0)) {
   1622			dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
   1623				req->cryptlen);
   1624			return ERR_PTR(dst_nents);
   1625		}
   1626	}
   1627
   1628	if (likely(req->src == req->dst)) {
   1629		mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
   1630					      DMA_BIDIRECTIONAL);
   1631		if (unlikely(!mapped_src_nents)) {
   1632			dev_err(jrdev, "unable to map source\n");
   1633			return ERR_PTR(-ENOMEM);
   1634		}
   1635	} else {
   1636		mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
   1637					      DMA_TO_DEVICE);
   1638		if (unlikely(!mapped_src_nents)) {
   1639			dev_err(jrdev, "unable to map source\n");
   1640			return ERR_PTR(-ENOMEM);
   1641		}
   1642		mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
   1643					      DMA_FROM_DEVICE);
   1644		if (unlikely(!mapped_dst_nents)) {
   1645			dev_err(jrdev, "unable to map destination\n");
   1646			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
   1647			return ERR_PTR(-ENOMEM);
   1648		}
   1649	}
   1650
   1651	if (!ivsize && mapped_src_nents == 1)
   1652		sec4_sg_ents = 0; // no need for an input hw s/g table
   1653	else
   1654		sec4_sg_ents = mapped_src_nents + !!ivsize;
   1655	dst_sg_idx = sec4_sg_ents;
   1656
   1657	/*
   1658	 * Input, output HW S/G tables: [IV, src][dst, IV]
   1659	 * IV entries point to the same buffer
   1660	 * If src == dst, S/G entries are reused (S/G tables overlap)
   1661	 *
   1662	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
   1663	 * the end of the table by allocating more S/G entries. Logic:
   1664	 * if (output S/G)
   1665	 *      pad output S/G, if needed
   1666	 * else if (input S/G) ...
   1667	 *      pad input S/G, if needed
   1668	 */
   1669	if (ivsize || mapped_dst_nents > 1) {
   1670		if (req->src == req->dst)
   1671			sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
   1672		else
   1673			sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
   1674						     !!ivsize);
   1675	} else {
   1676		sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
   1677	}
   1678
   1679	sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
   1680
   1681	/*
   1682	 * allocate space for base edesc and hw desc commands, link tables, IV
   1683	 */
   1684	edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
   1685			GFP_DMA | flags);
   1686	if (!edesc) {
   1687		dev_err(jrdev, "could not allocate extended descriptor\n");
   1688		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
   1689			   0, 0, 0);
   1690		return ERR_PTR(-ENOMEM);
   1691	}
   1692
   1693	edesc->src_nents = src_nents;
   1694	edesc->dst_nents = dst_nents;
   1695	edesc->mapped_src_nents = mapped_src_nents;
   1696	edesc->mapped_dst_nents = mapped_dst_nents;
   1697	edesc->sec4_sg_bytes = sec4_sg_bytes;
   1698	edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
   1699						  desc_bytes);
   1700	rctx->edesc = edesc;
   1701
   1702	/* Make sure IV is located in a DMAable area */
   1703	if (ivsize) {
   1704		iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
   1705		memcpy(iv, req->iv, ivsize);
   1706
   1707		iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
   1708		if (dma_mapping_error(jrdev, iv_dma)) {
   1709			dev_err(jrdev, "unable to map IV\n");
   1710			caam_unmap(jrdev, req->src, req->dst, src_nents,
   1711				   dst_nents, 0, 0, 0, 0);
   1712			kfree(edesc);
   1713			return ERR_PTR(-ENOMEM);
   1714		}
   1715
   1716		dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
   1717	}
   1718	if (dst_sg_idx)
   1719		sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
   1720			      !!ivsize, 0);
   1721
   1722	if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
   1723		sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
   1724			      dst_sg_idx, 0);
   1725
   1726	if (ivsize)
   1727		dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
   1728				   mapped_dst_nents, iv_dma, ivsize, 0);
   1729
   1730	if (ivsize || mapped_dst_nents > 1)
   1731		sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
   1732				    mapped_dst_nents - 1 + !!ivsize);
   1733
   1734	if (sec4_sg_bytes) {
   1735		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
   1736						    sec4_sg_bytes,
   1737						    DMA_TO_DEVICE);
   1738		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
   1739			dev_err(jrdev, "unable to map S/G table\n");
   1740			caam_unmap(jrdev, req->src, req->dst, src_nents,
   1741				   dst_nents, iv_dma, ivsize, 0, 0);
   1742			kfree(edesc);
   1743			return ERR_PTR(-ENOMEM);
   1744		}
   1745	}
   1746
   1747	edesc->iv_dma = iv_dma;
   1748
   1749	print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
   1750			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
   1751			     sec4_sg_bytes, 1);
   1752
   1753	return edesc;
   1754}
   1755
   1756static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
   1757{
   1758	struct skcipher_request *req = skcipher_request_cast(areq);
   1759	struct caam_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
   1760	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
   1761	u32 *desc = rctx->edesc->hw_desc;
   1762	int ret;
   1763
   1764	rctx->edesc->bklog = true;
   1765
   1766	ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
   1767
   1768	if (ret == -ENOSPC && engine->retry_support)
   1769		return ret;
   1770
   1771	if (ret != -EINPROGRESS) {
   1772		skcipher_unmap(ctx->jrdev, rctx->edesc, req);
   1773		kfree(rctx->edesc);
   1774	} else {
   1775		ret = 0;
   1776	}
   1777
   1778	return ret;
   1779}
   1780
   1781static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
   1782{
   1783	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
   1784	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
   1785
   1786	return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
   1787}
   1788
   1789static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
   1790{
   1791	struct skcipher_edesc *edesc;
   1792	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
   1793	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
   1794	struct device *jrdev = ctx->jrdev;
   1795	struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
   1796	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
   1797	u32 *desc;
   1798	int ret = 0;
   1799
   1800	/*
   1801	 * XTS is expected to return an error even for input length = 0
   1802	 * Note that the case input length < block size will be caught during
   1803	 * HW offloading and return an error.
   1804	 */
   1805	if (!req->cryptlen && !ctx->fallback)
   1806		return 0;
   1807
   1808	if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
   1809			      ctx->xts_key_fallback)) {
   1810		struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
   1811
   1812		skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
   1813		skcipher_request_set_callback(&rctx->fallback_req,
   1814					      req->base.flags,
   1815					      req->base.complete,
   1816					      req->base.data);
   1817		skcipher_request_set_crypt(&rctx->fallback_req, req->src,
   1818					   req->dst, req->cryptlen, req->iv);
   1819
   1820		return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
   1821				 crypto_skcipher_decrypt(&rctx->fallback_req);
   1822	}
   1823
   1824	/* allocate extended descriptor */
   1825	edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
   1826	if (IS_ERR(edesc))
   1827		return PTR_ERR(edesc);
   1828
   1829	/* Create and submit job descriptor*/
   1830	init_skcipher_job(req, edesc, encrypt);
   1831
   1832	print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
   1833			     DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
   1834			     desc_bytes(edesc->hw_desc), 1);
   1835
   1836	desc = edesc->hw_desc;
   1837	/*
   1838	 * Only the backlog request are sent to crypto-engine since the others
   1839	 * can be handled by CAAM, if free, especially since JR has up to 1024
   1840	 * entries (more than the 10 entries from crypto-engine).
   1841	 */
   1842	if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
   1843		ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine,
   1844								 req);
   1845	else
   1846		ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req);
   1847
   1848	if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
   1849		skcipher_unmap(jrdev, edesc, req);
   1850		kfree(edesc);
   1851	}
   1852
   1853	return ret;
   1854}
   1855
   1856static int skcipher_encrypt(struct skcipher_request *req)
   1857{
   1858	return skcipher_crypt(req, true);
   1859}
   1860
   1861static int skcipher_decrypt(struct skcipher_request *req)
   1862{
   1863	return skcipher_crypt(req, false);
   1864}
   1865
   1866static struct caam_skcipher_alg driver_algs[] = {
   1867	{
   1868		.skcipher = {
   1869			.base = {
   1870				.cra_name = "cbc(aes)",
   1871				.cra_driver_name = "cbc-aes-caam",
   1872				.cra_blocksize = AES_BLOCK_SIZE,
   1873			},
   1874			.setkey = aes_skcipher_setkey,
   1875			.encrypt = skcipher_encrypt,
   1876			.decrypt = skcipher_decrypt,
   1877			.min_keysize = AES_MIN_KEY_SIZE,
   1878			.max_keysize = AES_MAX_KEY_SIZE,
   1879			.ivsize = AES_BLOCK_SIZE,
   1880		},
   1881		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   1882	},
   1883	{
   1884		.skcipher = {
   1885			.base = {
   1886				.cra_name = "cbc(des3_ede)",
   1887				.cra_driver_name = "cbc-3des-caam",
   1888				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   1889			},
   1890			.setkey = des3_skcipher_setkey,
   1891			.encrypt = skcipher_encrypt,
   1892			.decrypt = skcipher_decrypt,
   1893			.min_keysize = DES3_EDE_KEY_SIZE,
   1894			.max_keysize = DES3_EDE_KEY_SIZE,
   1895			.ivsize = DES3_EDE_BLOCK_SIZE,
   1896		},
   1897		.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   1898	},
   1899	{
   1900		.skcipher = {
   1901			.base = {
   1902				.cra_name = "cbc(des)",
   1903				.cra_driver_name = "cbc-des-caam",
   1904				.cra_blocksize = DES_BLOCK_SIZE,
   1905			},
   1906			.setkey = des_skcipher_setkey,
   1907			.encrypt = skcipher_encrypt,
   1908			.decrypt = skcipher_decrypt,
   1909			.min_keysize = DES_KEY_SIZE,
   1910			.max_keysize = DES_KEY_SIZE,
   1911			.ivsize = DES_BLOCK_SIZE,
   1912		},
   1913		.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   1914	},
   1915	{
   1916		.skcipher = {
   1917			.base = {
   1918				.cra_name = "ctr(aes)",
   1919				.cra_driver_name = "ctr-aes-caam",
   1920				.cra_blocksize = 1,
   1921			},
   1922			.setkey = ctr_skcipher_setkey,
   1923			.encrypt = skcipher_encrypt,
   1924			.decrypt = skcipher_decrypt,
   1925			.min_keysize = AES_MIN_KEY_SIZE,
   1926			.max_keysize = AES_MAX_KEY_SIZE,
   1927			.ivsize = AES_BLOCK_SIZE,
   1928			.chunksize = AES_BLOCK_SIZE,
   1929		},
   1930		.caam.class1_alg_type = OP_ALG_ALGSEL_AES |
   1931					OP_ALG_AAI_CTR_MOD128,
   1932	},
   1933	{
   1934		.skcipher = {
   1935			.base = {
   1936				.cra_name = "rfc3686(ctr(aes))",
   1937				.cra_driver_name = "rfc3686-ctr-aes-caam",
   1938				.cra_blocksize = 1,
   1939			},
   1940			.setkey = rfc3686_skcipher_setkey,
   1941			.encrypt = skcipher_encrypt,
   1942			.decrypt = skcipher_decrypt,
   1943			.min_keysize = AES_MIN_KEY_SIZE +
   1944				       CTR_RFC3686_NONCE_SIZE,
   1945			.max_keysize = AES_MAX_KEY_SIZE +
   1946				       CTR_RFC3686_NONCE_SIZE,
   1947			.ivsize = CTR_RFC3686_IV_SIZE,
   1948			.chunksize = AES_BLOCK_SIZE,
   1949		},
   1950		.caam = {
   1951			.class1_alg_type = OP_ALG_ALGSEL_AES |
   1952					   OP_ALG_AAI_CTR_MOD128,
   1953			.rfc3686 = true,
   1954		},
   1955	},
   1956	{
   1957		.skcipher = {
   1958			.base = {
   1959				.cra_name = "xts(aes)",
   1960				.cra_driver_name = "xts-aes-caam",
   1961				.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
   1962				.cra_blocksize = AES_BLOCK_SIZE,
   1963			},
   1964			.setkey = xts_skcipher_setkey,
   1965			.encrypt = skcipher_encrypt,
   1966			.decrypt = skcipher_decrypt,
   1967			.min_keysize = 2 * AES_MIN_KEY_SIZE,
   1968			.max_keysize = 2 * AES_MAX_KEY_SIZE,
   1969			.ivsize = AES_BLOCK_SIZE,
   1970		},
   1971		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
   1972	},
   1973	{
   1974		.skcipher = {
   1975			.base = {
   1976				.cra_name = "ecb(des)",
   1977				.cra_driver_name = "ecb-des-caam",
   1978				.cra_blocksize = DES_BLOCK_SIZE,
   1979			},
   1980			.setkey = des_skcipher_setkey,
   1981			.encrypt = skcipher_encrypt,
   1982			.decrypt = skcipher_decrypt,
   1983			.min_keysize = DES_KEY_SIZE,
   1984			.max_keysize = DES_KEY_SIZE,
   1985		},
   1986		.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
   1987	},
   1988	{
   1989		.skcipher = {
   1990			.base = {
   1991				.cra_name = "ecb(aes)",
   1992				.cra_driver_name = "ecb-aes-caam",
   1993				.cra_blocksize = AES_BLOCK_SIZE,
   1994			},
   1995			.setkey = aes_skcipher_setkey,
   1996			.encrypt = skcipher_encrypt,
   1997			.decrypt = skcipher_decrypt,
   1998			.min_keysize = AES_MIN_KEY_SIZE,
   1999			.max_keysize = AES_MAX_KEY_SIZE,
   2000		},
   2001		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
   2002	},
   2003	{
   2004		.skcipher = {
   2005			.base = {
   2006				.cra_name = "ecb(des3_ede)",
   2007				.cra_driver_name = "ecb-des3-caam",
   2008				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2009			},
   2010			.setkey = des3_skcipher_setkey,
   2011			.encrypt = skcipher_encrypt,
   2012			.decrypt = skcipher_decrypt,
   2013			.min_keysize = DES3_EDE_KEY_SIZE,
   2014			.max_keysize = DES3_EDE_KEY_SIZE,
   2015		},
   2016		.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
   2017	},
   2018};
   2019
   2020static struct caam_aead_alg driver_aeads[] = {
   2021	{
   2022		.aead = {
   2023			.base = {
   2024				.cra_name = "rfc4106(gcm(aes))",
   2025				.cra_driver_name = "rfc4106-gcm-aes-caam",
   2026				.cra_blocksize = 1,
   2027			},
   2028			.setkey = rfc4106_setkey,
   2029			.setauthsize = rfc4106_setauthsize,
   2030			.encrypt = ipsec_gcm_encrypt,
   2031			.decrypt = ipsec_gcm_decrypt,
   2032			.ivsize = GCM_RFC4106_IV_SIZE,
   2033			.maxauthsize = AES_BLOCK_SIZE,
   2034		},
   2035		.caam = {
   2036			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
   2037			.nodkp = true,
   2038		},
   2039	},
   2040	{
   2041		.aead = {
   2042			.base = {
   2043				.cra_name = "rfc4543(gcm(aes))",
   2044				.cra_driver_name = "rfc4543-gcm-aes-caam",
   2045				.cra_blocksize = 1,
   2046			},
   2047			.setkey = rfc4543_setkey,
   2048			.setauthsize = rfc4543_setauthsize,
   2049			.encrypt = ipsec_gcm_encrypt,
   2050			.decrypt = ipsec_gcm_decrypt,
   2051			.ivsize = GCM_RFC4543_IV_SIZE,
   2052			.maxauthsize = AES_BLOCK_SIZE,
   2053		},
   2054		.caam = {
   2055			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
   2056			.nodkp = true,
   2057		},
   2058	},
   2059	/* Galois Counter Mode */
   2060	{
   2061		.aead = {
   2062			.base = {
   2063				.cra_name = "gcm(aes)",
   2064				.cra_driver_name = "gcm-aes-caam",
   2065				.cra_blocksize = 1,
   2066			},
   2067			.setkey = gcm_setkey,
   2068			.setauthsize = gcm_setauthsize,
   2069			.encrypt = gcm_encrypt,
   2070			.decrypt = gcm_decrypt,
   2071			.ivsize = GCM_AES_IV_SIZE,
   2072			.maxauthsize = AES_BLOCK_SIZE,
   2073		},
   2074		.caam = {
   2075			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
   2076			.nodkp = true,
   2077		},
   2078	},
   2079	/* single-pass ipsec_esp descriptor */
   2080	{
   2081		.aead = {
   2082			.base = {
   2083				.cra_name = "authenc(hmac(md5),"
   2084					    "ecb(cipher_null))",
   2085				.cra_driver_name = "authenc-hmac-md5-"
   2086						   "ecb-cipher_null-caam",
   2087				.cra_blocksize = NULL_BLOCK_SIZE,
   2088			},
   2089			.setkey = aead_setkey,
   2090			.setauthsize = aead_setauthsize,
   2091			.encrypt = aead_encrypt,
   2092			.decrypt = aead_decrypt,
   2093			.ivsize = NULL_IV_SIZE,
   2094			.maxauthsize = MD5_DIGEST_SIZE,
   2095		},
   2096		.caam = {
   2097			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2098					   OP_ALG_AAI_HMAC_PRECOMP,
   2099		},
   2100	},
   2101	{
   2102		.aead = {
   2103			.base = {
   2104				.cra_name = "authenc(hmac(sha1),"
   2105					    "ecb(cipher_null))",
   2106				.cra_driver_name = "authenc-hmac-sha1-"
   2107						   "ecb-cipher_null-caam",
   2108				.cra_blocksize = NULL_BLOCK_SIZE,
   2109			},
   2110			.setkey = aead_setkey,
   2111			.setauthsize = aead_setauthsize,
   2112			.encrypt = aead_encrypt,
   2113			.decrypt = aead_decrypt,
   2114			.ivsize = NULL_IV_SIZE,
   2115			.maxauthsize = SHA1_DIGEST_SIZE,
   2116		},
   2117		.caam = {
   2118			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2119					   OP_ALG_AAI_HMAC_PRECOMP,
   2120		},
   2121	},
   2122	{
   2123		.aead = {
   2124			.base = {
   2125				.cra_name = "authenc(hmac(sha224),"
   2126					    "ecb(cipher_null))",
   2127				.cra_driver_name = "authenc-hmac-sha224-"
   2128						   "ecb-cipher_null-caam",
   2129				.cra_blocksize = NULL_BLOCK_SIZE,
   2130			},
   2131			.setkey = aead_setkey,
   2132			.setauthsize = aead_setauthsize,
   2133			.encrypt = aead_encrypt,
   2134			.decrypt = aead_decrypt,
   2135			.ivsize = NULL_IV_SIZE,
   2136			.maxauthsize = SHA224_DIGEST_SIZE,
   2137		},
   2138		.caam = {
   2139			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2140					   OP_ALG_AAI_HMAC_PRECOMP,
   2141		},
   2142	},
   2143	{
   2144		.aead = {
   2145			.base = {
   2146				.cra_name = "authenc(hmac(sha256),"
   2147					    "ecb(cipher_null))",
   2148				.cra_driver_name = "authenc-hmac-sha256-"
   2149						   "ecb-cipher_null-caam",
   2150				.cra_blocksize = NULL_BLOCK_SIZE,
   2151			},
   2152			.setkey = aead_setkey,
   2153			.setauthsize = aead_setauthsize,
   2154			.encrypt = aead_encrypt,
   2155			.decrypt = aead_decrypt,
   2156			.ivsize = NULL_IV_SIZE,
   2157			.maxauthsize = SHA256_DIGEST_SIZE,
   2158		},
   2159		.caam = {
   2160			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2161					   OP_ALG_AAI_HMAC_PRECOMP,
   2162		},
   2163	},
   2164	{
   2165		.aead = {
   2166			.base = {
   2167				.cra_name = "authenc(hmac(sha384),"
   2168					    "ecb(cipher_null))",
   2169				.cra_driver_name = "authenc-hmac-sha384-"
   2170						   "ecb-cipher_null-caam",
   2171				.cra_blocksize = NULL_BLOCK_SIZE,
   2172			},
   2173			.setkey = aead_setkey,
   2174			.setauthsize = aead_setauthsize,
   2175			.encrypt = aead_encrypt,
   2176			.decrypt = aead_decrypt,
   2177			.ivsize = NULL_IV_SIZE,
   2178			.maxauthsize = SHA384_DIGEST_SIZE,
   2179		},
   2180		.caam = {
   2181			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2182					   OP_ALG_AAI_HMAC_PRECOMP,
   2183		},
   2184	},
   2185	{
   2186		.aead = {
   2187			.base = {
   2188				.cra_name = "authenc(hmac(sha512),"
   2189					    "ecb(cipher_null))",
   2190				.cra_driver_name = "authenc-hmac-sha512-"
   2191						   "ecb-cipher_null-caam",
   2192				.cra_blocksize = NULL_BLOCK_SIZE,
   2193			},
   2194			.setkey = aead_setkey,
   2195			.setauthsize = aead_setauthsize,
   2196			.encrypt = aead_encrypt,
   2197			.decrypt = aead_decrypt,
   2198			.ivsize = NULL_IV_SIZE,
   2199			.maxauthsize = SHA512_DIGEST_SIZE,
   2200		},
   2201		.caam = {
   2202			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   2203					   OP_ALG_AAI_HMAC_PRECOMP,
   2204		},
   2205	},
   2206	{
   2207		.aead = {
   2208			.base = {
   2209				.cra_name = "authenc(hmac(md5),cbc(aes))",
   2210				.cra_driver_name = "authenc-hmac-md5-"
   2211						   "cbc-aes-caam",
   2212				.cra_blocksize = AES_BLOCK_SIZE,
   2213			},
   2214			.setkey = aead_setkey,
   2215			.setauthsize = aead_setauthsize,
   2216			.encrypt = aead_encrypt,
   2217			.decrypt = aead_decrypt,
   2218			.ivsize = AES_BLOCK_SIZE,
   2219			.maxauthsize = MD5_DIGEST_SIZE,
   2220		},
   2221		.caam = {
   2222			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2223			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2224					   OP_ALG_AAI_HMAC_PRECOMP,
   2225		},
   2226	},
   2227	{
   2228		.aead = {
   2229			.base = {
   2230				.cra_name = "echainiv(authenc(hmac(md5),"
   2231					    "cbc(aes)))",
   2232				.cra_driver_name = "echainiv-authenc-hmac-md5-"
   2233						   "cbc-aes-caam",
   2234				.cra_blocksize = AES_BLOCK_SIZE,
   2235			},
   2236			.setkey = aead_setkey,
   2237			.setauthsize = aead_setauthsize,
   2238			.encrypt = aead_encrypt,
   2239			.decrypt = aead_decrypt,
   2240			.ivsize = AES_BLOCK_SIZE,
   2241			.maxauthsize = MD5_DIGEST_SIZE,
   2242		},
   2243		.caam = {
   2244			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2245			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2246					   OP_ALG_AAI_HMAC_PRECOMP,
   2247			.geniv = true,
   2248		},
   2249	},
   2250	{
   2251		.aead = {
   2252			.base = {
   2253				.cra_name = "authenc(hmac(sha1),cbc(aes))",
   2254				.cra_driver_name = "authenc-hmac-sha1-"
   2255						   "cbc-aes-caam",
   2256				.cra_blocksize = AES_BLOCK_SIZE,
   2257			},
   2258			.setkey = aead_setkey,
   2259			.setauthsize = aead_setauthsize,
   2260			.encrypt = aead_encrypt,
   2261			.decrypt = aead_decrypt,
   2262			.ivsize = AES_BLOCK_SIZE,
   2263			.maxauthsize = SHA1_DIGEST_SIZE,
   2264		},
   2265		.caam = {
   2266			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2267			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2268					   OP_ALG_AAI_HMAC_PRECOMP,
   2269		},
   2270	},
   2271	{
   2272		.aead = {
   2273			.base = {
   2274				.cra_name = "echainiv(authenc(hmac(sha1),"
   2275					    "cbc(aes)))",
   2276				.cra_driver_name = "echainiv-authenc-"
   2277						   "hmac-sha1-cbc-aes-caam",
   2278				.cra_blocksize = AES_BLOCK_SIZE,
   2279			},
   2280			.setkey = aead_setkey,
   2281			.setauthsize = aead_setauthsize,
   2282			.encrypt = aead_encrypt,
   2283			.decrypt = aead_decrypt,
   2284			.ivsize = AES_BLOCK_SIZE,
   2285			.maxauthsize = SHA1_DIGEST_SIZE,
   2286		},
   2287		.caam = {
   2288			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2289			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2290					   OP_ALG_AAI_HMAC_PRECOMP,
   2291			.geniv = true,
   2292		},
   2293	},
   2294	{
   2295		.aead = {
   2296			.base = {
   2297				.cra_name = "authenc(hmac(sha224),cbc(aes))",
   2298				.cra_driver_name = "authenc-hmac-sha224-"
   2299						   "cbc-aes-caam",
   2300				.cra_blocksize = AES_BLOCK_SIZE,
   2301			},
   2302			.setkey = aead_setkey,
   2303			.setauthsize = aead_setauthsize,
   2304			.encrypt = aead_encrypt,
   2305			.decrypt = aead_decrypt,
   2306			.ivsize = AES_BLOCK_SIZE,
   2307			.maxauthsize = SHA224_DIGEST_SIZE,
   2308		},
   2309		.caam = {
   2310			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2311			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2312					   OP_ALG_AAI_HMAC_PRECOMP,
   2313		},
   2314	},
   2315	{
   2316		.aead = {
   2317			.base = {
   2318				.cra_name = "echainiv(authenc(hmac(sha224),"
   2319					    "cbc(aes)))",
   2320				.cra_driver_name = "echainiv-authenc-"
   2321						   "hmac-sha224-cbc-aes-caam",
   2322				.cra_blocksize = AES_BLOCK_SIZE,
   2323			},
   2324			.setkey = aead_setkey,
   2325			.setauthsize = aead_setauthsize,
   2326			.encrypt = aead_encrypt,
   2327			.decrypt = aead_decrypt,
   2328			.ivsize = AES_BLOCK_SIZE,
   2329			.maxauthsize = SHA224_DIGEST_SIZE,
   2330		},
   2331		.caam = {
   2332			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2333			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2334					   OP_ALG_AAI_HMAC_PRECOMP,
   2335			.geniv = true,
   2336		},
   2337	},
   2338	{
   2339		.aead = {
   2340			.base = {
   2341				.cra_name = "authenc(hmac(sha256),cbc(aes))",
   2342				.cra_driver_name = "authenc-hmac-sha256-"
   2343						   "cbc-aes-caam",
   2344				.cra_blocksize = AES_BLOCK_SIZE,
   2345			},
   2346			.setkey = aead_setkey,
   2347			.setauthsize = aead_setauthsize,
   2348			.encrypt = aead_encrypt,
   2349			.decrypt = aead_decrypt,
   2350			.ivsize = AES_BLOCK_SIZE,
   2351			.maxauthsize = SHA256_DIGEST_SIZE,
   2352		},
   2353		.caam = {
   2354			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2355			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2356					   OP_ALG_AAI_HMAC_PRECOMP,
   2357		},
   2358	},
   2359	{
   2360		.aead = {
   2361			.base = {
   2362				.cra_name = "echainiv(authenc(hmac(sha256),"
   2363					    "cbc(aes)))",
   2364				.cra_driver_name = "echainiv-authenc-"
   2365						   "hmac-sha256-cbc-aes-caam",
   2366				.cra_blocksize = AES_BLOCK_SIZE,
   2367			},
   2368			.setkey = aead_setkey,
   2369			.setauthsize = aead_setauthsize,
   2370			.encrypt = aead_encrypt,
   2371			.decrypt = aead_decrypt,
   2372			.ivsize = AES_BLOCK_SIZE,
   2373			.maxauthsize = SHA256_DIGEST_SIZE,
   2374		},
   2375		.caam = {
   2376			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2377			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2378					   OP_ALG_AAI_HMAC_PRECOMP,
   2379			.geniv = true,
   2380		},
   2381	},
   2382	{
   2383		.aead = {
   2384			.base = {
   2385				.cra_name = "authenc(hmac(sha384),cbc(aes))",
   2386				.cra_driver_name = "authenc-hmac-sha384-"
   2387						   "cbc-aes-caam",
   2388				.cra_blocksize = AES_BLOCK_SIZE,
   2389			},
   2390			.setkey = aead_setkey,
   2391			.setauthsize = aead_setauthsize,
   2392			.encrypt = aead_encrypt,
   2393			.decrypt = aead_decrypt,
   2394			.ivsize = AES_BLOCK_SIZE,
   2395			.maxauthsize = SHA384_DIGEST_SIZE,
   2396		},
   2397		.caam = {
   2398			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2399			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2400					   OP_ALG_AAI_HMAC_PRECOMP,
   2401		},
   2402	},
   2403	{
   2404		.aead = {
   2405			.base = {
   2406				.cra_name = "echainiv(authenc(hmac(sha384),"
   2407					    "cbc(aes)))",
   2408				.cra_driver_name = "echainiv-authenc-"
   2409						   "hmac-sha384-cbc-aes-caam",
   2410				.cra_blocksize = AES_BLOCK_SIZE,
   2411			},
   2412			.setkey = aead_setkey,
   2413			.setauthsize = aead_setauthsize,
   2414			.encrypt = aead_encrypt,
   2415			.decrypt = aead_decrypt,
   2416			.ivsize = AES_BLOCK_SIZE,
   2417			.maxauthsize = SHA384_DIGEST_SIZE,
   2418		},
   2419		.caam = {
   2420			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2421			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2422					   OP_ALG_AAI_HMAC_PRECOMP,
   2423			.geniv = true,
   2424		},
   2425	},
   2426	{
   2427		.aead = {
   2428			.base = {
   2429				.cra_name = "authenc(hmac(sha512),cbc(aes))",
   2430				.cra_driver_name = "authenc-hmac-sha512-"
   2431						   "cbc-aes-caam",
   2432				.cra_blocksize = AES_BLOCK_SIZE,
   2433			},
   2434			.setkey = aead_setkey,
   2435			.setauthsize = aead_setauthsize,
   2436			.encrypt = aead_encrypt,
   2437			.decrypt = aead_decrypt,
   2438			.ivsize = AES_BLOCK_SIZE,
   2439			.maxauthsize = SHA512_DIGEST_SIZE,
   2440		},
   2441		.caam = {
   2442			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2443			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   2444					   OP_ALG_AAI_HMAC_PRECOMP,
   2445		},
   2446	},
   2447	{
   2448		.aead = {
   2449			.base = {
   2450				.cra_name = "echainiv(authenc(hmac(sha512),"
   2451					    "cbc(aes)))",
   2452				.cra_driver_name = "echainiv-authenc-"
   2453						   "hmac-sha512-cbc-aes-caam",
   2454				.cra_blocksize = AES_BLOCK_SIZE,
   2455			},
   2456			.setkey = aead_setkey,
   2457			.setauthsize = aead_setauthsize,
   2458			.encrypt = aead_encrypt,
   2459			.decrypt = aead_decrypt,
   2460			.ivsize = AES_BLOCK_SIZE,
   2461			.maxauthsize = SHA512_DIGEST_SIZE,
   2462		},
   2463		.caam = {
   2464			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
   2465			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   2466					   OP_ALG_AAI_HMAC_PRECOMP,
   2467			.geniv = true,
   2468		},
   2469	},
   2470	{
   2471		.aead = {
   2472			.base = {
   2473				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
   2474				.cra_driver_name = "authenc-hmac-md5-"
   2475						   "cbc-des3_ede-caam",
   2476				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2477			},
   2478			.setkey = des3_aead_setkey,
   2479			.setauthsize = aead_setauthsize,
   2480			.encrypt = aead_encrypt,
   2481			.decrypt = aead_decrypt,
   2482			.ivsize = DES3_EDE_BLOCK_SIZE,
   2483			.maxauthsize = MD5_DIGEST_SIZE,
   2484		},
   2485		.caam = {
   2486			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2487			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2488					   OP_ALG_AAI_HMAC_PRECOMP,
   2489		}
   2490	},
   2491	{
   2492		.aead = {
   2493			.base = {
   2494				.cra_name = "echainiv(authenc(hmac(md5),"
   2495					    "cbc(des3_ede)))",
   2496				.cra_driver_name = "echainiv-authenc-hmac-md5-"
   2497						   "cbc-des3_ede-caam",
   2498				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2499			},
   2500			.setkey = des3_aead_setkey,
   2501			.setauthsize = aead_setauthsize,
   2502			.encrypt = aead_encrypt,
   2503			.decrypt = aead_decrypt,
   2504			.ivsize = DES3_EDE_BLOCK_SIZE,
   2505			.maxauthsize = MD5_DIGEST_SIZE,
   2506		},
   2507		.caam = {
   2508			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2509			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2510					   OP_ALG_AAI_HMAC_PRECOMP,
   2511			.geniv = true,
   2512		}
   2513	},
   2514	{
   2515		.aead = {
   2516			.base = {
   2517				.cra_name = "authenc(hmac(sha1),"
   2518					    "cbc(des3_ede))",
   2519				.cra_driver_name = "authenc-hmac-sha1-"
   2520						   "cbc-des3_ede-caam",
   2521				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2522			},
   2523			.setkey = des3_aead_setkey,
   2524			.setauthsize = aead_setauthsize,
   2525			.encrypt = aead_encrypt,
   2526			.decrypt = aead_decrypt,
   2527			.ivsize = DES3_EDE_BLOCK_SIZE,
   2528			.maxauthsize = SHA1_DIGEST_SIZE,
   2529		},
   2530		.caam = {
   2531			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2532			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2533					   OP_ALG_AAI_HMAC_PRECOMP,
   2534		},
   2535	},
   2536	{
   2537		.aead = {
   2538			.base = {
   2539				.cra_name = "echainiv(authenc(hmac(sha1),"
   2540					    "cbc(des3_ede)))",
   2541				.cra_driver_name = "echainiv-authenc-"
   2542						   "hmac-sha1-"
   2543						   "cbc-des3_ede-caam",
   2544				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2545			},
   2546			.setkey = des3_aead_setkey,
   2547			.setauthsize = aead_setauthsize,
   2548			.encrypt = aead_encrypt,
   2549			.decrypt = aead_decrypt,
   2550			.ivsize = DES3_EDE_BLOCK_SIZE,
   2551			.maxauthsize = SHA1_DIGEST_SIZE,
   2552		},
   2553		.caam = {
   2554			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2555			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2556					   OP_ALG_AAI_HMAC_PRECOMP,
   2557			.geniv = true,
   2558		},
   2559	},
   2560	{
   2561		.aead = {
   2562			.base = {
   2563				.cra_name = "authenc(hmac(sha224),"
   2564					    "cbc(des3_ede))",
   2565				.cra_driver_name = "authenc-hmac-sha224-"
   2566						   "cbc-des3_ede-caam",
   2567				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2568			},
   2569			.setkey = des3_aead_setkey,
   2570			.setauthsize = aead_setauthsize,
   2571			.encrypt = aead_encrypt,
   2572			.decrypt = aead_decrypt,
   2573			.ivsize = DES3_EDE_BLOCK_SIZE,
   2574			.maxauthsize = SHA224_DIGEST_SIZE,
   2575		},
   2576		.caam = {
   2577			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2578			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2579					   OP_ALG_AAI_HMAC_PRECOMP,
   2580		},
   2581	},
   2582	{
   2583		.aead = {
   2584			.base = {
   2585				.cra_name = "echainiv(authenc(hmac(sha224),"
   2586					    "cbc(des3_ede)))",
   2587				.cra_driver_name = "echainiv-authenc-"
   2588						   "hmac-sha224-"
   2589						   "cbc-des3_ede-caam",
   2590				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2591			},
   2592			.setkey = des3_aead_setkey,
   2593			.setauthsize = aead_setauthsize,
   2594			.encrypt = aead_encrypt,
   2595			.decrypt = aead_decrypt,
   2596			.ivsize = DES3_EDE_BLOCK_SIZE,
   2597			.maxauthsize = SHA224_DIGEST_SIZE,
   2598		},
   2599		.caam = {
   2600			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2601			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2602					   OP_ALG_AAI_HMAC_PRECOMP,
   2603			.geniv = true,
   2604		},
   2605	},
   2606	{
   2607		.aead = {
   2608			.base = {
   2609				.cra_name = "authenc(hmac(sha256),"
   2610					    "cbc(des3_ede))",
   2611				.cra_driver_name = "authenc-hmac-sha256-"
   2612						   "cbc-des3_ede-caam",
   2613				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2614			},
   2615			.setkey = des3_aead_setkey,
   2616			.setauthsize = aead_setauthsize,
   2617			.encrypt = aead_encrypt,
   2618			.decrypt = aead_decrypt,
   2619			.ivsize = DES3_EDE_BLOCK_SIZE,
   2620			.maxauthsize = SHA256_DIGEST_SIZE,
   2621		},
   2622		.caam = {
   2623			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2624			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2625					   OP_ALG_AAI_HMAC_PRECOMP,
   2626		},
   2627	},
   2628	{
   2629		.aead = {
   2630			.base = {
   2631				.cra_name = "echainiv(authenc(hmac(sha256),"
   2632					    "cbc(des3_ede)))",
   2633				.cra_driver_name = "echainiv-authenc-"
   2634						   "hmac-sha256-"
   2635						   "cbc-des3_ede-caam",
   2636				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2637			},
   2638			.setkey = des3_aead_setkey,
   2639			.setauthsize = aead_setauthsize,
   2640			.encrypt = aead_encrypt,
   2641			.decrypt = aead_decrypt,
   2642			.ivsize = DES3_EDE_BLOCK_SIZE,
   2643			.maxauthsize = SHA256_DIGEST_SIZE,
   2644		},
   2645		.caam = {
   2646			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2647			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2648					   OP_ALG_AAI_HMAC_PRECOMP,
   2649			.geniv = true,
   2650		},
   2651	},
   2652	{
   2653		.aead = {
   2654			.base = {
   2655				.cra_name = "authenc(hmac(sha384),"
   2656					    "cbc(des3_ede))",
   2657				.cra_driver_name = "authenc-hmac-sha384-"
   2658						   "cbc-des3_ede-caam",
   2659				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2660			},
   2661			.setkey = des3_aead_setkey,
   2662			.setauthsize = aead_setauthsize,
   2663			.encrypt = aead_encrypt,
   2664			.decrypt = aead_decrypt,
   2665			.ivsize = DES3_EDE_BLOCK_SIZE,
   2666			.maxauthsize = SHA384_DIGEST_SIZE,
   2667		},
   2668		.caam = {
   2669			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2670			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2671					   OP_ALG_AAI_HMAC_PRECOMP,
   2672		},
   2673	},
   2674	{
   2675		.aead = {
   2676			.base = {
   2677				.cra_name = "echainiv(authenc(hmac(sha384),"
   2678					    "cbc(des3_ede)))",
   2679				.cra_driver_name = "echainiv-authenc-"
   2680						   "hmac-sha384-"
   2681						   "cbc-des3_ede-caam",
   2682				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2683			},
   2684			.setkey = des3_aead_setkey,
   2685			.setauthsize = aead_setauthsize,
   2686			.encrypt = aead_encrypt,
   2687			.decrypt = aead_decrypt,
   2688			.ivsize = DES3_EDE_BLOCK_SIZE,
   2689			.maxauthsize = SHA384_DIGEST_SIZE,
   2690		},
   2691		.caam = {
   2692			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2693			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2694					   OP_ALG_AAI_HMAC_PRECOMP,
   2695			.geniv = true,
   2696		},
   2697	},
   2698	{
   2699		.aead = {
   2700			.base = {
   2701				.cra_name = "authenc(hmac(sha512),"
   2702					    "cbc(des3_ede))",
   2703				.cra_driver_name = "authenc-hmac-sha512-"
   2704						   "cbc-des3_ede-caam",
   2705				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2706			},
   2707			.setkey = des3_aead_setkey,
   2708			.setauthsize = aead_setauthsize,
   2709			.encrypt = aead_encrypt,
   2710			.decrypt = aead_decrypt,
   2711			.ivsize = DES3_EDE_BLOCK_SIZE,
   2712			.maxauthsize = SHA512_DIGEST_SIZE,
   2713		},
   2714		.caam = {
   2715			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2716			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   2717					   OP_ALG_AAI_HMAC_PRECOMP,
   2718		},
   2719	},
   2720	{
   2721		.aead = {
   2722			.base = {
   2723				.cra_name = "echainiv(authenc(hmac(sha512),"
   2724					    "cbc(des3_ede)))",
   2725				.cra_driver_name = "echainiv-authenc-"
   2726						   "hmac-sha512-"
   2727						   "cbc-des3_ede-caam",
   2728				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
   2729			},
   2730			.setkey = des3_aead_setkey,
   2731			.setauthsize = aead_setauthsize,
   2732			.encrypt = aead_encrypt,
   2733			.decrypt = aead_decrypt,
   2734			.ivsize = DES3_EDE_BLOCK_SIZE,
   2735			.maxauthsize = SHA512_DIGEST_SIZE,
   2736		},
   2737		.caam = {
   2738			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
   2739			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   2740					   OP_ALG_AAI_HMAC_PRECOMP,
   2741			.geniv = true,
   2742		},
   2743	},
   2744	{
   2745		.aead = {
   2746			.base = {
   2747				.cra_name = "authenc(hmac(md5),cbc(des))",
   2748				.cra_driver_name = "authenc-hmac-md5-"
   2749						   "cbc-des-caam",
   2750				.cra_blocksize = DES_BLOCK_SIZE,
   2751			},
   2752			.setkey = aead_setkey,
   2753			.setauthsize = aead_setauthsize,
   2754			.encrypt = aead_encrypt,
   2755			.decrypt = aead_decrypt,
   2756			.ivsize = DES_BLOCK_SIZE,
   2757			.maxauthsize = MD5_DIGEST_SIZE,
   2758		},
   2759		.caam = {
   2760			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2761			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2762					   OP_ALG_AAI_HMAC_PRECOMP,
   2763		},
   2764	},
   2765	{
   2766		.aead = {
   2767			.base = {
   2768				.cra_name = "echainiv(authenc(hmac(md5),"
   2769					    "cbc(des)))",
   2770				.cra_driver_name = "echainiv-authenc-hmac-md5-"
   2771						   "cbc-des-caam",
   2772				.cra_blocksize = DES_BLOCK_SIZE,
   2773			},
   2774			.setkey = aead_setkey,
   2775			.setauthsize = aead_setauthsize,
   2776			.encrypt = aead_encrypt,
   2777			.decrypt = aead_decrypt,
   2778			.ivsize = DES_BLOCK_SIZE,
   2779			.maxauthsize = MD5_DIGEST_SIZE,
   2780		},
   2781		.caam = {
   2782			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2783			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   2784					   OP_ALG_AAI_HMAC_PRECOMP,
   2785			.geniv = true,
   2786		},
   2787	},
   2788	{
   2789		.aead = {
   2790			.base = {
   2791				.cra_name = "authenc(hmac(sha1),cbc(des))",
   2792				.cra_driver_name = "authenc-hmac-sha1-"
   2793						   "cbc-des-caam",
   2794				.cra_blocksize = DES_BLOCK_SIZE,
   2795			},
   2796			.setkey = aead_setkey,
   2797			.setauthsize = aead_setauthsize,
   2798			.encrypt = aead_encrypt,
   2799			.decrypt = aead_decrypt,
   2800			.ivsize = DES_BLOCK_SIZE,
   2801			.maxauthsize = SHA1_DIGEST_SIZE,
   2802		},
   2803		.caam = {
   2804			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2805			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2806					   OP_ALG_AAI_HMAC_PRECOMP,
   2807		},
   2808	},
   2809	{
   2810		.aead = {
   2811			.base = {
   2812				.cra_name = "echainiv(authenc(hmac(sha1),"
   2813					    "cbc(des)))",
   2814				.cra_driver_name = "echainiv-authenc-"
   2815						   "hmac-sha1-cbc-des-caam",
   2816				.cra_blocksize = DES_BLOCK_SIZE,
   2817			},
   2818			.setkey = aead_setkey,
   2819			.setauthsize = aead_setauthsize,
   2820			.encrypt = aead_encrypt,
   2821			.decrypt = aead_decrypt,
   2822			.ivsize = DES_BLOCK_SIZE,
   2823			.maxauthsize = SHA1_DIGEST_SIZE,
   2824		},
   2825		.caam = {
   2826			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2827			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   2828					   OP_ALG_AAI_HMAC_PRECOMP,
   2829			.geniv = true,
   2830		},
   2831	},
   2832	{
   2833		.aead = {
   2834			.base = {
   2835				.cra_name = "authenc(hmac(sha224),cbc(des))",
   2836				.cra_driver_name = "authenc-hmac-sha224-"
   2837						   "cbc-des-caam",
   2838				.cra_blocksize = DES_BLOCK_SIZE,
   2839			},
   2840			.setkey = aead_setkey,
   2841			.setauthsize = aead_setauthsize,
   2842			.encrypt = aead_encrypt,
   2843			.decrypt = aead_decrypt,
   2844			.ivsize = DES_BLOCK_SIZE,
   2845			.maxauthsize = SHA224_DIGEST_SIZE,
   2846		},
   2847		.caam = {
   2848			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2849			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2850					   OP_ALG_AAI_HMAC_PRECOMP,
   2851		},
   2852	},
   2853	{
   2854		.aead = {
   2855			.base = {
   2856				.cra_name = "echainiv(authenc(hmac(sha224),"
   2857					    "cbc(des)))",
   2858				.cra_driver_name = "echainiv-authenc-"
   2859						   "hmac-sha224-cbc-des-caam",
   2860				.cra_blocksize = DES_BLOCK_SIZE,
   2861			},
   2862			.setkey = aead_setkey,
   2863			.setauthsize = aead_setauthsize,
   2864			.encrypt = aead_encrypt,
   2865			.decrypt = aead_decrypt,
   2866			.ivsize = DES_BLOCK_SIZE,
   2867			.maxauthsize = SHA224_DIGEST_SIZE,
   2868		},
   2869		.caam = {
   2870			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2871			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   2872					   OP_ALG_AAI_HMAC_PRECOMP,
   2873			.geniv = true,
   2874		},
   2875	},
   2876	{
   2877		.aead = {
   2878			.base = {
   2879				.cra_name = "authenc(hmac(sha256),cbc(des))",
   2880				.cra_driver_name = "authenc-hmac-sha256-"
   2881						   "cbc-des-caam",
   2882				.cra_blocksize = DES_BLOCK_SIZE,
   2883			},
   2884			.setkey = aead_setkey,
   2885			.setauthsize = aead_setauthsize,
   2886			.encrypt = aead_encrypt,
   2887			.decrypt = aead_decrypt,
   2888			.ivsize = DES_BLOCK_SIZE,
   2889			.maxauthsize = SHA256_DIGEST_SIZE,
   2890		},
   2891		.caam = {
   2892			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2893			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2894					   OP_ALG_AAI_HMAC_PRECOMP,
   2895		},
   2896	},
   2897	{
   2898		.aead = {
   2899			.base = {
   2900				.cra_name = "echainiv(authenc(hmac(sha256),"
   2901					    "cbc(des)))",
   2902				.cra_driver_name = "echainiv-authenc-"
   2903						   "hmac-sha256-cbc-des-caam",
   2904				.cra_blocksize = DES_BLOCK_SIZE,
   2905			},
   2906			.setkey = aead_setkey,
   2907			.setauthsize = aead_setauthsize,
   2908			.encrypt = aead_encrypt,
   2909			.decrypt = aead_decrypt,
   2910			.ivsize = DES_BLOCK_SIZE,
   2911			.maxauthsize = SHA256_DIGEST_SIZE,
   2912		},
   2913		.caam = {
   2914			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2915			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   2916					   OP_ALG_AAI_HMAC_PRECOMP,
   2917			.geniv = true,
   2918		},
   2919	},
   2920	{
   2921		.aead = {
   2922			.base = {
   2923				.cra_name = "authenc(hmac(sha384),cbc(des))",
   2924				.cra_driver_name = "authenc-hmac-sha384-"
   2925						   "cbc-des-caam",
   2926				.cra_blocksize = DES_BLOCK_SIZE,
   2927			},
   2928			.setkey = aead_setkey,
   2929			.setauthsize = aead_setauthsize,
   2930			.encrypt = aead_encrypt,
   2931			.decrypt = aead_decrypt,
   2932			.ivsize = DES_BLOCK_SIZE,
   2933			.maxauthsize = SHA384_DIGEST_SIZE,
   2934		},
   2935		.caam = {
   2936			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2937			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2938					   OP_ALG_AAI_HMAC_PRECOMP,
   2939		},
   2940	},
   2941	{
   2942		.aead = {
   2943			.base = {
   2944				.cra_name = "echainiv(authenc(hmac(sha384),"
   2945					    "cbc(des)))",
   2946				.cra_driver_name = "echainiv-authenc-"
   2947						   "hmac-sha384-cbc-des-caam",
   2948				.cra_blocksize = DES_BLOCK_SIZE,
   2949			},
   2950			.setkey = aead_setkey,
   2951			.setauthsize = aead_setauthsize,
   2952			.encrypt = aead_encrypt,
   2953			.decrypt = aead_decrypt,
   2954			.ivsize = DES_BLOCK_SIZE,
   2955			.maxauthsize = SHA384_DIGEST_SIZE,
   2956		},
   2957		.caam = {
   2958			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2959			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   2960					   OP_ALG_AAI_HMAC_PRECOMP,
   2961			.geniv = true,
   2962		},
   2963	},
   2964	{
   2965		.aead = {
   2966			.base = {
   2967				.cra_name = "authenc(hmac(sha512),cbc(des))",
   2968				.cra_driver_name = "authenc-hmac-sha512-"
   2969						   "cbc-des-caam",
   2970				.cra_blocksize = DES_BLOCK_SIZE,
   2971			},
   2972			.setkey = aead_setkey,
   2973			.setauthsize = aead_setauthsize,
   2974			.encrypt = aead_encrypt,
   2975			.decrypt = aead_decrypt,
   2976			.ivsize = DES_BLOCK_SIZE,
   2977			.maxauthsize = SHA512_DIGEST_SIZE,
   2978		},
   2979		.caam = {
   2980			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   2981			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   2982					   OP_ALG_AAI_HMAC_PRECOMP,
   2983		},
   2984	},
   2985	{
   2986		.aead = {
   2987			.base = {
   2988				.cra_name = "echainiv(authenc(hmac(sha512),"
   2989					    "cbc(des)))",
   2990				.cra_driver_name = "echainiv-authenc-"
   2991						   "hmac-sha512-cbc-des-caam",
   2992				.cra_blocksize = DES_BLOCK_SIZE,
   2993			},
   2994			.setkey = aead_setkey,
   2995			.setauthsize = aead_setauthsize,
   2996			.encrypt = aead_encrypt,
   2997			.decrypt = aead_decrypt,
   2998			.ivsize = DES_BLOCK_SIZE,
   2999			.maxauthsize = SHA512_DIGEST_SIZE,
   3000		},
   3001		.caam = {
   3002			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
   3003			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   3004					   OP_ALG_AAI_HMAC_PRECOMP,
   3005			.geniv = true,
   3006		},
   3007	},
   3008	{
   3009		.aead = {
   3010			.base = {
   3011				.cra_name = "authenc(hmac(md5),"
   3012					    "rfc3686(ctr(aes)))",
   3013				.cra_driver_name = "authenc-hmac-md5-"
   3014						   "rfc3686-ctr-aes-caam",
   3015				.cra_blocksize = 1,
   3016			},
   3017			.setkey = aead_setkey,
   3018			.setauthsize = aead_setauthsize,
   3019			.encrypt = aead_encrypt,
   3020			.decrypt = aead_decrypt,
   3021			.ivsize = CTR_RFC3686_IV_SIZE,
   3022			.maxauthsize = MD5_DIGEST_SIZE,
   3023		},
   3024		.caam = {
   3025			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3026					   OP_ALG_AAI_CTR_MOD128,
   3027			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   3028					   OP_ALG_AAI_HMAC_PRECOMP,
   3029			.rfc3686 = true,
   3030		},
   3031	},
   3032	{
   3033		.aead = {
   3034			.base = {
   3035				.cra_name = "seqiv(authenc("
   3036					    "hmac(md5),rfc3686(ctr(aes))))",
   3037				.cra_driver_name = "seqiv-authenc-hmac-md5-"
   3038						   "rfc3686-ctr-aes-caam",
   3039				.cra_blocksize = 1,
   3040			},
   3041			.setkey = aead_setkey,
   3042			.setauthsize = aead_setauthsize,
   3043			.encrypt = aead_encrypt,
   3044			.decrypt = aead_decrypt,
   3045			.ivsize = CTR_RFC3686_IV_SIZE,
   3046			.maxauthsize = MD5_DIGEST_SIZE,
   3047		},
   3048		.caam = {
   3049			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3050					   OP_ALG_AAI_CTR_MOD128,
   3051			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
   3052					   OP_ALG_AAI_HMAC_PRECOMP,
   3053			.rfc3686 = true,
   3054			.geniv = true,
   3055		},
   3056	},
   3057	{
   3058		.aead = {
   3059			.base = {
   3060				.cra_name = "authenc(hmac(sha1),"
   3061					    "rfc3686(ctr(aes)))",
   3062				.cra_driver_name = "authenc-hmac-sha1-"
   3063						   "rfc3686-ctr-aes-caam",
   3064				.cra_blocksize = 1,
   3065			},
   3066			.setkey = aead_setkey,
   3067			.setauthsize = aead_setauthsize,
   3068			.encrypt = aead_encrypt,
   3069			.decrypt = aead_decrypt,
   3070			.ivsize = CTR_RFC3686_IV_SIZE,
   3071			.maxauthsize = SHA1_DIGEST_SIZE,
   3072		},
   3073		.caam = {
   3074			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3075					   OP_ALG_AAI_CTR_MOD128,
   3076			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   3077					   OP_ALG_AAI_HMAC_PRECOMP,
   3078			.rfc3686 = true,
   3079		},
   3080	},
   3081	{
   3082		.aead = {
   3083			.base = {
   3084				.cra_name = "seqiv(authenc("
   3085					    "hmac(sha1),rfc3686(ctr(aes))))",
   3086				.cra_driver_name = "seqiv-authenc-hmac-sha1-"
   3087						   "rfc3686-ctr-aes-caam",
   3088				.cra_blocksize = 1,
   3089			},
   3090			.setkey = aead_setkey,
   3091			.setauthsize = aead_setauthsize,
   3092			.encrypt = aead_encrypt,
   3093			.decrypt = aead_decrypt,
   3094			.ivsize = CTR_RFC3686_IV_SIZE,
   3095			.maxauthsize = SHA1_DIGEST_SIZE,
   3096		},
   3097		.caam = {
   3098			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3099					   OP_ALG_AAI_CTR_MOD128,
   3100			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
   3101					   OP_ALG_AAI_HMAC_PRECOMP,
   3102			.rfc3686 = true,
   3103			.geniv = true,
   3104		},
   3105	},
   3106	{
   3107		.aead = {
   3108			.base = {
   3109				.cra_name = "authenc(hmac(sha224),"
   3110					    "rfc3686(ctr(aes)))",
   3111				.cra_driver_name = "authenc-hmac-sha224-"
   3112						   "rfc3686-ctr-aes-caam",
   3113				.cra_blocksize = 1,
   3114			},
   3115			.setkey = aead_setkey,
   3116			.setauthsize = aead_setauthsize,
   3117			.encrypt = aead_encrypt,
   3118			.decrypt = aead_decrypt,
   3119			.ivsize = CTR_RFC3686_IV_SIZE,
   3120			.maxauthsize = SHA224_DIGEST_SIZE,
   3121		},
   3122		.caam = {
   3123			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3124					   OP_ALG_AAI_CTR_MOD128,
   3125			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   3126					   OP_ALG_AAI_HMAC_PRECOMP,
   3127			.rfc3686 = true,
   3128		},
   3129	},
   3130	{
   3131		.aead = {
   3132			.base = {
   3133				.cra_name = "seqiv(authenc("
   3134					    "hmac(sha224),rfc3686(ctr(aes))))",
   3135				.cra_driver_name = "seqiv-authenc-hmac-sha224-"
   3136						   "rfc3686-ctr-aes-caam",
   3137				.cra_blocksize = 1,
   3138			},
   3139			.setkey = aead_setkey,
   3140			.setauthsize = aead_setauthsize,
   3141			.encrypt = aead_encrypt,
   3142			.decrypt = aead_decrypt,
   3143			.ivsize = CTR_RFC3686_IV_SIZE,
   3144			.maxauthsize = SHA224_DIGEST_SIZE,
   3145		},
   3146		.caam = {
   3147			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3148					   OP_ALG_AAI_CTR_MOD128,
   3149			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
   3150					   OP_ALG_AAI_HMAC_PRECOMP,
   3151			.rfc3686 = true,
   3152			.geniv = true,
   3153		},
   3154	},
   3155	{
   3156		.aead = {
   3157			.base = {
   3158				.cra_name = "authenc(hmac(sha256),"
   3159					    "rfc3686(ctr(aes)))",
   3160				.cra_driver_name = "authenc-hmac-sha256-"
   3161						   "rfc3686-ctr-aes-caam",
   3162				.cra_blocksize = 1,
   3163			},
   3164			.setkey = aead_setkey,
   3165			.setauthsize = aead_setauthsize,
   3166			.encrypt = aead_encrypt,
   3167			.decrypt = aead_decrypt,
   3168			.ivsize = CTR_RFC3686_IV_SIZE,
   3169			.maxauthsize = SHA256_DIGEST_SIZE,
   3170		},
   3171		.caam = {
   3172			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3173					   OP_ALG_AAI_CTR_MOD128,
   3174			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   3175					   OP_ALG_AAI_HMAC_PRECOMP,
   3176			.rfc3686 = true,
   3177		},
   3178	},
   3179	{
   3180		.aead = {
   3181			.base = {
   3182				.cra_name = "seqiv(authenc(hmac(sha256),"
   3183					    "rfc3686(ctr(aes))))",
   3184				.cra_driver_name = "seqiv-authenc-hmac-sha256-"
   3185						   "rfc3686-ctr-aes-caam",
   3186				.cra_blocksize = 1,
   3187			},
   3188			.setkey = aead_setkey,
   3189			.setauthsize = aead_setauthsize,
   3190			.encrypt = aead_encrypt,
   3191			.decrypt = aead_decrypt,
   3192			.ivsize = CTR_RFC3686_IV_SIZE,
   3193			.maxauthsize = SHA256_DIGEST_SIZE,
   3194		},
   3195		.caam = {
   3196			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3197					   OP_ALG_AAI_CTR_MOD128,
   3198			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
   3199					   OP_ALG_AAI_HMAC_PRECOMP,
   3200			.rfc3686 = true,
   3201			.geniv = true,
   3202		},
   3203	},
   3204	{
   3205		.aead = {
   3206			.base = {
   3207				.cra_name = "authenc(hmac(sha384),"
   3208					    "rfc3686(ctr(aes)))",
   3209				.cra_driver_name = "authenc-hmac-sha384-"
   3210						   "rfc3686-ctr-aes-caam",
   3211				.cra_blocksize = 1,
   3212			},
   3213			.setkey = aead_setkey,
   3214			.setauthsize = aead_setauthsize,
   3215			.encrypt = aead_encrypt,
   3216			.decrypt = aead_decrypt,
   3217			.ivsize = CTR_RFC3686_IV_SIZE,
   3218			.maxauthsize = SHA384_DIGEST_SIZE,
   3219		},
   3220		.caam = {
   3221			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3222					   OP_ALG_AAI_CTR_MOD128,
   3223			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   3224					   OP_ALG_AAI_HMAC_PRECOMP,
   3225			.rfc3686 = true,
   3226		},
   3227	},
   3228	{
   3229		.aead = {
   3230			.base = {
   3231				.cra_name = "seqiv(authenc(hmac(sha384),"
   3232					    "rfc3686(ctr(aes))))",
   3233				.cra_driver_name = "seqiv-authenc-hmac-sha384-"
   3234						   "rfc3686-ctr-aes-caam",
   3235				.cra_blocksize = 1,
   3236			},
   3237			.setkey = aead_setkey,
   3238			.setauthsize = aead_setauthsize,
   3239			.encrypt = aead_encrypt,
   3240			.decrypt = aead_decrypt,
   3241			.ivsize = CTR_RFC3686_IV_SIZE,
   3242			.maxauthsize = SHA384_DIGEST_SIZE,
   3243		},
   3244		.caam = {
   3245			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3246					   OP_ALG_AAI_CTR_MOD128,
   3247			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
   3248					   OP_ALG_AAI_HMAC_PRECOMP,
   3249			.rfc3686 = true,
   3250			.geniv = true,
   3251		},
   3252	},
   3253	{
   3254		.aead = {
   3255			.base = {
   3256				.cra_name = "authenc(hmac(sha512),"
   3257					    "rfc3686(ctr(aes)))",
   3258				.cra_driver_name = "authenc-hmac-sha512-"
   3259						   "rfc3686-ctr-aes-caam",
   3260				.cra_blocksize = 1,
   3261			},
   3262			.setkey = aead_setkey,
   3263			.setauthsize = aead_setauthsize,
   3264			.encrypt = aead_encrypt,
   3265			.decrypt = aead_decrypt,
   3266			.ivsize = CTR_RFC3686_IV_SIZE,
   3267			.maxauthsize = SHA512_DIGEST_SIZE,
   3268		},
   3269		.caam = {
   3270			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3271					   OP_ALG_AAI_CTR_MOD128,
   3272			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   3273					   OP_ALG_AAI_HMAC_PRECOMP,
   3274			.rfc3686 = true,
   3275		},
   3276	},
   3277	{
   3278		.aead = {
   3279			.base = {
   3280				.cra_name = "seqiv(authenc(hmac(sha512),"
   3281					    "rfc3686(ctr(aes))))",
   3282				.cra_driver_name = "seqiv-authenc-hmac-sha512-"
   3283						   "rfc3686-ctr-aes-caam",
   3284				.cra_blocksize = 1,
   3285			},
   3286			.setkey = aead_setkey,
   3287			.setauthsize = aead_setauthsize,
   3288			.encrypt = aead_encrypt,
   3289			.decrypt = aead_decrypt,
   3290			.ivsize = CTR_RFC3686_IV_SIZE,
   3291			.maxauthsize = SHA512_DIGEST_SIZE,
   3292		},
   3293		.caam = {
   3294			.class1_alg_type = OP_ALG_ALGSEL_AES |
   3295					   OP_ALG_AAI_CTR_MOD128,
   3296			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
   3297					   OP_ALG_AAI_HMAC_PRECOMP,
   3298			.rfc3686 = true,
   3299			.geniv = true,
   3300		},
   3301	},
   3302	{
   3303		.aead = {
   3304			.base = {
   3305				.cra_name = "rfc7539(chacha20,poly1305)",
   3306				.cra_driver_name = "rfc7539-chacha20-poly1305-"
   3307						   "caam",
   3308				.cra_blocksize = 1,
   3309			},
   3310			.setkey = chachapoly_setkey,
   3311			.setauthsize = chachapoly_setauthsize,
   3312			.encrypt = chachapoly_encrypt,
   3313			.decrypt = chachapoly_decrypt,
   3314			.ivsize = CHACHAPOLY_IV_SIZE,
   3315			.maxauthsize = POLY1305_DIGEST_SIZE,
   3316		},
   3317		.caam = {
   3318			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
   3319					   OP_ALG_AAI_AEAD,
   3320			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
   3321					   OP_ALG_AAI_AEAD,
   3322			.nodkp = true,
   3323		},
   3324	},
   3325	{
   3326		.aead = {
   3327			.base = {
   3328				.cra_name = "rfc7539esp(chacha20,poly1305)",
   3329				.cra_driver_name = "rfc7539esp-chacha20-"
   3330						   "poly1305-caam",
   3331				.cra_blocksize = 1,
   3332			},
   3333			.setkey = chachapoly_setkey,
   3334			.setauthsize = chachapoly_setauthsize,
   3335			.encrypt = chachapoly_encrypt,
   3336			.decrypt = chachapoly_decrypt,
   3337			.ivsize = 8,
   3338			.maxauthsize = POLY1305_DIGEST_SIZE,
   3339		},
   3340		.caam = {
   3341			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
   3342					   OP_ALG_AAI_AEAD,
   3343			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
   3344					   OP_ALG_AAI_AEAD,
   3345			.nodkp = true,
   3346		},
   3347	},
   3348};
   3349
   3350static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
   3351			    bool uses_dkp)
   3352{
   3353	dma_addr_t dma_addr;
   3354	struct caam_drv_private *priv;
   3355	const size_t sh_desc_enc_offset = offsetof(struct caam_ctx,
   3356						   sh_desc_enc);
   3357
   3358	ctx->jrdev = caam_jr_alloc();
   3359	if (IS_ERR(ctx->jrdev)) {
   3360		pr_err("Job Ring Device allocation for transform failed\n");
   3361		return PTR_ERR(ctx->jrdev);
   3362	}
   3363
   3364	priv = dev_get_drvdata(ctx->jrdev->parent);
   3365	if (priv->era >= 6 && uses_dkp)
   3366		ctx->dir = DMA_BIDIRECTIONAL;
   3367	else
   3368		ctx->dir = DMA_TO_DEVICE;
   3369
   3370	dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
   3371					offsetof(struct caam_ctx,
   3372						 sh_desc_enc_dma) -
   3373					sh_desc_enc_offset,
   3374					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
   3375	if (dma_mapping_error(ctx->jrdev, dma_addr)) {
   3376		dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
   3377		caam_jr_free(ctx->jrdev);
   3378		return -ENOMEM;
   3379	}
   3380
   3381	ctx->sh_desc_enc_dma = dma_addr;
   3382	ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
   3383						   sh_desc_dec) -
   3384					sh_desc_enc_offset;
   3385	ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) -
   3386					sh_desc_enc_offset;
   3387
   3388	/* copy descriptor header template value */
   3389	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
   3390	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
   3391
   3392	return 0;
   3393}
   3394
   3395static int caam_cra_init(struct crypto_skcipher *tfm)
   3396{
   3397	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
   3398	struct caam_skcipher_alg *caam_alg =
   3399		container_of(alg, typeof(*caam_alg), skcipher);
   3400	struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
   3401	u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
   3402	int ret = 0;
   3403
   3404	ctx->enginectx.op.do_one_request = skcipher_do_one_req;
   3405
   3406	if (alg_aai == OP_ALG_AAI_XTS) {
   3407		const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
   3408		struct crypto_skcipher *fallback;
   3409
   3410		fallback = crypto_alloc_skcipher(tfm_name, 0,
   3411						 CRYPTO_ALG_NEED_FALLBACK);
   3412		if (IS_ERR(fallback)) {
   3413			pr_err("Failed to allocate %s fallback: %ld\n",
   3414			       tfm_name, PTR_ERR(fallback));
   3415			return PTR_ERR(fallback);
   3416		}
   3417
   3418		ctx->fallback = fallback;
   3419		crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
   3420					    crypto_skcipher_reqsize(fallback));
   3421	} else {
   3422		crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx));
   3423	}
   3424
   3425	ret = caam_init_common(ctx, &caam_alg->caam, false);
   3426	if (ret && ctx->fallback)
   3427		crypto_free_skcipher(ctx->fallback);
   3428
   3429	return ret;
   3430}
   3431
   3432static int caam_aead_init(struct crypto_aead *tfm)
   3433{
   3434	struct aead_alg *alg = crypto_aead_alg(tfm);
   3435	struct caam_aead_alg *caam_alg =
   3436		 container_of(alg, struct caam_aead_alg, aead);
   3437	struct caam_ctx *ctx = crypto_aead_ctx(tfm);
   3438
   3439	crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx));
   3440
   3441	ctx->enginectx.op.do_one_request = aead_do_one_req;
   3442
   3443	return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
   3444}
   3445
   3446static void caam_exit_common(struct caam_ctx *ctx)
   3447{
   3448	dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
   3449			       offsetof(struct caam_ctx, sh_desc_enc_dma) -
   3450			       offsetof(struct caam_ctx, sh_desc_enc),
   3451			       ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
   3452	caam_jr_free(ctx->jrdev);
   3453}
   3454
   3455static void caam_cra_exit(struct crypto_skcipher *tfm)
   3456{
   3457	struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
   3458
   3459	if (ctx->fallback)
   3460		crypto_free_skcipher(ctx->fallback);
   3461	caam_exit_common(ctx);
   3462}
   3463
   3464static void caam_aead_exit(struct crypto_aead *tfm)
   3465{
   3466	caam_exit_common(crypto_aead_ctx(tfm));
   3467}
   3468
   3469void caam_algapi_exit(void)
   3470{
   3471	int i;
   3472
   3473	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
   3474		struct caam_aead_alg *t_alg = driver_aeads + i;
   3475
   3476		if (t_alg->registered)
   3477			crypto_unregister_aead(&t_alg->aead);
   3478	}
   3479
   3480	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
   3481		struct caam_skcipher_alg *t_alg = driver_algs + i;
   3482
   3483		if (t_alg->registered)
   3484			crypto_unregister_skcipher(&t_alg->skcipher);
   3485	}
   3486}
   3487
   3488static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
   3489{
   3490	struct skcipher_alg *alg = &t_alg->skcipher;
   3491
   3492	alg->base.cra_module = THIS_MODULE;
   3493	alg->base.cra_priority = CAAM_CRA_PRIORITY;
   3494	alg->base.cra_ctxsize = sizeof(struct caam_ctx);
   3495	alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
   3496			      CRYPTO_ALG_KERN_DRIVER_ONLY);
   3497
   3498	alg->init = caam_cra_init;
   3499	alg->exit = caam_cra_exit;
   3500}
   3501
   3502static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
   3503{
   3504	struct aead_alg *alg = &t_alg->aead;
   3505
   3506	alg->base.cra_module = THIS_MODULE;
   3507	alg->base.cra_priority = CAAM_CRA_PRIORITY;
   3508	alg->base.cra_ctxsize = sizeof(struct caam_ctx);
   3509	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
   3510			      CRYPTO_ALG_KERN_DRIVER_ONLY;
   3511
   3512	alg->init = caam_aead_init;
   3513	alg->exit = caam_aead_exit;
   3514}
   3515
   3516int caam_algapi_init(struct device *ctrldev)
   3517{
   3518	struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
   3519	int i = 0, err = 0;
   3520	u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
   3521	unsigned int md_limit = SHA512_DIGEST_SIZE;
   3522	bool registered = false, gcm_support;
   3523
   3524	/*
   3525	 * Register crypto algorithms the device supports.
   3526	 * First, detect presence and attributes of DES, AES, and MD blocks.
   3527	 */
   3528	if (priv->era < 10) {
   3529		u32 cha_vid, cha_inst, aes_rn;
   3530
   3531		cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
   3532		aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
   3533		md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
   3534
   3535		cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
   3536		des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
   3537			   CHA_ID_LS_DES_SHIFT;
   3538		aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
   3539		md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
   3540		ccha_inst = 0;
   3541		ptha_inst = 0;
   3542
   3543		aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) &
   3544			 CHA_ID_LS_AES_MASK;
   3545		gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
   3546	} else {
   3547		u32 aesa, mdha;
   3548
   3549		aesa = rd_reg32(&priv->ctrl->vreg.aesa);
   3550		mdha = rd_reg32(&priv->ctrl->vreg.mdha);
   3551
   3552		aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
   3553		md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
   3554
   3555		des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
   3556		aes_inst = aesa & CHA_VER_NUM_MASK;
   3557		md_inst = mdha & CHA_VER_NUM_MASK;
   3558		ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
   3559		ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
   3560
   3561		gcm_support = aesa & CHA_VER_MISC_AES_GCM;
   3562	}
   3563
   3564	/* If MD is present, limit digest size based on LP256 */
   3565	if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
   3566		md_limit = SHA256_DIGEST_SIZE;
   3567
   3568	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
   3569		struct caam_skcipher_alg *t_alg = driver_algs + i;
   3570		u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
   3571
   3572		/* Skip DES algorithms if not supported by device */
   3573		if (!des_inst &&
   3574		    ((alg_sel == OP_ALG_ALGSEL_3DES) ||
   3575		     (alg_sel == OP_ALG_ALGSEL_DES)))
   3576				continue;
   3577
   3578		/* Skip AES algorithms if not supported by device */
   3579		if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
   3580				continue;
   3581
   3582		/*
   3583		 * Check support for AES modes not available
   3584		 * on LP devices.
   3585		 */
   3586		if (aes_vid == CHA_VER_VID_AES_LP &&
   3587		    (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
   3588		    OP_ALG_AAI_XTS)
   3589			continue;
   3590
   3591		caam_skcipher_alg_init(t_alg);
   3592
   3593		err = crypto_register_skcipher(&t_alg->skcipher);
   3594		if (err) {
   3595			pr_warn("%s alg registration failed\n",
   3596				t_alg->skcipher.base.cra_driver_name);
   3597			continue;
   3598		}
   3599
   3600		t_alg->registered = true;
   3601		registered = true;
   3602	}
   3603
   3604	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
   3605		struct caam_aead_alg *t_alg = driver_aeads + i;
   3606		u32 c1_alg_sel = t_alg->caam.class1_alg_type &
   3607				 OP_ALG_ALGSEL_MASK;
   3608		u32 c2_alg_sel = t_alg->caam.class2_alg_type &
   3609				 OP_ALG_ALGSEL_MASK;
   3610		u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
   3611
   3612		/* Skip DES algorithms if not supported by device */
   3613		if (!des_inst &&
   3614		    ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
   3615		     (c1_alg_sel == OP_ALG_ALGSEL_DES)))
   3616				continue;
   3617
   3618		/* Skip AES algorithms if not supported by device */
   3619		if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
   3620				continue;
   3621
   3622		/* Skip CHACHA20 algorithms if not supported by device */
   3623		if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
   3624			continue;
   3625
   3626		/* Skip POLY1305 algorithms if not supported by device */
   3627		if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
   3628			continue;
   3629
   3630		/* Skip GCM algorithms if not supported by device */
   3631		if (c1_alg_sel == OP_ALG_ALGSEL_AES &&
   3632		    alg_aai == OP_ALG_AAI_GCM && !gcm_support)
   3633			continue;
   3634
   3635		/*
   3636		 * Skip algorithms requiring message digests
   3637		 * if MD or MD size is not supported by device.
   3638		 */
   3639		if (is_mdha(c2_alg_sel) &&
   3640		    (!md_inst || t_alg->aead.maxauthsize > md_limit))
   3641			continue;
   3642
   3643		caam_aead_alg_init(t_alg);
   3644
   3645		err = crypto_register_aead(&t_alg->aead);
   3646		if (err) {
   3647			pr_warn("%s alg registration failed\n",
   3648				t_alg->aead.base.cra_driver_name);
   3649			continue;
   3650		}
   3651
   3652		t_alg->registered = true;
   3653		registered = true;
   3654	}
   3655
   3656	if (registered)
   3657		pr_info("caam algorithms registered in /proc/crypto\n");
   3658
   3659	return err;
   3660}