cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

tcrypt.c (76569B)

      1// SPDX-License-Identifier: GPL-2.0-or-later
      2/*
      3 * Quick & dirty crypto testing module.
      4 *
      5 * This will only exist until we have a better testing mechanism
      6 * (e.g. a char device).
      7 *
      8 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
      9 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
     10 * Copyright (c) 2007 Nokia Siemens Networks
     11 *
     12 * Updated RFC4106 AES-GCM testing.
     13 *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
     14 *             Adrian Hoban <adrian.hoban@intel.com>
     15 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
     16 *             Tadeusz Struk (tadeusz.struk@intel.com)
     17 *             Copyright (c) 2010, Intel Corporation.
     18 */
     19
     20#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
     21
     22#include <crypto/aead.h>
     23#include <crypto/hash.h>
     24#include <crypto/skcipher.h>
     25#include <linux/err.h>
     26#include <linux/fips.h>
     27#include <linux/init.h>
     28#include <linux/gfp.h>
     29#include <linux/module.h>
     30#include <linux/scatterlist.h>
     31#include <linux/string.h>
     32#include <linux/moduleparam.h>
     33#include <linux/jiffies.h>
     34#include <linux/timex.h>
     35#include <linux/interrupt.h>
     36#include "tcrypt.h"
     37
     38/*
     39 * Need slab memory for testing (size in number of pages).
     40 */
     41#define TVMEMSIZE	4
     42
     43/*
     44* Used by test_cipher_speed()
     45*/
     46#define ENCRYPT 1
     47#define DECRYPT 0
     48
     49#define MAX_DIGEST_SIZE		64
     50
     51/*
     52 * return a string with the driver name
     53 */
     54#define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))
     55
     56/*
     57 * Used by test_cipher_speed()
     58 */
     59static unsigned int sec;
     60
     61static char *alg = NULL;
     62static u32 type;
     63static u32 mask;
     64static int mode;
     65static u32 num_mb = 8;
     66static unsigned int klen;
     67static char *tvmem[TVMEMSIZE];
     68
     69static const char *check[] = {
     70	"des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3",
     71	"blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes",
     72	"cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",
     73	"khazad", "wp512", "wp384", "wp256", "xeta",  "fcrypt",
     74	"camellia", "seed", "rmd160",
     75	"lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384",
     76	"sha3-512", "streebog256", "streebog512",
     77	NULL
     78};
     79
     80static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 };
     81static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 };
     82
     83#define XBUFSIZE 8
     84#define MAX_IVLEN 32
     85
     86static int testmgr_alloc_buf(char *buf[XBUFSIZE])
     87{
     88	int i;
     89
     90	for (i = 0; i < XBUFSIZE; i++) {
     91		buf[i] = (void *)__get_free_page(GFP_KERNEL);
     92		if (!buf[i])
     93			goto err_free_buf;
     94	}
     95
     96	return 0;
     97
     98err_free_buf:
     99	while (i-- > 0)
    100		free_page((unsigned long)buf[i]);
    101
    102	return -ENOMEM;
    103}
    104
    105static void testmgr_free_buf(char *buf[XBUFSIZE])
    106{
    107	int i;
    108
    109	for (i = 0; i < XBUFSIZE; i++)
    110		free_page((unsigned long)buf[i]);
    111}
    112
    113static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
    114			 unsigned int buflen, const void *assoc,
    115			 unsigned int aad_size)
    116{
    117	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
    118	int k, rem;
    119
    120	if (np > XBUFSIZE) {
    121		rem = PAGE_SIZE;
    122		np = XBUFSIZE;
    123	} else {
    124		rem = buflen % PAGE_SIZE;
    125	}
    126
    127	sg_init_table(sg, np + 1);
    128
    129	sg_set_buf(&sg[0], assoc, aad_size);
    130
    131	if (rem)
    132		np--;
    133	for (k = 0; k < np; k++)
    134		sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
    135
    136	if (rem)
    137		sg_set_buf(&sg[k + 1], xbuf[k], rem);
    138}
    139
    140static inline int do_one_aead_op(struct aead_request *req, int ret)
    141{
    142	struct crypto_wait *wait = req->base.data;
    143
    144	return crypto_wait_req(ret, wait);
    145}
    146
    147struct test_mb_aead_data {
    148	struct scatterlist sg[XBUFSIZE];
    149	struct scatterlist sgout[XBUFSIZE];
    150	struct aead_request *req;
    151	struct crypto_wait wait;
    152	char *xbuf[XBUFSIZE];
    153	char *xoutbuf[XBUFSIZE];
    154	char *axbuf[XBUFSIZE];
    155};
    156
    157static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
    158				u32 num_mb, int *rc)
    159{
    160	int i, err = 0;
    161
    162	/* Fire up a bunch of concurrent requests */
    163	for (i = 0; i < num_mb; i++) {
    164		if (enc == ENCRYPT)
    165			rc[i] = crypto_aead_encrypt(data[i].req);
    166		else
    167			rc[i] = crypto_aead_decrypt(data[i].req);
    168	}
    169
    170	/* Wait for all requests to finish */
    171	for (i = 0; i < num_mb; i++) {
    172		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
    173
    174		if (rc[i]) {
    175			pr_info("concurrent request %d error %d\n", i, rc[i]);
    176			err = rc[i];
    177		}
    178	}
    179
    180	return err;
    181}
    182
    183static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
    184				int blen, int secs, u32 num_mb)
    185{
    186	unsigned long start, end;
    187	int bcount;
    188	int ret = 0;
    189	int *rc;
    190
    191	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
    192	if (!rc)
    193		return -ENOMEM;
    194
    195	for (start = jiffies, end = start + secs * HZ, bcount = 0;
    196	     time_before(jiffies, end); bcount++) {
    197		ret = do_mult_aead_op(data, enc, num_mb, rc);
    198		if (ret)
    199			goto out;
    200	}
    201
    202	pr_cont("%d operations in %d seconds (%llu bytes)\n",
    203		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
    204
    205out:
    206	kfree(rc);
    207	return ret;
    208}
    209
    210static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
    211			       int blen, u32 num_mb)
    212{
    213	unsigned long cycles = 0;
    214	int ret = 0;
    215	int i;
    216	int *rc;
    217
    218	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
    219	if (!rc)
    220		return -ENOMEM;
    221
    222	/* Warm-up run. */
    223	for (i = 0; i < 4; i++) {
    224		ret = do_mult_aead_op(data, enc, num_mb, rc);
    225		if (ret)
    226			goto out;
    227	}
    228
    229	/* The real thing. */
    230	for (i = 0; i < 8; i++) {
    231		cycles_t start, end;
    232
    233		start = get_cycles();
    234		ret = do_mult_aead_op(data, enc, num_mb, rc);
    235		end = get_cycles();
    236
    237		if (ret)
    238			goto out;
    239
    240		cycles += end - start;
    241	}
    242
    243	pr_cont("1 operation in %lu cycles (%d bytes)\n",
    244		(cycles + 4) / (8 * num_mb), blen);
    245
    246out:
    247	kfree(rc);
    248	return ret;
    249}
    250
    251static void test_mb_aead_speed(const char *algo, int enc, int secs,
    252			       struct aead_speed_template *template,
    253			       unsigned int tcount, u8 authsize,
    254			       unsigned int aad_size, u8 *keysize, u32 num_mb)
    255{
    256	struct test_mb_aead_data *data;
    257	struct crypto_aead *tfm;
    258	unsigned int i, j, iv_len;
    259	const int *b_size;
    260	const char *key;
    261	const char *e;
    262	void *assoc;
    263	char *iv;
    264	int ret;
    265
    266
    267	if (aad_size >= PAGE_SIZE) {
    268		pr_err("associate data length (%u) too big\n", aad_size);
    269		return;
    270	}
    271
    272	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
    273	if (!iv)
    274		return;
    275
    276	if (enc == ENCRYPT)
    277		e = "encryption";
    278	else
    279		e = "decryption";
    280
    281	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
    282	if (!data)
    283		goto out_free_iv;
    284
    285	tfm = crypto_alloc_aead(algo, 0, 0);
    286	if (IS_ERR(tfm)) {
    287		pr_err("failed to load transform for %s: %ld\n",
    288			algo, PTR_ERR(tfm));
    289		goto out_free_data;
    290	}
    291
    292	ret = crypto_aead_setauthsize(tfm, authsize);
    293	if (ret) {
    294		pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
    295		       ret);
    296		goto out_free_tfm;
    297	}
    298
    299	for (i = 0; i < num_mb; ++i)
    300		if (testmgr_alloc_buf(data[i].xbuf)) {
    301			while (i--)
    302				testmgr_free_buf(data[i].xbuf);
    303			goto out_free_tfm;
    304		}
    305
    306	for (i = 0; i < num_mb; ++i)
    307		if (testmgr_alloc_buf(data[i].axbuf)) {
    308			while (i--)
    309				testmgr_free_buf(data[i].axbuf);
    310			goto out_free_xbuf;
    311		}
    312
    313	for (i = 0; i < num_mb; ++i)
    314		if (testmgr_alloc_buf(data[i].xoutbuf)) {
    315			while (i--)
    316				testmgr_free_buf(data[i].xoutbuf);
    317			goto out_free_axbuf;
    318		}
    319
    320	for (i = 0; i < num_mb; ++i) {
    321		data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
    322		if (!data[i].req) {
    323			pr_err("alg: aead: Failed to allocate request for %s\n",
    324			       algo);
    325			while (i--)
    326				aead_request_free(data[i].req);
    327			goto out_free_xoutbuf;
    328		}
    329	}
    330
    331	for (i = 0; i < num_mb; ++i) {
    332		crypto_init_wait(&data[i].wait);
    333		aead_request_set_callback(data[i].req,
    334					  CRYPTO_TFM_REQ_MAY_BACKLOG,
    335					  crypto_req_done, &data[i].wait);
    336	}
    337
    338	pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
    339		get_driver_name(crypto_aead, tfm), e);
    340
    341	i = 0;
    342	do {
    343		b_size = aead_sizes;
    344		do {
    345			int bs = round_up(*b_size, crypto_aead_blocksize(tfm));
    346
    347			if (bs + authsize > XBUFSIZE * PAGE_SIZE) {
    348				pr_err("template (%u) too big for buffer (%lu)\n",
    349				       authsize + bs,
    350				       XBUFSIZE * PAGE_SIZE);
    351				goto out;
    352			}
    353
    354			pr_info("test %u (%d bit key, %d byte blocks): ", i,
    355				*keysize * 8, bs);
    356
    357			/* Set up tfm global state, i.e. the key */
    358
    359			memset(tvmem[0], 0xff, PAGE_SIZE);
    360			key = tvmem[0];
    361			for (j = 0; j < tcount; j++) {
    362				if (template[j].klen == *keysize) {
    363					key = template[j].key;
    364					break;
    365				}
    366			}
    367
    368			crypto_aead_clear_flags(tfm, ~0);
    369
    370			ret = crypto_aead_setkey(tfm, key, *keysize);
    371			if (ret) {
    372				pr_err("setkey() failed flags=%x\n",
    373				       crypto_aead_get_flags(tfm));
    374				goto out;
    375			}
    376
    377			iv_len = crypto_aead_ivsize(tfm);
    378			if (iv_len)
    379				memset(iv, 0xff, iv_len);
    380
    381			/* Now setup per request stuff, i.e. buffers */
    382
    383			for (j = 0; j < num_mb; ++j) {
    384				struct test_mb_aead_data *cur = &data[j];
    385
    386				assoc = cur->axbuf[0];
    387				memset(assoc, 0xff, aad_size);
    388
    389				sg_init_aead(cur->sg, cur->xbuf,
    390					     bs + (enc ? 0 : authsize),
    391					     assoc, aad_size);
    392
    393				sg_init_aead(cur->sgout, cur->xoutbuf,
    394					     bs + (enc ? authsize : 0),
    395					     assoc, aad_size);
    396
    397				aead_request_set_ad(cur->req, aad_size);
    398
    399				if (!enc) {
    400
    401					aead_request_set_crypt(cur->req,
    402							       cur->sgout,
    403							       cur->sg,
    404							       bs, iv);
    405					ret = crypto_aead_encrypt(cur->req);
    406					ret = do_one_aead_op(cur->req, ret);
    407
    408					if (ret) {
    409						pr_err("calculating auth failed (%d)\n",
    410						       ret);
    411						break;
    412					}
    413				}
    414
    415				aead_request_set_crypt(cur->req, cur->sg,
    416						       cur->sgout, bs +
    417						       (enc ? 0 : authsize),
    418						       iv);
    419
    420			}
    421
    422			if (secs) {
    423				ret = test_mb_aead_jiffies(data, enc, bs,
    424							   secs, num_mb);
    425				cond_resched();
    426			} else {
    427				ret = test_mb_aead_cycles(data, enc, bs,
    428							  num_mb);
    429			}
    430
    431			if (ret) {
    432				pr_err("%s() failed return code=%d\n", e, ret);
    433				break;
    434			}
    435			b_size++;
    436			i++;
    437		} while (*b_size);
    438		keysize++;
    439	} while (*keysize);
    440
    441out:
    442	for (i = 0; i < num_mb; ++i)
    443		aead_request_free(data[i].req);
    444out_free_xoutbuf:
    445	for (i = 0; i < num_mb; ++i)
    446		testmgr_free_buf(data[i].xoutbuf);
    447out_free_axbuf:
    448	for (i = 0; i < num_mb; ++i)
    449		testmgr_free_buf(data[i].axbuf);
    450out_free_xbuf:
    451	for (i = 0; i < num_mb; ++i)
    452		testmgr_free_buf(data[i].xbuf);
    453out_free_tfm:
    454	crypto_free_aead(tfm);
    455out_free_data:
    456	kfree(data);
    457out_free_iv:
    458	kfree(iv);
    459}
    460
    461static int test_aead_jiffies(struct aead_request *req, int enc,
    462				int blen, int secs)
    463{
    464	unsigned long start, end;
    465	int bcount;
    466	int ret;
    467
    468	for (start = jiffies, end = start + secs * HZ, bcount = 0;
    469	     time_before(jiffies, end); bcount++) {
    470		if (enc)
    471			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
    472		else
    473			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
    474
    475		if (ret)
    476			return ret;
    477	}
    478
    479	pr_cont("%d operations in %d seconds (%llu bytes)\n",
    480	        bcount, secs, (u64)bcount * blen);
    481	return 0;
    482}
    483
    484static int test_aead_cycles(struct aead_request *req, int enc, int blen)
    485{
    486	unsigned long cycles = 0;
    487	int ret = 0;
    488	int i;
    489
    490	/* Warm-up run. */
    491	for (i = 0; i < 4; i++) {
    492		if (enc)
    493			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
    494		else
    495			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
    496
    497		if (ret)
    498			goto out;
    499	}
    500
    501	/* The real thing. */
    502	for (i = 0; i < 8; i++) {
    503		cycles_t start, end;
    504
    505		start = get_cycles();
    506		if (enc)
    507			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
    508		else
    509			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
    510		end = get_cycles();
    511
    512		if (ret)
    513			goto out;
    514
    515		cycles += end - start;
    516	}
    517
    518out:
    519	if (ret == 0)
    520		printk("1 operation in %lu cycles (%d bytes)\n",
    521		       (cycles + 4) / 8, blen);
    522
    523	return ret;
    524}
    525
    526static void test_aead_speed(const char *algo, int enc, unsigned int secs,
    527			    struct aead_speed_template *template,
    528			    unsigned int tcount, u8 authsize,
    529			    unsigned int aad_size, u8 *keysize)
    530{
    531	unsigned int i, j;
    532	struct crypto_aead *tfm;
    533	int ret = -ENOMEM;
    534	const char *key;
    535	struct aead_request *req;
    536	struct scatterlist *sg;
    537	struct scatterlist *sgout;
    538	const char *e;
    539	void *assoc;
    540	char *iv;
    541	char *xbuf[XBUFSIZE];
    542	char *xoutbuf[XBUFSIZE];
    543	char *axbuf[XBUFSIZE];
    544	const int *b_size;
    545	unsigned int iv_len;
    546	struct crypto_wait wait;
    547
    548	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
    549	if (!iv)
    550		return;
    551
    552	if (aad_size >= PAGE_SIZE) {
    553		pr_err("associate data length (%u) too big\n", aad_size);
    554		goto out_noxbuf;
    555	}
    556
    557	if (enc == ENCRYPT)
    558		e = "encryption";
    559	else
    560		e = "decryption";
    561
    562	if (testmgr_alloc_buf(xbuf))
    563		goto out_noxbuf;
    564	if (testmgr_alloc_buf(axbuf))
    565		goto out_noaxbuf;
    566	if (testmgr_alloc_buf(xoutbuf))
    567		goto out_nooutbuf;
    568
    569	sg = kmalloc(sizeof(*sg) * 9 * 2, GFP_KERNEL);
    570	if (!sg)
    571		goto out_nosg;
    572	sgout = &sg[9];
    573
    574	tfm = crypto_alloc_aead(algo, 0, 0);
    575	if (IS_ERR(tfm)) {
    576		pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
    577		       PTR_ERR(tfm));
    578		goto out_notfm;
    579	}
    580
    581	ret = crypto_aead_setauthsize(tfm, authsize);
    582	if (ret) {
    583		pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
    584		       ret);
    585		goto out_noreq;
    586	}
    587
    588	crypto_init_wait(&wait);
    589	printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo,
    590			get_driver_name(crypto_aead, tfm), e);
    591
    592	req = aead_request_alloc(tfm, GFP_KERNEL);
    593	if (!req) {
    594		pr_err("alg: aead: Failed to allocate request for %s\n",
    595		       algo);
    596		goto out_noreq;
    597	}
    598
    599	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
    600				  crypto_req_done, &wait);
    601
    602	i = 0;
    603	do {
    604		b_size = aead_sizes;
    605		do {
    606			u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm));
    607
    608			assoc = axbuf[0];
    609			memset(assoc, 0xff, aad_size);
    610
    611			if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
    612				pr_err("template (%u) too big for tvmem (%lu)\n",
    613				       *keysize + bs,
    614					TVMEMSIZE * PAGE_SIZE);
    615				goto out;
    616			}
    617
    618			key = tvmem[0];
    619			for (j = 0; j < tcount; j++) {
    620				if (template[j].klen == *keysize) {
    621					key = template[j].key;
    622					break;
    623				}
    624			}
    625
    626			ret = crypto_aead_setkey(tfm, key, *keysize);
    627			if (ret) {
    628				pr_err("setkey() failed flags=%x: %d\n",
    629					crypto_aead_get_flags(tfm), ret);
    630				goto out;
    631			}
    632
    633			iv_len = crypto_aead_ivsize(tfm);
    634			if (iv_len)
    635				memset(iv, 0xff, iv_len);
    636
    637			crypto_aead_clear_flags(tfm, ~0);
    638			printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
    639					i, *keysize * 8, bs);
    640
    641			memset(tvmem[0], 0xff, PAGE_SIZE);
    642
    643			sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize),
    644				     assoc, aad_size);
    645
    646			sg_init_aead(sgout, xoutbuf,
    647				     bs + (enc ? authsize : 0), assoc,
    648				     aad_size);
    649
    650			aead_request_set_ad(req, aad_size);
    651
    652			if (!enc) {
    653
    654				/*
    655				 * For decryption we need a proper auth so
    656				 * we do the encryption path once with buffers
    657				 * reversed (input <-> output) to calculate it
    658				 */
    659				aead_request_set_crypt(req, sgout, sg,
    660						       bs, iv);
    661				ret = do_one_aead_op(req,
    662						     crypto_aead_encrypt(req));
    663
    664				if (ret) {
    665					pr_err("calculating auth failed (%d)\n",
    666					       ret);
    667					break;
    668				}
    669			}
    670
    671			aead_request_set_crypt(req, sg, sgout,
    672					       bs + (enc ? 0 : authsize),
    673					       iv);
    674
    675			if (secs) {
    676				ret = test_aead_jiffies(req, enc, bs,
    677							secs);
    678				cond_resched();
    679			} else {
    680				ret = test_aead_cycles(req, enc, bs);
    681			}
    682
    683			if (ret) {
    684				pr_err("%s() failed return code=%d\n", e, ret);
    685				break;
    686			}
    687			b_size++;
    688			i++;
    689		} while (*b_size);
    690		keysize++;
    691	} while (*keysize);
    692
    693out:
    694	aead_request_free(req);
    695out_noreq:
    696	crypto_free_aead(tfm);
    697out_notfm:
    698	kfree(sg);
    699out_nosg:
    700	testmgr_free_buf(xoutbuf);
    701out_nooutbuf:
    702	testmgr_free_buf(axbuf);
    703out_noaxbuf:
    704	testmgr_free_buf(xbuf);
    705out_noxbuf:
    706	kfree(iv);
    707}
    708
    709static void test_hash_sg_init(struct scatterlist *sg)
    710{
    711	int i;
    712
    713	sg_init_table(sg, TVMEMSIZE);
    714	for (i = 0; i < TVMEMSIZE; i++) {
    715		sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
    716		memset(tvmem[i], 0xff, PAGE_SIZE);
    717	}
    718}
    719
    720static inline int do_one_ahash_op(struct ahash_request *req, int ret)
    721{
    722	struct crypto_wait *wait = req->base.data;
    723
    724	return crypto_wait_req(ret, wait);
    725}
    726
    727static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
    728				     char *out, int secs)
    729{
    730	unsigned long start, end;
    731	int bcount;
    732	int ret;
    733
    734	for (start = jiffies, end = start + secs * HZ, bcount = 0;
    735	     time_before(jiffies, end); bcount++) {
    736		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
    737		if (ret)
    738			return ret;
    739	}
    740
    741	printk("%6u opers/sec, %9lu bytes/sec\n",
    742	       bcount / secs, ((long)bcount * blen) / secs);
    743
    744	return 0;
    745}
    746
    747static int test_ahash_jiffies(struct ahash_request *req, int blen,
    748			      int plen, char *out, int secs)
    749{
    750	unsigned long start, end;
    751	int bcount, pcount;
    752	int ret;
    753
    754	if (plen == blen)
    755		return test_ahash_jiffies_digest(req, blen, out, secs);
    756
    757	for (start = jiffies, end = start + secs * HZ, bcount = 0;
    758	     time_before(jiffies, end); bcount++) {
    759		ret = do_one_ahash_op(req, crypto_ahash_init(req));
    760		if (ret)
    761			return ret;
    762		for (pcount = 0; pcount < blen; pcount += plen) {
    763			ret = do_one_ahash_op(req, crypto_ahash_update(req));
    764			if (ret)
    765				return ret;
    766		}
    767		/* we assume there is enough space in 'out' for the result */
    768		ret = do_one_ahash_op(req, crypto_ahash_final(req));
    769		if (ret)
    770			return ret;
    771	}
    772
    773	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
    774		bcount / secs, ((long)bcount * blen) / secs);
    775
    776	return 0;
    777}
    778
    779static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
    780				    char *out)
    781{
    782	unsigned long cycles = 0;
    783	int ret, i;
    784
    785	/* Warm-up run. */
    786	for (i = 0; i < 4; i++) {
    787		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
    788		if (ret)
    789			goto out;
    790	}
    791
    792	/* The real thing. */
    793	for (i = 0; i < 8; i++) {
    794		cycles_t start, end;
    795
    796		start = get_cycles();
    797
    798		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
    799		if (ret)
    800			goto out;
    801
    802		end = get_cycles();
    803
    804		cycles += end - start;
    805	}
    806
    807out:
    808	if (ret)
    809		return ret;
    810
    811	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
    812		cycles / 8, cycles / (8 * blen));
    813
    814	return 0;
    815}
    816
    817static int test_ahash_cycles(struct ahash_request *req, int blen,
    818			     int plen, char *out)
    819{
    820	unsigned long cycles = 0;
    821	int i, pcount, ret;
    822
    823	if (plen == blen)
    824		return test_ahash_cycles_digest(req, blen, out);
    825
    826	/* Warm-up run. */
    827	for (i = 0; i < 4; i++) {
    828		ret = do_one_ahash_op(req, crypto_ahash_init(req));
    829		if (ret)
    830			goto out;
    831		for (pcount = 0; pcount < blen; pcount += plen) {
    832			ret = do_one_ahash_op(req, crypto_ahash_update(req));
    833			if (ret)
    834				goto out;
    835		}
    836		ret = do_one_ahash_op(req, crypto_ahash_final(req));
    837		if (ret)
    838			goto out;
    839	}
    840
    841	/* The real thing. */
    842	for (i = 0; i < 8; i++) {
    843		cycles_t start, end;
    844
    845		start = get_cycles();
    846
    847		ret = do_one_ahash_op(req, crypto_ahash_init(req));
    848		if (ret)
    849			goto out;
    850		for (pcount = 0; pcount < blen; pcount += plen) {
    851			ret = do_one_ahash_op(req, crypto_ahash_update(req));
    852			if (ret)
    853				goto out;
    854		}
    855		ret = do_one_ahash_op(req, crypto_ahash_final(req));
    856		if (ret)
    857			goto out;
    858
    859		end = get_cycles();
    860
    861		cycles += end - start;
    862	}
    863
    864out:
    865	if (ret)
    866		return ret;
    867
    868	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
    869		cycles / 8, cycles / (8 * blen));
    870
    871	return 0;
    872}
    873
    874static void test_ahash_speed_common(const char *algo, unsigned int secs,
    875				    struct hash_speed *speed, unsigned mask)
    876{
    877	struct scatterlist sg[TVMEMSIZE];
    878	struct crypto_wait wait;
    879	struct ahash_request *req;
    880	struct crypto_ahash *tfm;
    881	char *output;
    882	int i, ret;
    883
    884	tfm = crypto_alloc_ahash(algo, 0, mask);
    885	if (IS_ERR(tfm)) {
    886		pr_err("failed to load transform for %s: %ld\n",
    887		       algo, PTR_ERR(tfm));
    888		return;
    889	}
    890
    891	printk(KERN_INFO "\ntesting speed of async %s (%s)\n", algo,
    892			get_driver_name(crypto_ahash, tfm));
    893
    894	if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
    895		pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
    896		       MAX_DIGEST_SIZE);
    897		goto out;
    898	}
    899
    900	test_hash_sg_init(sg);
    901	req = ahash_request_alloc(tfm, GFP_KERNEL);
    902	if (!req) {
    903		pr_err("ahash request allocation failure\n");
    904		goto out;
    905	}
    906
    907	crypto_init_wait(&wait);
    908	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
    909				   crypto_req_done, &wait);
    910
    911	output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
    912	if (!output)
    913		goto out_nomem;
    914
    915	for (i = 0; speed[i].blen != 0; i++) {
    916		if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
    917			pr_err("template (%u) too big for tvmem (%lu)\n",
    918			       speed[i].blen, TVMEMSIZE * PAGE_SIZE);
    919			break;
    920		}
    921
    922		if (klen)
    923			crypto_ahash_setkey(tfm, tvmem[0], klen);
    924
    925		pr_info("test%3u "
    926			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
    927			i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
    928
    929		ahash_request_set_crypt(req, sg, output, speed[i].plen);
    930
    931		if (secs) {
    932			ret = test_ahash_jiffies(req, speed[i].blen,
    933						 speed[i].plen, output, secs);
    934			cond_resched();
    935		} else {
    936			ret = test_ahash_cycles(req, speed[i].blen,
    937						speed[i].plen, output);
    938		}
    939
    940		if (ret) {
    941			pr_err("hashing failed ret=%d\n", ret);
    942			break;
    943		}
    944	}
    945
    946	kfree(output);
    947
    948out_nomem:
    949	ahash_request_free(req);
    950
    951out:
    952	crypto_free_ahash(tfm);
    953}
    954
    955static void test_ahash_speed(const char *algo, unsigned int secs,
    956			     struct hash_speed *speed)
    957{
    958	return test_ahash_speed_common(algo, secs, speed, 0);
    959}
    960
    961static void test_hash_speed(const char *algo, unsigned int secs,
    962			    struct hash_speed *speed)
    963{
    964	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
    965}
    966
    967struct test_mb_skcipher_data {
    968	struct scatterlist sg[XBUFSIZE];
    969	struct skcipher_request *req;
    970	struct crypto_wait wait;
    971	char *xbuf[XBUFSIZE];
    972};
    973
    974static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
    975				u32 num_mb, int *rc)
    976{
    977	int i, err = 0;
    978
    979	/* Fire up a bunch of concurrent requests */
    980	for (i = 0; i < num_mb; i++) {
    981		if (enc == ENCRYPT)
    982			rc[i] = crypto_skcipher_encrypt(data[i].req);
    983		else
    984			rc[i] = crypto_skcipher_decrypt(data[i].req);
    985	}
    986
    987	/* Wait for all requests to finish */
    988	for (i = 0; i < num_mb; i++) {
    989		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
    990
    991		if (rc[i]) {
    992			pr_info("concurrent request %d error %d\n", i, rc[i]);
    993			err = rc[i];
    994		}
    995	}
    996
    997	return err;
    998}
    999
   1000static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
   1001				int blen, int secs, u32 num_mb)
   1002{
   1003	unsigned long start, end;
   1004	int bcount;
   1005	int ret = 0;
   1006	int *rc;
   1007
   1008	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
   1009	if (!rc)
   1010		return -ENOMEM;
   1011
   1012	for (start = jiffies, end = start + secs * HZ, bcount = 0;
   1013	     time_before(jiffies, end); bcount++) {
   1014		ret = do_mult_acipher_op(data, enc, num_mb, rc);
   1015		if (ret)
   1016			goto out;
   1017	}
   1018
   1019	pr_cont("%d operations in %d seconds (%llu bytes)\n",
   1020		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
   1021
   1022out:
   1023	kfree(rc);
   1024	return ret;
   1025}
   1026
   1027static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
   1028			       int blen, u32 num_mb)
   1029{
   1030	unsigned long cycles = 0;
   1031	int ret = 0;
   1032	int i;
   1033	int *rc;
   1034
   1035	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
   1036	if (!rc)
   1037		return -ENOMEM;
   1038
   1039	/* Warm-up run. */
   1040	for (i = 0; i < 4; i++) {
   1041		ret = do_mult_acipher_op(data, enc, num_mb, rc);
   1042		if (ret)
   1043			goto out;
   1044	}
   1045
   1046	/* The real thing. */
   1047	for (i = 0; i < 8; i++) {
   1048		cycles_t start, end;
   1049
   1050		start = get_cycles();
   1051		ret = do_mult_acipher_op(data, enc, num_mb, rc);
   1052		end = get_cycles();
   1053
   1054		if (ret)
   1055			goto out;
   1056
   1057		cycles += end - start;
   1058	}
   1059
   1060	pr_cont("1 operation in %lu cycles (%d bytes)\n",
   1061		(cycles + 4) / (8 * num_mb), blen);
   1062
   1063out:
   1064	kfree(rc);
   1065	return ret;
   1066}
   1067
   1068static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
   1069				   struct cipher_speed_template *template,
   1070				   unsigned int tcount, u8 *keysize, u32 num_mb)
   1071{
   1072	struct test_mb_skcipher_data *data;
   1073	struct crypto_skcipher *tfm;
   1074	unsigned int i, j, iv_len;
   1075	const int *b_size;
   1076	const char *key;
   1077	const char *e;
   1078	char iv[128];
   1079	int ret;
   1080
   1081	if (enc == ENCRYPT)
   1082		e = "encryption";
   1083	else
   1084		e = "decryption";
   1085
   1086	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
   1087	if (!data)
   1088		return;
   1089
   1090	tfm = crypto_alloc_skcipher(algo, 0, 0);
   1091	if (IS_ERR(tfm)) {
   1092		pr_err("failed to load transform for %s: %ld\n",
   1093			algo, PTR_ERR(tfm));
   1094		goto out_free_data;
   1095	}
   1096
   1097	for (i = 0; i < num_mb; ++i)
   1098		if (testmgr_alloc_buf(data[i].xbuf)) {
   1099			while (i--)
   1100				testmgr_free_buf(data[i].xbuf);
   1101			goto out_free_tfm;
   1102		}
   1103
   1104
   1105	for (i = 0; i < num_mb; ++i)
   1106		if (testmgr_alloc_buf(data[i].xbuf)) {
   1107			while (i--)
   1108				testmgr_free_buf(data[i].xbuf);
   1109			goto out_free_tfm;
   1110		}
   1111
   1112
   1113	for (i = 0; i < num_mb; ++i) {
   1114		data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
   1115		if (!data[i].req) {
   1116			pr_err("alg: skcipher: Failed to allocate request for %s\n",
   1117			       algo);
   1118			while (i--)
   1119				skcipher_request_free(data[i].req);
   1120			goto out_free_xbuf;
   1121		}
   1122	}
   1123
   1124	for (i = 0; i < num_mb; ++i) {
   1125		skcipher_request_set_callback(data[i].req,
   1126					      CRYPTO_TFM_REQ_MAY_BACKLOG,
   1127					      crypto_req_done, &data[i].wait);
   1128		crypto_init_wait(&data[i].wait);
   1129	}
   1130
   1131	pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
   1132		get_driver_name(crypto_skcipher, tfm), e);
   1133
   1134	i = 0;
   1135	do {
   1136		b_size = block_sizes;
   1137		do {
   1138			u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
   1139
   1140			if (bs > XBUFSIZE * PAGE_SIZE) {
   1141				pr_err("template (%u) too big for buffer (%lu)\n",
   1142				       bs, XBUFSIZE * PAGE_SIZE);
   1143				goto out;
   1144			}
   1145
   1146			pr_info("test %u (%d bit key, %d byte blocks): ", i,
   1147				*keysize * 8, bs);
   1148
   1149			/* Set up tfm global state, i.e. the key */
   1150
   1151			memset(tvmem[0], 0xff, PAGE_SIZE);
   1152			key = tvmem[0];
   1153			for (j = 0; j < tcount; j++) {
   1154				if (template[j].klen == *keysize) {
   1155					key = template[j].key;
   1156					break;
   1157				}
   1158			}
   1159
   1160			crypto_skcipher_clear_flags(tfm, ~0);
   1161
   1162			ret = crypto_skcipher_setkey(tfm, key, *keysize);
   1163			if (ret) {
   1164				pr_err("setkey() failed flags=%x\n",
   1165				       crypto_skcipher_get_flags(tfm));
   1166				goto out;
   1167			}
   1168
   1169			iv_len = crypto_skcipher_ivsize(tfm);
   1170			if (iv_len)
   1171				memset(&iv, 0xff, iv_len);
   1172
   1173			/* Now setup per request stuff, i.e. buffers */
   1174
   1175			for (j = 0; j < num_mb; ++j) {
   1176				struct test_mb_skcipher_data *cur = &data[j];
   1177				unsigned int k = bs;
   1178				unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
   1179				unsigned int p = 0;
   1180
   1181				sg_init_table(cur->sg, pages);
   1182
   1183				while (k > PAGE_SIZE) {
   1184					sg_set_buf(cur->sg + p, cur->xbuf[p],
   1185						   PAGE_SIZE);
   1186					memset(cur->xbuf[p], 0xff, PAGE_SIZE);
   1187					p++;
   1188					k -= PAGE_SIZE;
   1189				}
   1190
   1191				sg_set_buf(cur->sg + p, cur->xbuf[p], k);
   1192				memset(cur->xbuf[p], 0xff, k);
   1193
   1194				skcipher_request_set_crypt(cur->req, cur->sg,
   1195							   cur->sg, bs, iv);
   1196			}
   1197
   1198			if (secs) {
   1199				ret = test_mb_acipher_jiffies(data, enc,
   1200							      bs, secs,
   1201							      num_mb);
   1202				cond_resched();
   1203			} else {
   1204				ret = test_mb_acipher_cycles(data, enc,
   1205							     bs, num_mb);
   1206			}
   1207
   1208			if (ret) {
   1209				pr_err("%s() failed flags=%x\n", e,
   1210				       crypto_skcipher_get_flags(tfm));
   1211				break;
   1212			}
   1213			b_size++;
   1214			i++;
   1215		} while (*b_size);
   1216		keysize++;
   1217	} while (*keysize);
   1218
   1219out:
   1220	for (i = 0; i < num_mb; ++i)
   1221		skcipher_request_free(data[i].req);
   1222out_free_xbuf:
   1223	for (i = 0; i < num_mb; ++i)
   1224		testmgr_free_buf(data[i].xbuf);
   1225out_free_tfm:
   1226	crypto_free_skcipher(tfm);
   1227out_free_data:
   1228	kfree(data);
   1229}
   1230
   1231static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
   1232{
   1233	struct crypto_wait *wait = req->base.data;
   1234
   1235	return crypto_wait_req(ret, wait);
   1236}
   1237
   1238static int test_acipher_jiffies(struct skcipher_request *req, int enc,
   1239				int blen, int secs)
   1240{
   1241	unsigned long start, end;
   1242	int bcount;
   1243	int ret;
   1244
   1245	for (start = jiffies, end = start + secs * HZ, bcount = 0;
   1246	     time_before(jiffies, end); bcount++) {
   1247		if (enc)
   1248			ret = do_one_acipher_op(req,
   1249						crypto_skcipher_encrypt(req));
   1250		else
   1251			ret = do_one_acipher_op(req,
   1252						crypto_skcipher_decrypt(req));
   1253
   1254		if (ret)
   1255			return ret;
   1256	}
   1257
   1258	pr_cont("%d operations in %d seconds (%llu bytes)\n",
   1259		bcount, secs, (u64)bcount * blen);
   1260	return 0;
   1261}
   1262
   1263static int test_acipher_cycles(struct skcipher_request *req, int enc,
   1264			       int blen)
   1265{
   1266	unsigned long cycles = 0;
   1267	int ret = 0;
   1268	int i;
   1269
   1270	/* Warm-up run. */
   1271	for (i = 0; i < 4; i++) {
   1272		if (enc)
   1273			ret = do_one_acipher_op(req,
   1274						crypto_skcipher_encrypt(req));
   1275		else
   1276			ret = do_one_acipher_op(req,
   1277						crypto_skcipher_decrypt(req));
   1278
   1279		if (ret)
   1280			goto out;
   1281	}
   1282
   1283	/* The real thing. */
   1284	for (i = 0; i < 8; i++) {
   1285		cycles_t start, end;
   1286
   1287		start = get_cycles();
   1288		if (enc)
   1289			ret = do_one_acipher_op(req,
   1290						crypto_skcipher_encrypt(req));
   1291		else
   1292			ret = do_one_acipher_op(req,
   1293						crypto_skcipher_decrypt(req));
   1294		end = get_cycles();
   1295
   1296		if (ret)
   1297			goto out;
   1298
   1299		cycles += end - start;
   1300	}
   1301
   1302out:
   1303	if (ret == 0)
   1304		pr_cont("1 operation in %lu cycles (%d bytes)\n",
   1305			(cycles + 4) / 8, blen);
   1306
   1307	return ret;
   1308}
   1309
   1310static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
   1311				struct cipher_speed_template *template,
   1312				unsigned int tcount, u8 *keysize, bool async)
   1313{
   1314	unsigned int ret, i, j, k, iv_len;
   1315	struct crypto_wait wait;
   1316	const char *key;
   1317	char iv[128];
   1318	struct skcipher_request *req;
   1319	struct crypto_skcipher *tfm;
   1320	const int *b_size;
   1321	const char *e;
   1322
   1323	if (enc == ENCRYPT)
   1324		e = "encryption";
   1325	else
   1326		e = "decryption";
   1327
   1328	crypto_init_wait(&wait);
   1329
   1330	tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
   1331
   1332	if (IS_ERR(tfm)) {
   1333		pr_err("failed to load transform for %s: %ld\n", algo,
   1334		       PTR_ERR(tfm));
   1335		return;
   1336	}
   1337
   1338	pr_info("\ntesting speed of %s %s (%s) %s\n", async ? "async" : "sync",
   1339		algo, get_driver_name(crypto_skcipher, tfm), e);
   1340
   1341	req = skcipher_request_alloc(tfm, GFP_KERNEL);
   1342	if (!req) {
   1343		pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",
   1344		       algo);
   1345		goto out;
   1346	}
   1347
   1348	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
   1349				      crypto_req_done, &wait);
   1350
   1351	i = 0;
   1352	do {
   1353		b_size = block_sizes;
   1354
   1355		do {
   1356			u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
   1357			struct scatterlist sg[TVMEMSIZE];
   1358
   1359			if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
   1360				pr_err("template (%u) too big for "
   1361				       "tvmem (%lu)\n", *keysize + bs,
   1362				       TVMEMSIZE * PAGE_SIZE);
   1363				goto out_free_req;
   1364			}
   1365
   1366			pr_info("test %u (%d bit key, %d byte blocks): ", i,
   1367				*keysize * 8, bs);
   1368
   1369			memset(tvmem[0], 0xff, PAGE_SIZE);
   1370
   1371			/* set key, plain text and IV */
   1372			key = tvmem[0];
   1373			for (j = 0; j < tcount; j++) {
   1374				if (template[j].klen == *keysize) {
   1375					key = template[j].key;
   1376					break;
   1377				}
   1378			}
   1379
   1380			crypto_skcipher_clear_flags(tfm, ~0);
   1381
   1382			ret = crypto_skcipher_setkey(tfm, key, *keysize);
   1383			if (ret) {
   1384				pr_err("setkey() failed flags=%x\n",
   1385					crypto_skcipher_get_flags(tfm));
   1386				goto out_free_req;
   1387			}
   1388
   1389			k = *keysize + bs;
   1390			sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));
   1391
   1392			if (k > PAGE_SIZE) {
   1393				sg_set_buf(sg, tvmem[0] + *keysize,
   1394				   PAGE_SIZE - *keysize);
   1395				k -= PAGE_SIZE;
   1396				j = 1;
   1397				while (k > PAGE_SIZE) {
   1398					sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
   1399					memset(tvmem[j], 0xff, PAGE_SIZE);
   1400					j++;
   1401					k -= PAGE_SIZE;
   1402				}
   1403				sg_set_buf(sg + j, tvmem[j], k);
   1404				memset(tvmem[j], 0xff, k);
   1405			} else {
   1406				sg_set_buf(sg, tvmem[0] + *keysize, bs);
   1407			}
   1408
   1409			iv_len = crypto_skcipher_ivsize(tfm);
   1410			if (iv_len)
   1411				memset(&iv, 0xff, iv_len);
   1412
   1413			skcipher_request_set_crypt(req, sg, sg, bs, iv);
   1414
   1415			if (secs) {
   1416				ret = test_acipher_jiffies(req, enc,
   1417							   bs, secs);
   1418				cond_resched();
   1419			} else {
   1420				ret = test_acipher_cycles(req, enc,
   1421							  bs);
   1422			}
   1423
   1424			if (ret) {
   1425				pr_err("%s() failed flags=%x\n", e,
   1426				       crypto_skcipher_get_flags(tfm));
   1427				break;
   1428			}
   1429			b_size++;
   1430			i++;
   1431		} while (*b_size);
   1432		keysize++;
   1433	} while (*keysize);
   1434
   1435out_free_req:
   1436	skcipher_request_free(req);
   1437out:
   1438	crypto_free_skcipher(tfm);
   1439}
   1440
   1441static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
   1442			       struct cipher_speed_template *template,
   1443			       unsigned int tcount, u8 *keysize)
   1444{
   1445	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
   1446				   true);
   1447}
   1448
   1449static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
   1450			      struct cipher_speed_template *template,
   1451			      unsigned int tcount, u8 *keysize)
   1452{
   1453	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
   1454				   false);
   1455}
   1456
   1457static void test_available(void)
   1458{
   1459	const char **name = check;
   1460
   1461	while (*name) {
   1462		printk("alg %s ", *name);
   1463		printk(crypto_has_alg(*name, 0, 0) ?
   1464		       "found\n" : "not found\n");
   1465		name++;
   1466	}
   1467}
   1468
   1469static inline int tcrypt_test(const char *alg)
   1470{
   1471	int ret;
   1472
   1473	pr_debug("testing %s\n", alg);
   1474
   1475	ret = alg_test(alg, alg, 0, 0);
   1476	/* non-fips algs return -EINVAL or -ECANCELED in fips mode */
   1477	if (fips_enabled && (ret == -EINVAL || ret == -ECANCELED))
   1478		ret = 0;
   1479	return ret;
   1480}
   1481
   1482static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
   1483{
   1484	int i;
   1485	int ret = 0;
   1486
   1487	switch (m) {
   1488	case 0:
   1489		if (alg) {
   1490			if (!crypto_has_alg(alg, type,
   1491					    mask ?: CRYPTO_ALG_TYPE_MASK))
   1492				ret = -ENOENT;
   1493			break;
   1494		}
   1495
   1496		for (i = 1; i < 200; i++)
   1497			ret += do_test(NULL, 0, 0, i, num_mb);
   1498		break;
   1499
   1500	case 1:
   1501		ret += tcrypt_test("md5");
   1502		break;
   1503
   1504	case 2:
   1505		ret += tcrypt_test("sha1");
   1506		break;
   1507
   1508	case 3:
   1509		ret += tcrypt_test("ecb(des)");
   1510		ret += tcrypt_test("cbc(des)");
   1511		ret += tcrypt_test("ctr(des)");
   1512		break;
   1513
   1514	case 4:
   1515		ret += tcrypt_test("ecb(des3_ede)");
   1516		ret += tcrypt_test("cbc(des3_ede)");
   1517		ret += tcrypt_test("ctr(des3_ede)");
   1518		break;
   1519
   1520	case 5:
   1521		ret += tcrypt_test("md4");
   1522		break;
   1523
   1524	case 6:
   1525		ret += tcrypt_test("sha256");
   1526		break;
   1527
   1528	case 7:
   1529		ret += tcrypt_test("ecb(blowfish)");
   1530		ret += tcrypt_test("cbc(blowfish)");
   1531		ret += tcrypt_test("ctr(blowfish)");
   1532		break;
   1533
   1534	case 8:
   1535		ret += tcrypt_test("ecb(twofish)");
   1536		ret += tcrypt_test("cbc(twofish)");
   1537		ret += tcrypt_test("ctr(twofish)");
   1538		ret += tcrypt_test("lrw(twofish)");
   1539		ret += tcrypt_test("xts(twofish)");
   1540		break;
   1541
   1542	case 9:
   1543		ret += tcrypt_test("ecb(serpent)");
   1544		ret += tcrypt_test("cbc(serpent)");
   1545		ret += tcrypt_test("ctr(serpent)");
   1546		ret += tcrypt_test("lrw(serpent)");
   1547		ret += tcrypt_test("xts(serpent)");
   1548		break;
   1549
   1550	case 10:
   1551		ret += tcrypt_test("ecb(aes)");
   1552		ret += tcrypt_test("cbc(aes)");
   1553		ret += tcrypt_test("lrw(aes)");
   1554		ret += tcrypt_test("xts(aes)");
   1555		ret += tcrypt_test("ctr(aes)");
   1556		ret += tcrypt_test("rfc3686(ctr(aes))");
   1557		ret += tcrypt_test("ofb(aes)");
   1558		ret += tcrypt_test("cfb(aes)");
   1559		break;
   1560
   1561	case 11:
   1562		ret += tcrypt_test("sha384");
   1563		break;
   1564
   1565	case 12:
   1566		ret += tcrypt_test("sha512");
   1567		break;
   1568
   1569	case 13:
   1570		ret += tcrypt_test("deflate");
   1571		break;
   1572
   1573	case 14:
   1574		ret += tcrypt_test("ecb(cast5)");
   1575		ret += tcrypt_test("cbc(cast5)");
   1576		ret += tcrypt_test("ctr(cast5)");
   1577		break;
   1578
   1579	case 15:
   1580		ret += tcrypt_test("ecb(cast6)");
   1581		ret += tcrypt_test("cbc(cast6)");
   1582		ret += tcrypt_test("ctr(cast6)");
   1583		ret += tcrypt_test("lrw(cast6)");
   1584		ret += tcrypt_test("xts(cast6)");
   1585		break;
   1586
   1587	case 16:
   1588		ret += tcrypt_test("ecb(arc4)");
   1589		break;
   1590
   1591	case 17:
   1592		ret += tcrypt_test("michael_mic");
   1593		break;
   1594
   1595	case 18:
   1596		ret += tcrypt_test("crc32c");
   1597		break;
   1598
   1599	case 19:
   1600		ret += tcrypt_test("ecb(tea)");
   1601		break;
   1602
   1603	case 20:
   1604		ret += tcrypt_test("ecb(xtea)");
   1605		break;
   1606
   1607	case 21:
   1608		ret += tcrypt_test("ecb(khazad)");
   1609		break;
   1610
   1611	case 22:
   1612		ret += tcrypt_test("wp512");
   1613		break;
   1614
   1615	case 23:
   1616		ret += tcrypt_test("wp384");
   1617		break;
   1618
   1619	case 24:
   1620		ret += tcrypt_test("wp256");
   1621		break;
   1622
   1623	case 26:
   1624		ret += tcrypt_test("ecb(anubis)");
   1625		ret += tcrypt_test("cbc(anubis)");
   1626		break;
   1627
   1628	case 30:
   1629		ret += tcrypt_test("ecb(xeta)");
   1630		break;
   1631
   1632	case 31:
   1633		ret += tcrypt_test("pcbc(fcrypt)");
   1634		break;
   1635
   1636	case 32:
   1637		ret += tcrypt_test("ecb(camellia)");
   1638		ret += tcrypt_test("cbc(camellia)");
   1639		ret += tcrypt_test("ctr(camellia)");
   1640		ret += tcrypt_test("lrw(camellia)");
   1641		ret += tcrypt_test("xts(camellia)");
   1642		break;
   1643
   1644	case 33:
   1645		ret += tcrypt_test("sha224");
   1646		break;
   1647
   1648	case 35:
   1649		ret += tcrypt_test("gcm(aes)");
   1650		break;
   1651
   1652	case 36:
   1653		ret += tcrypt_test("lzo");
   1654		break;
   1655
   1656	case 37:
   1657		ret += tcrypt_test("ccm(aes)");
   1658		break;
   1659
   1660	case 38:
   1661		ret += tcrypt_test("cts(cbc(aes))");
   1662		break;
   1663
   1664        case 39:
   1665		ret += tcrypt_test("xxhash64");
   1666		break;
   1667
   1668        case 40:
   1669		ret += tcrypt_test("rmd160");
   1670		break;
   1671
   1672	case 41:
   1673		ret += tcrypt_test("blake2s-256");
   1674		break;
   1675
   1676	case 42:
   1677		ret += tcrypt_test("blake2b-512");
   1678		break;
   1679
   1680	case 43:
   1681		ret += tcrypt_test("ecb(seed)");
   1682		break;
   1683
   1684	case 45:
   1685		ret += tcrypt_test("rfc4309(ccm(aes))");
   1686		break;
   1687
   1688	case 46:
   1689		ret += tcrypt_test("ghash");
   1690		break;
   1691
   1692	case 47:
   1693		ret += tcrypt_test("crct10dif");
   1694		break;
   1695
   1696	case 48:
   1697		ret += tcrypt_test("sha3-224");
   1698		break;
   1699
   1700	case 49:
   1701		ret += tcrypt_test("sha3-256");
   1702		break;
   1703
   1704	case 50:
   1705		ret += tcrypt_test("sha3-384");
   1706		break;
   1707
   1708	case 51:
   1709		ret += tcrypt_test("sha3-512");
   1710		break;
   1711
   1712	case 52:
   1713		ret += tcrypt_test("sm3");
   1714		break;
   1715
   1716	case 53:
   1717		ret += tcrypt_test("streebog256");
   1718		break;
   1719
   1720	case 54:
   1721		ret += tcrypt_test("streebog512");
   1722		break;
   1723
   1724	case 55:
   1725		ret += tcrypt_test("gcm(sm4)");
   1726		break;
   1727
   1728	case 56:
   1729		ret += tcrypt_test("ccm(sm4)");
   1730		break;
   1731
   1732	case 100:
   1733		ret += tcrypt_test("hmac(md5)");
   1734		break;
   1735
   1736	case 101:
   1737		ret += tcrypt_test("hmac(sha1)");
   1738		break;
   1739
   1740	case 102:
   1741		ret += tcrypt_test("hmac(sha256)");
   1742		break;
   1743
   1744	case 103:
   1745		ret += tcrypt_test("hmac(sha384)");
   1746		break;
   1747
   1748	case 104:
   1749		ret += tcrypt_test("hmac(sha512)");
   1750		break;
   1751
   1752	case 105:
   1753		ret += tcrypt_test("hmac(sha224)");
   1754		break;
   1755
   1756	case 106:
   1757		ret += tcrypt_test("xcbc(aes)");
   1758		break;
   1759
   1760	case 108:
   1761		ret += tcrypt_test("hmac(rmd160)");
   1762		break;
   1763
   1764	case 109:
   1765		ret += tcrypt_test("vmac64(aes)");
   1766		break;
   1767
   1768	case 111:
   1769		ret += tcrypt_test("hmac(sha3-224)");
   1770		break;
   1771
   1772	case 112:
   1773		ret += tcrypt_test("hmac(sha3-256)");
   1774		break;
   1775
   1776	case 113:
   1777		ret += tcrypt_test("hmac(sha3-384)");
   1778		break;
   1779
   1780	case 114:
   1781		ret += tcrypt_test("hmac(sha3-512)");
   1782		break;
   1783
   1784	case 115:
   1785		ret += tcrypt_test("hmac(streebog256)");
   1786		break;
   1787
   1788	case 116:
   1789		ret += tcrypt_test("hmac(streebog512)");
   1790		break;
   1791
   1792	case 150:
   1793		ret += tcrypt_test("ansi_cprng");
   1794		break;
   1795
   1796	case 151:
   1797		ret += tcrypt_test("rfc4106(gcm(aes))");
   1798		break;
   1799
   1800	case 152:
   1801		ret += tcrypt_test("rfc4543(gcm(aes))");
   1802		break;
   1803
   1804	case 153:
   1805		ret += tcrypt_test("cmac(aes)");
   1806		break;
   1807
   1808	case 154:
   1809		ret += tcrypt_test("cmac(des3_ede)");
   1810		break;
   1811
   1812	case 155:
   1813		ret += tcrypt_test("authenc(hmac(sha1),cbc(aes))");
   1814		break;
   1815
   1816	case 156:
   1817		ret += tcrypt_test("authenc(hmac(md5),ecb(cipher_null))");
   1818		break;
   1819
   1820	case 157:
   1821		ret += tcrypt_test("authenc(hmac(sha1),ecb(cipher_null))");
   1822		break;
   1823
   1824	case 158:
   1825		ret += tcrypt_test("cbcmac(sm4)");
   1826		break;
   1827
   1828	case 159:
   1829		ret += tcrypt_test("cmac(sm4)");
   1830		break;
   1831
   1832	case 181:
   1833		ret += tcrypt_test("authenc(hmac(sha1),cbc(des))");
   1834		break;
   1835	case 182:
   1836		ret += tcrypt_test("authenc(hmac(sha1),cbc(des3_ede))");
   1837		break;
   1838	case 183:
   1839		ret += tcrypt_test("authenc(hmac(sha224),cbc(des))");
   1840		break;
   1841	case 184:
   1842		ret += tcrypt_test("authenc(hmac(sha224),cbc(des3_ede))");
   1843		break;
   1844	case 185:
   1845		ret += tcrypt_test("authenc(hmac(sha256),cbc(des))");
   1846		break;
   1847	case 186:
   1848		ret += tcrypt_test("authenc(hmac(sha256),cbc(des3_ede))");
   1849		break;
   1850	case 187:
   1851		ret += tcrypt_test("authenc(hmac(sha384),cbc(des))");
   1852		break;
   1853	case 188:
   1854		ret += tcrypt_test("authenc(hmac(sha384),cbc(des3_ede))");
   1855		break;
   1856	case 189:
   1857		ret += tcrypt_test("authenc(hmac(sha512),cbc(des))");
   1858		break;
   1859	case 190:
   1860		ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))");
   1861		break;
   1862	case 191:
   1863		ret += tcrypt_test("ecb(sm4)");
   1864		ret += tcrypt_test("cbc(sm4)");
   1865		ret += tcrypt_test("cfb(sm4)");
   1866		ret += tcrypt_test("ctr(sm4)");
   1867		break;
   1868	case 200:
   1869		test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
   1870				speed_template_16_24_32);
   1871		test_cipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
   1872				speed_template_16_24_32);
   1873		test_cipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
   1874				speed_template_16_24_32);
   1875		test_cipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
   1876				speed_template_16_24_32);
   1877		test_cipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
   1878				speed_template_32_40_48);
   1879		test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
   1880				speed_template_32_40_48);
   1881		test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
   1882				speed_template_32_64);
   1883		test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
   1884				speed_template_32_64);
   1885		test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
   1886				speed_template_16_24_32);
   1887		test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
   1888				speed_template_16_24_32);
   1889		test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
   1890				speed_template_16_24_32);
   1891		test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
   1892				speed_template_16_24_32);
   1893		test_cipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
   1894				speed_template_16_24_32);
   1895		test_cipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
   1896				speed_template_16_24_32);
   1897		break;
   1898
   1899	case 201:
   1900		test_cipher_speed("ecb(des3_ede)", ENCRYPT, sec,
   1901				des3_speed_template, DES3_SPEED_VECTORS,
   1902				speed_template_24);
   1903		test_cipher_speed("ecb(des3_ede)", DECRYPT, sec,
   1904				des3_speed_template, DES3_SPEED_VECTORS,
   1905				speed_template_24);
   1906		test_cipher_speed("cbc(des3_ede)", ENCRYPT, sec,
   1907				des3_speed_template, DES3_SPEED_VECTORS,
   1908				speed_template_24);
   1909		test_cipher_speed("cbc(des3_ede)", DECRYPT, sec,
   1910				des3_speed_template, DES3_SPEED_VECTORS,
   1911				speed_template_24);
   1912		test_cipher_speed("ctr(des3_ede)", ENCRYPT, sec,
   1913				des3_speed_template, DES3_SPEED_VECTORS,
   1914				speed_template_24);
   1915		test_cipher_speed("ctr(des3_ede)", DECRYPT, sec,
   1916				des3_speed_template, DES3_SPEED_VECTORS,
   1917				speed_template_24);
   1918		break;
   1919
   1920	case 202:
   1921		test_cipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
   1922				speed_template_16_24_32);
   1923		test_cipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
   1924				speed_template_16_24_32);
   1925		test_cipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
   1926				speed_template_16_24_32);
   1927		test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
   1928				speed_template_16_24_32);
   1929		test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
   1930				speed_template_16_24_32);
   1931		test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
   1932				speed_template_16_24_32);
   1933		test_cipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
   1934				speed_template_32_40_48);
   1935		test_cipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
   1936				speed_template_32_40_48);
   1937		test_cipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
   1938				speed_template_32_48_64);
   1939		test_cipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
   1940				speed_template_32_48_64);
   1941		break;
   1942
   1943	case 203:
   1944		test_cipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
   1945				  speed_template_8_32);
   1946		test_cipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
   1947				  speed_template_8_32);
   1948		test_cipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
   1949				  speed_template_8_32);
   1950		test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
   1951				  speed_template_8_32);
   1952		test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
   1953				  speed_template_8_32);
   1954		test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
   1955				  speed_template_8_32);
   1956		break;
   1957
   1958	case 204:
   1959		test_cipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
   1960				  speed_template_8);
   1961		test_cipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
   1962				  speed_template_8);
   1963		test_cipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
   1964				  speed_template_8);
   1965		test_cipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
   1966				  speed_template_8);
   1967		break;
   1968
   1969	case 205:
   1970		test_cipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
   1971				speed_template_16_24_32);
   1972		test_cipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
   1973				speed_template_16_24_32);
   1974		test_cipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
   1975				speed_template_16_24_32);
   1976		test_cipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
   1977				speed_template_16_24_32);
   1978		test_cipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
   1979				speed_template_16_24_32);
   1980		test_cipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
   1981				speed_template_16_24_32);
   1982		test_cipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
   1983				speed_template_32_40_48);
   1984		test_cipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
   1985				speed_template_32_40_48);
   1986		test_cipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
   1987				speed_template_32_48_64);
   1988		test_cipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
   1989				speed_template_32_48_64);
   1990		break;
   1991
   1992	case 207:
   1993		test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
   1994				  speed_template_16_32);
   1995		test_cipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
   1996				  speed_template_16_32);
   1997		test_cipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
   1998				  speed_template_16_32);
   1999		test_cipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
   2000				  speed_template_16_32);
   2001		test_cipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
   2002				  speed_template_16_32);
   2003		test_cipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
   2004				  speed_template_16_32);
   2005		test_cipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
   2006				  speed_template_32_48);
   2007		test_cipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
   2008				  speed_template_32_48);
   2009		test_cipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
   2010				  speed_template_32_64);
   2011		test_cipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
   2012				  speed_template_32_64);
   2013		break;
   2014
   2015	case 208:
   2016		test_cipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
   2017				  speed_template_8);
   2018		break;
   2019
   2020	case 209:
   2021		test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
   2022				  speed_template_8_16);
   2023		test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
   2024				  speed_template_8_16);
   2025		test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
   2026				  speed_template_8_16);
   2027		test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
   2028				  speed_template_8_16);
   2029		test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
   2030				  speed_template_8_16);
   2031		test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
   2032				  speed_template_8_16);
   2033		break;
   2034
   2035	case 210:
   2036		test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
   2037				  speed_template_16_32);
   2038		test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
   2039				  speed_template_16_32);
   2040		test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
   2041				  speed_template_16_32);
   2042		test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
   2043				  speed_template_16_32);
   2044		test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
   2045				  speed_template_16_32);
   2046		test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
   2047				  speed_template_16_32);
   2048		test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
   2049				  speed_template_32_48);
   2050		test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
   2051				  speed_template_32_48);
   2052		test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
   2053				  speed_template_32_64);
   2054		test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
   2055				  speed_template_32_64);
   2056		break;
   2057
   2058	case 211:
   2059		test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec,
   2060				NULL, 0, 16, 16, aead_speed_template_20);
   2061		test_aead_speed("gcm(aes)", ENCRYPT, sec,
   2062				NULL, 0, 16, 8, speed_template_16_24_32);
   2063		test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
   2064				NULL, 0, 16, 16, aead_speed_template_20);
   2065		test_aead_speed("gcm(aes)", DECRYPT, sec,
   2066				NULL, 0, 16, 8, speed_template_16_24_32);
   2067		break;
   2068
   2069	case 212:
   2070		test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
   2071				NULL, 0, 16, 16, aead_speed_template_19);
   2072		test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
   2073				NULL, 0, 16, 16, aead_speed_template_19);
   2074		break;
   2075
   2076	case 213:
   2077		test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
   2078				NULL, 0, 16, 8, aead_speed_template_36);
   2079		test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
   2080				NULL, 0, 16, 8, aead_speed_template_36);
   2081		break;
   2082
   2083	case 214:
   2084		test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0,
   2085				  speed_template_32);
   2086		break;
   2087
   2088	case 215:
   2089		test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
   2090				   0, 16, 16, aead_speed_template_20, num_mb);
   2091		test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
   2092				   speed_template_16_24_32, num_mb);
   2093		test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
   2094				   0, 16, 16, aead_speed_template_20, num_mb);
   2095		test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
   2096				   speed_template_16_24_32, num_mb);
   2097		break;
   2098
   2099	case 216:
   2100		test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
   2101				   16, 16, aead_speed_template_19, num_mb);
   2102		test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
   2103				   16, 16, aead_speed_template_19, num_mb);
   2104		break;
   2105
   2106	case 217:
   2107		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
   2108				   sec, NULL, 0, 16, 8, aead_speed_template_36,
   2109				   num_mb);
   2110		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
   2111				   sec, NULL, 0, 16, 8, aead_speed_template_36,
   2112				   num_mb);
   2113		break;
   2114
   2115	case 218:
   2116		test_cipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
   2117				speed_template_16);
   2118		test_cipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
   2119				speed_template_16);
   2120		test_cipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
   2121				speed_template_16);
   2122		test_cipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
   2123				speed_template_16);
   2124		test_cipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0,
   2125				speed_template_16);
   2126		test_cipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0,
   2127				speed_template_16);
   2128		test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
   2129				speed_template_16);
   2130		test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
   2131				speed_template_16);
   2132		break;
   2133
   2134	case 219:
   2135		test_cipher_speed("adiantum(xchacha12,aes)", ENCRYPT, sec, NULL,
   2136				  0, speed_template_32);
   2137		test_cipher_speed("adiantum(xchacha12,aes)", DECRYPT, sec, NULL,
   2138				  0, speed_template_32);
   2139		test_cipher_speed("adiantum(xchacha20,aes)", ENCRYPT, sec, NULL,
   2140				  0, speed_template_32);
   2141		test_cipher_speed("adiantum(xchacha20,aes)", DECRYPT, sec, NULL,
   2142				  0, speed_template_32);
   2143		break;
   2144
   2145	case 220:
   2146		test_acipher_speed("essiv(cbc(aes),sha256)",
   2147				  ENCRYPT, sec, NULL, 0,
   2148				  speed_template_16_24_32);
   2149		test_acipher_speed("essiv(cbc(aes),sha256)",
   2150				  DECRYPT, sec, NULL, 0,
   2151				  speed_template_16_24_32);
   2152		break;
   2153
   2154	case 221:
   2155		test_aead_speed("aegis128", ENCRYPT, sec,
   2156				NULL, 0, 16, 8, speed_template_16);
   2157		test_aead_speed("aegis128", DECRYPT, sec,
   2158				NULL, 0, 16, 8, speed_template_16);
   2159		break;
   2160
   2161	case 222:
   2162		test_aead_speed("gcm(sm4)", ENCRYPT, sec,
   2163				NULL, 0, 16, 8, speed_template_16);
   2164		test_aead_speed("gcm(sm4)", DECRYPT, sec,
   2165				NULL, 0, 16, 8, speed_template_16);
   2166		break;
   2167
   2168	case 223:
   2169		test_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec,
   2170				NULL, 0, 16, 16, aead_speed_template_19);
   2171		test_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec,
   2172				NULL, 0, 16, 16, aead_speed_template_19);
   2173		break;
   2174
   2175	case 224:
   2176		test_mb_aead_speed("gcm(sm4)", ENCRYPT, sec, NULL, 0, 16, 8,
   2177				   speed_template_16, num_mb);
   2178		test_mb_aead_speed("gcm(sm4)", DECRYPT, sec, NULL, 0, 16, 8,
   2179				   speed_template_16, num_mb);
   2180		break;
   2181
   2182	case 225:
   2183		test_mb_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec, NULL, 0,
   2184				   16, 16, aead_speed_template_19, num_mb);
   2185		test_mb_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec, NULL, 0,
   2186				   16, 16, aead_speed_template_19, num_mb);
   2187		break;
   2188
   2189	case 300:
   2190		if (alg) {
   2191			test_hash_speed(alg, sec, generic_hash_speed_template);
   2192			break;
   2193		}
   2194		fallthrough;
   2195	case 301:
   2196		test_hash_speed("md4", sec, generic_hash_speed_template);
   2197		if (mode > 300 && mode < 400) break;
   2198		fallthrough;
   2199	case 302:
   2200		test_hash_speed("md5", sec, generic_hash_speed_template);
   2201		if (mode > 300 && mode < 400) break;
   2202		fallthrough;
   2203	case 303:
   2204		test_hash_speed("sha1", sec, generic_hash_speed_template);
   2205		if (mode > 300 && mode < 400) break;
   2206		fallthrough;
   2207	case 304:
   2208		test_hash_speed("sha256", sec, generic_hash_speed_template);
   2209		if (mode > 300 && mode < 400) break;
   2210		fallthrough;
   2211	case 305:
   2212		test_hash_speed("sha384", sec, generic_hash_speed_template);
   2213		if (mode > 300 && mode < 400) break;
   2214		fallthrough;
   2215	case 306:
   2216		test_hash_speed("sha512", sec, generic_hash_speed_template);
   2217		if (mode > 300 && mode < 400) break;
   2218		fallthrough;
   2219	case 307:
   2220		test_hash_speed("wp256", sec, generic_hash_speed_template);
   2221		if (mode > 300 && mode < 400) break;
   2222		fallthrough;
   2223	case 308:
   2224		test_hash_speed("wp384", sec, generic_hash_speed_template);
   2225		if (mode > 300 && mode < 400) break;
   2226		fallthrough;
   2227	case 309:
   2228		test_hash_speed("wp512", sec, generic_hash_speed_template);
   2229		if (mode > 300 && mode < 400) break;
   2230		fallthrough;
   2231	case 313:
   2232		test_hash_speed("sha224", sec, generic_hash_speed_template);
   2233		if (mode > 300 && mode < 400) break;
   2234		fallthrough;
   2235	case 314:
   2236		test_hash_speed("xxhash64", sec, generic_hash_speed_template);
   2237		if (mode > 300 && mode < 400) break;
   2238		fallthrough;
   2239	case 315:
   2240		test_hash_speed("rmd160", sec, generic_hash_speed_template);
   2241		if (mode > 300 && mode < 400) break;
   2242		fallthrough;
   2243	case 316:
   2244		test_hash_speed("blake2s-256", sec, generic_hash_speed_template);
   2245		if (mode > 300 && mode < 400) break;
   2246		fallthrough;
   2247	case 317:
   2248		test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
   2249		if (mode > 300 && mode < 400) break;
   2250		fallthrough;
   2251	case 318:
   2252		klen = 16;
   2253		test_hash_speed("ghash", sec, generic_hash_speed_template);
   2254		if (mode > 300 && mode < 400) break;
   2255		fallthrough;
   2256	case 319:
   2257		test_hash_speed("crc32c", sec, generic_hash_speed_template);
   2258		if (mode > 300 && mode < 400) break;
   2259		fallthrough;
   2260	case 320:
   2261		test_hash_speed("crct10dif", sec, generic_hash_speed_template);
   2262		if (mode > 300 && mode < 400) break;
   2263		fallthrough;
   2264	case 321:
   2265		test_hash_speed("poly1305", sec, poly1305_speed_template);
   2266		if (mode > 300 && mode < 400) break;
   2267		fallthrough;
   2268	case 322:
   2269		test_hash_speed("sha3-224", sec, generic_hash_speed_template);
   2270		if (mode > 300 && mode < 400) break;
   2271		fallthrough;
   2272	case 323:
   2273		test_hash_speed("sha3-256", sec, generic_hash_speed_template);
   2274		if (mode > 300 && mode < 400) break;
   2275		fallthrough;
   2276	case 324:
   2277		test_hash_speed("sha3-384", sec, generic_hash_speed_template);
   2278		if (mode > 300 && mode < 400) break;
   2279		fallthrough;
   2280	case 325:
   2281		test_hash_speed("sha3-512", sec, generic_hash_speed_template);
   2282		if (mode > 300 && mode < 400) break;
   2283		fallthrough;
   2284	case 326:
   2285		test_hash_speed("sm3", sec, generic_hash_speed_template);
   2286		if (mode > 300 && mode < 400) break;
   2287		fallthrough;
   2288	case 327:
   2289		test_hash_speed("streebog256", sec,
   2290				generic_hash_speed_template);
   2291		if (mode > 300 && mode < 400) break;
   2292		fallthrough;
   2293	case 328:
   2294		test_hash_speed("streebog512", sec,
   2295				generic_hash_speed_template);
   2296		if (mode > 300 && mode < 400) break;
   2297		fallthrough;
   2298	case 399:
   2299		break;
   2300
   2301	case 400:
   2302		if (alg) {
   2303			test_ahash_speed(alg, sec, generic_hash_speed_template);
   2304			break;
   2305		}
   2306		fallthrough;
   2307	case 401:
   2308		test_ahash_speed("md4", sec, generic_hash_speed_template);
   2309		if (mode > 400 && mode < 500) break;
   2310		fallthrough;
   2311	case 402:
   2312		test_ahash_speed("md5", sec, generic_hash_speed_template);
   2313		if (mode > 400 && mode < 500) break;
   2314		fallthrough;
   2315	case 403:
   2316		test_ahash_speed("sha1", sec, generic_hash_speed_template);
   2317		if (mode > 400 && mode < 500) break;
   2318		fallthrough;
   2319	case 404:
   2320		test_ahash_speed("sha256", sec, generic_hash_speed_template);
   2321		if (mode > 400 && mode < 500) break;
   2322		fallthrough;
   2323	case 405:
   2324		test_ahash_speed("sha384", sec, generic_hash_speed_template);
   2325		if (mode > 400 && mode < 500) break;
   2326		fallthrough;
   2327	case 406:
   2328		test_ahash_speed("sha512", sec, generic_hash_speed_template);
   2329		if (mode > 400 && mode < 500) break;
   2330		fallthrough;
   2331	case 407:
   2332		test_ahash_speed("wp256", sec, generic_hash_speed_template);
   2333		if (mode > 400 && mode < 500) break;
   2334		fallthrough;
   2335	case 408:
   2336		test_ahash_speed("wp384", sec, generic_hash_speed_template);
   2337		if (mode > 400 && mode < 500) break;
   2338		fallthrough;
   2339	case 409:
   2340		test_ahash_speed("wp512", sec, generic_hash_speed_template);
   2341		if (mode > 400 && mode < 500) break;
   2342		fallthrough;
   2343	case 413:
   2344		test_ahash_speed("sha224", sec, generic_hash_speed_template);
   2345		if (mode > 400 && mode < 500) break;
   2346		fallthrough;
   2347	case 414:
   2348		test_ahash_speed("xxhash64", sec, generic_hash_speed_template);
   2349		if (mode > 400 && mode < 500) break;
   2350		fallthrough;
   2351	case 415:
   2352		test_ahash_speed("rmd160", sec, generic_hash_speed_template);
   2353		if (mode > 400 && mode < 500) break;
   2354		fallthrough;
   2355	case 416:
   2356		test_ahash_speed("blake2s-256", sec, generic_hash_speed_template);
   2357		if (mode > 400 && mode < 500) break;
   2358		fallthrough;
   2359	case 417:
   2360		test_ahash_speed("blake2b-512", sec, generic_hash_speed_template);
   2361		if (mode > 400 && mode < 500) break;
   2362		fallthrough;
   2363	case 418:
   2364		test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
   2365		if (mode > 400 && mode < 500) break;
   2366		fallthrough;
   2367	case 419:
   2368		test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
   2369		if (mode > 400 && mode < 500) break;
   2370		fallthrough;
   2371	case 420:
   2372		test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
   2373		if (mode > 400 && mode < 500) break;
   2374		fallthrough;
   2375	case 421:
   2376		test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
   2377		if (mode > 400 && mode < 500) break;
   2378		fallthrough;
   2379	case 422:
   2380		test_ahash_speed("sm3", sec, generic_hash_speed_template);
   2381		if (mode > 400 && mode < 500) break;
   2382		fallthrough;
   2383	case 499:
   2384		break;
   2385
   2386	case 500:
   2387		test_acipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
   2388				   speed_template_16_24_32);
   2389		test_acipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
   2390				   speed_template_16_24_32);
   2391		test_acipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
   2392				   speed_template_16_24_32);
   2393		test_acipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
   2394				   speed_template_16_24_32);
   2395		test_acipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
   2396				   speed_template_32_40_48);
   2397		test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
   2398				   speed_template_32_40_48);
   2399		test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
   2400				   speed_template_32_64);
   2401		test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
   2402				   speed_template_32_64);
   2403		test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
   2404				   speed_template_16_24_32);
   2405		test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
   2406				   speed_template_16_24_32);
   2407		test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
   2408				   speed_template_16_24_32);
   2409		test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
   2410				   speed_template_16_24_32);
   2411		test_acipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
   2412				   speed_template_16_24_32);
   2413		test_acipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
   2414				   speed_template_16_24_32);
   2415		test_acipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
   2416				   speed_template_16_24_32);
   2417		test_acipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
   2418				   speed_template_16_24_32);
   2419		test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0,
   2420				   speed_template_20_28_36);
   2421		test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0,
   2422				   speed_template_20_28_36);
   2423		break;
   2424
   2425	case 501:
   2426		test_acipher_speed("ecb(des3_ede)", ENCRYPT, sec,
   2427				   des3_speed_template, DES3_SPEED_VECTORS,
   2428				   speed_template_24);
   2429		test_acipher_speed("ecb(des3_ede)", DECRYPT, sec,
   2430				   des3_speed_template, DES3_SPEED_VECTORS,
   2431				   speed_template_24);
   2432		test_acipher_speed("cbc(des3_ede)", ENCRYPT, sec,
   2433				   des3_speed_template, DES3_SPEED_VECTORS,
   2434				   speed_template_24);
   2435		test_acipher_speed("cbc(des3_ede)", DECRYPT, sec,
   2436				   des3_speed_template, DES3_SPEED_VECTORS,
   2437				   speed_template_24);
   2438		test_acipher_speed("cfb(des3_ede)", ENCRYPT, sec,
   2439				   des3_speed_template, DES3_SPEED_VECTORS,
   2440				   speed_template_24);
   2441		test_acipher_speed("cfb(des3_ede)", DECRYPT, sec,
   2442				   des3_speed_template, DES3_SPEED_VECTORS,
   2443				   speed_template_24);
   2444		test_acipher_speed("ofb(des3_ede)", ENCRYPT, sec,
   2445				   des3_speed_template, DES3_SPEED_VECTORS,
   2446				   speed_template_24);
   2447		test_acipher_speed("ofb(des3_ede)", DECRYPT, sec,
   2448				   des3_speed_template, DES3_SPEED_VECTORS,
   2449				   speed_template_24);
   2450		break;
   2451
   2452	case 502:
   2453		test_acipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
   2454				   speed_template_8);
   2455		test_acipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
   2456				   speed_template_8);
   2457		test_acipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
   2458				   speed_template_8);
   2459		test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
   2460				   speed_template_8);
   2461		test_acipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
   2462				   speed_template_8);
   2463		test_acipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
   2464				   speed_template_8);
   2465		test_acipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
   2466				   speed_template_8);
   2467		test_acipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
   2468				   speed_template_8);
   2469		break;
   2470
   2471	case 503:
   2472		test_acipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
   2473				   speed_template_16_32);
   2474		test_acipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
   2475				   speed_template_16_32);
   2476		test_acipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
   2477				   speed_template_16_32);
   2478		test_acipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
   2479				   speed_template_16_32);
   2480		test_acipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
   2481				   speed_template_16_32);
   2482		test_acipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
   2483				   speed_template_16_32);
   2484		test_acipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
   2485				   speed_template_32_48);
   2486		test_acipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
   2487				   speed_template_32_48);
   2488		test_acipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
   2489				   speed_template_32_64);
   2490		test_acipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
   2491				   speed_template_32_64);
   2492		break;
   2493
   2494	case 504:
   2495		test_acipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
   2496				   speed_template_16_24_32);
   2497		test_acipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
   2498				   speed_template_16_24_32);
   2499		test_acipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
   2500				   speed_template_16_24_32);
   2501		test_acipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
   2502				   speed_template_16_24_32);
   2503		test_acipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
   2504				   speed_template_16_24_32);
   2505		test_acipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
   2506				   speed_template_16_24_32);
   2507		test_acipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
   2508				   speed_template_32_40_48);
   2509		test_acipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
   2510				   speed_template_32_40_48);
   2511		test_acipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
   2512				   speed_template_32_48_64);
   2513		test_acipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
   2514				   speed_template_32_48_64);
   2515		break;
   2516
   2517	case 505:
   2518		test_acipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
   2519				   speed_template_8);
   2520		break;
   2521
   2522	case 506:
   2523		test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
   2524				   speed_template_8_16);
   2525		test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
   2526				   speed_template_8_16);
   2527		test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
   2528				   speed_template_8_16);
   2529		test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
   2530				   speed_template_8_16);
   2531		test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
   2532				   speed_template_8_16);
   2533		test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
   2534				   speed_template_8_16);
   2535		break;
   2536
   2537	case 507:
   2538		test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
   2539				   speed_template_16_32);
   2540		test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
   2541				   speed_template_16_32);
   2542		test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
   2543				   speed_template_16_32);
   2544		test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
   2545				   speed_template_16_32);
   2546		test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
   2547				   speed_template_16_32);
   2548		test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
   2549				   speed_template_16_32);
   2550		test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
   2551				   speed_template_32_48);
   2552		test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
   2553				   speed_template_32_48);
   2554		test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
   2555				   speed_template_32_64);
   2556		test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
   2557				   speed_template_32_64);
   2558		break;
   2559
   2560	case 508:
   2561		test_acipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
   2562				   speed_template_16_32);
   2563		test_acipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
   2564				   speed_template_16_32);
   2565		test_acipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
   2566				   speed_template_16_32);
   2567		test_acipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
   2568				   speed_template_16_32);
   2569		test_acipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
   2570				   speed_template_16_32);
   2571		test_acipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
   2572				   speed_template_16_32);
   2573		test_acipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
   2574				   speed_template_32_48);
   2575		test_acipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
   2576				   speed_template_32_48);
   2577		test_acipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
   2578				   speed_template_32_64);
   2579		test_acipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
   2580				   speed_template_32_64);
   2581		break;
   2582
   2583	case 509:
   2584		test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
   2585				   speed_template_8_32);
   2586		test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
   2587				   speed_template_8_32);
   2588		test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
   2589				   speed_template_8_32);
   2590		test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
   2591				   speed_template_8_32);
   2592		test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
   2593				   speed_template_8_32);
   2594		test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
   2595				   speed_template_8_32);
   2596		break;
   2597
   2598	case 518:
   2599		test_acipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
   2600				speed_template_16);
   2601		test_acipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
   2602				speed_template_16);
   2603		test_acipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
   2604				speed_template_16);
   2605		test_acipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
   2606				speed_template_16);
   2607		test_acipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0,
   2608				speed_template_16);
   2609		test_acipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0,
   2610				speed_template_16);
   2611		test_acipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
   2612				speed_template_16);
   2613		test_acipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
   2614				speed_template_16);
   2615		break;
   2616
   2617	case 600:
   2618		test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
   2619				       speed_template_16_24_32, num_mb);
   2620		test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
   2621				       speed_template_16_24_32, num_mb);
   2622		test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
   2623				       speed_template_16_24_32, num_mb);
   2624		test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
   2625				       speed_template_16_24_32, num_mb);
   2626		test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
   2627				       speed_template_32_40_48, num_mb);
   2628		test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
   2629				       speed_template_32_40_48, num_mb);
   2630		test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
   2631				       speed_template_32_64, num_mb);
   2632		test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
   2633				       speed_template_32_64, num_mb);
   2634		test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
   2635				       speed_template_16_24_32, num_mb);
   2636		test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
   2637				       speed_template_16_24_32, num_mb);
   2638		test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
   2639				       speed_template_16_24_32, num_mb);
   2640		test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
   2641				       speed_template_16_24_32, num_mb);
   2642		test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
   2643				       speed_template_16_24_32, num_mb);
   2644		test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
   2645				       speed_template_16_24_32, num_mb);
   2646		test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
   2647				       speed_template_16_24_32, num_mb);
   2648		test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
   2649				       speed_template_16_24_32, num_mb);
   2650		test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
   2651				       0, speed_template_20_28_36, num_mb);
   2652		test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
   2653				       0, speed_template_20_28_36, num_mb);
   2654		break;
   2655
   2656	case 601:
   2657		test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
   2658				       des3_speed_template, DES3_SPEED_VECTORS,
   2659				       speed_template_24, num_mb);
   2660		test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
   2661				       des3_speed_template, DES3_SPEED_VECTORS,
   2662				       speed_template_24, num_mb);
   2663		test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
   2664				       des3_speed_template, DES3_SPEED_VECTORS,
   2665				       speed_template_24, num_mb);
   2666		test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
   2667				       des3_speed_template, DES3_SPEED_VECTORS,
   2668				       speed_template_24, num_mb);
   2669		test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec,
   2670				       des3_speed_template, DES3_SPEED_VECTORS,
   2671				       speed_template_24, num_mb);
   2672		test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec,
   2673				       des3_speed_template, DES3_SPEED_VECTORS,
   2674				       speed_template_24, num_mb);
   2675		test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec,
   2676				       des3_speed_template, DES3_SPEED_VECTORS,
   2677				       speed_template_24, num_mb);
   2678		test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec,
   2679				       des3_speed_template, DES3_SPEED_VECTORS,
   2680				       speed_template_24, num_mb);
   2681		break;
   2682
   2683	case 602:
   2684		test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
   2685				       speed_template_8, num_mb);
   2686		test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
   2687				       speed_template_8, num_mb);
   2688		test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
   2689				       speed_template_8, num_mb);
   2690		test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
   2691				       speed_template_8, num_mb);
   2692		test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
   2693				       speed_template_8, num_mb);
   2694		test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
   2695				       speed_template_8, num_mb);
   2696		test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
   2697				       speed_template_8, num_mb);
   2698		test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
   2699				       speed_template_8, num_mb);
   2700		break;
   2701
   2702	case 603:
   2703		test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
   2704				       speed_template_16_32, num_mb);
   2705		test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
   2706				       speed_template_16_32, num_mb);
   2707		test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
   2708				       speed_template_16_32, num_mb);
   2709		test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
   2710				       speed_template_16_32, num_mb);
   2711		test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
   2712				       speed_template_16_32, num_mb);
   2713		test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
   2714				       speed_template_16_32, num_mb);
   2715		test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
   2716				       speed_template_32_48, num_mb);
   2717		test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
   2718				       speed_template_32_48, num_mb);
   2719		test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
   2720				       speed_template_32_64, num_mb);
   2721		test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
   2722				       speed_template_32_64, num_mb);
   2723		break;
   2724
   2725	case 604:
   2726		test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
   2727				       speed_template_16_24_32, num_mb);
   2728		test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
   2729				       speed_template_16_24_32, num_mb);
   2730		test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
   2731				       speed_template_16_24_32, num_mb);
   2732		test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
   2733				       speed_template_16_24_32, num_mb);
   2734		test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
   2735				       speed_template_16_24_32, num_mb);
   2736		test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
   2737				       speed_template_16_24_32, num_mb);
   2738		test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
   2739				       speed_template_32_40_48, num_mb);
   2740		test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
   2741				       speed_template_32_40_48, num_mb);
   2742		test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
   2743				       speed_template_32_48_64, num_mb);
   2744		test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
   2745				       speed_template_32_48_64, num_mb);
   2746		break;
   2747
   2748	case 605:
   2749		test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
   2750				       speed_template_8, num_mb);
   2751		break;
   2752
   2753	case 606:
   2754		test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
   2755				       speed_template_8_16, num_mb);
   2756		test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
   2757				       speed_template_8_16, num_mb);
   2758		test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
   2759				       speed_template_8_16, num_mb);
   2760		test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
   2761				       speed_template_8_16, num_mb);
   2762		test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
   2763				       speed_template_8_16, num_mb);
   2764		test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
   2765				       speed_template_8_16, num_mb);
   2766		break;
   2767
   2768	case 607:
   2769		test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
   2770				       speed_template_16_32, num_mb);
   2771		test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
   2772				       speed_template_16_32, num_mb);
   2773		test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
   2774				       speed_template_16_32, num_mb);
   2775		test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
   2776				       speed_template_16_32, num_mb);
   2777		test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
   2778				       speed_template_16_32, num_mb);
   2779		test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
   2780				       speed_template_16_32, num_mb);
   2781		test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
   2782				       speed_template_32_48, num_mb);
   2783		test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
   2784				       speed_template_32_48, num_mb);
   2785		test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
   2786				       speed_template_32_64, num_mb);
   2787		test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
   2788				       speed_template_32_64, num_mb);
   2789		break;
   2790
   2791	case 608:
   2792		test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
   2793				       speed_template_16_32, num_mb);
   2794		test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
   2795				       speed_template_16_32, num_mb);
   2796		test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
   2797				       speed_template_16_32, num_mb);
   2798		test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
   2799				       speed_template_16_32, num_mb);
   2800		test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
   2801				       speed_template_16_32, num_mb);
   2802		test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
   2803				       speed_template_16_32, num_mb);
   2804		test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
   2805				       speed_template_32_48, num_mb);
   2806		test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
   2807				       speed_template_32_48, num_mb);
   2808		test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
   2809				       speed_template_32_64, num_mb);
   2810		test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
   2811				       speed_template_32_64, num_mb);
   2812		break;
   2813
   2814	case 609:
   2815		test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
   2816				       speed_template_8_32, num_mb);
   2817		test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
   2818				       speed_template_8_32, num_mb);
   2819		test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
   2820				       speed_template_8_32, num_mb);
   2821		test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
   2822				       speed_template_8_32, num_mb);
   2823		test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
   2824				       speed_template_8_32, num_mb);
   2825		test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
   2826				       speed_template_8_32, num_mb);
   2827		break;
   2828
   2829	case 1000:
   2830		test_available();
   2831		break;
   2832	}
   2833
   2834	return ret;
   2835}
   2836
   2837static int __init tcrypt_mod_init(void)
   2838{
   2839	int err = -ENOMEM;
   2840	int i;
   2841
   2842	for (i = 0; i < TVMEMSIZE; i++) {
   2843		tvmem[i] = (void *)__get_free_page(GFP_KERNEL);
   2844		if (!tvmem[i])
   2845			goto err_free_tv;
   2846	}
   2847
   2848	err = do_test(alg, type, mask, mode, num_mb);
   2849
   2850	if (err) {
   2851		printk(KERN_ERR "tcrypt: one or more tests failed!\n");
   2852		goto err_free_tv;
   2853	} else {
   2854		pr_debug("all tests passed\n");
   2855	}
   2856
   2857	/* We intentionaly return -EAGAIN to prevent keeping the module,
   2858	 * unless we're running in fips mode. It does all its work from
   2859	 * init() and doesn't offer any runtime functionality, but in
   2860	 * the fips case, checking for a successful load is helpful.
   2861	 * => we don't need it in the memory, do we?
   2862	 *                                        -- mludvig
   2863	 */
   2864	if (!fips_enabled)
   2865		err = -EAGAIN;
   2866
   2867err_free_tv:
   2868	for (i = 0; i < TVMEMSIZE && tvmem[i]; i++)
   2869		free_page((unsigned long)tvmem[i]);
   2870
   2871	return err;
   2872}
   2873
   2874/*
   2875 * If an init function is provided, an exit function must also be provided
   2876 * to allow module unload.
   2877 */
   2878static void __exit tcrypt_mod_fini(void) { }
   2879
   2880late_initcall(tcrypt_mod_init);
   2881module_exit(tcrypt_mod_fini);
   2882
   2883module_param(alg, charp, 0);
   2884module_param(type, uint, 0);
   2885module_param(mask, uint, 0);
   2886module_param(mode, int, 0);
   2887module_param(sec, uint, 0);
   2888MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
   2889		      "(defaults to zero which uses CPU cycles instead)");
   2890module_param(num_mb, uint, 0000);
   2891MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
   2892module_param(klen, uint, 0);
   2893MODULE_PARM_DESC(klen, "Key length (defaults to 0)");
   2894
   2895MODULE_LICENSE("GPL");
   2896MODULE_DESCRIPTION("Quick & dirty crypto testing module");
   2897MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");