cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

sha512_ssse3_glue.c (8829B)


      1/*
      2 * Cryptographic API.
      3 *
      4 * Glue code for the SHA512 Secure Hash Algorithm assembler
      5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
      6 *
      7 * This file is based on sha512_generic.c
      8 *
      9 * Copyright (C) 2013 Intel Corporation
     10 * Author: Tim Chen <tim.c.chen@linux.intel.com>
     11 *
     12 * This program is free software; you can redistribute it and/or modify it
     13 * under the terms of the GNU General Public License as published by the Free
     14 * Software Foundation; either version 2 of the License, or (at your option)
     15 * any later version.
     16 *
     17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
     18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
     20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
     21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
     22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
     23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
     24 * SOFTWARE.
     25 *
     26 */
     27
     28#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
     29
     30#include <crypto/internal/hash.h>
     31#include <crypto/internal/simd.h>
     32#include <linux/init.h>
     33#include <linux/module.h>
     34#include <linux/mm.h>
     35#include <linux/string.h>
     36#include <linux/types.h>
     37#include <crypto/sha2.h>
     38#include <crypto/sha512_base.h>
     39#include <asm/simd.h>
     40
     41asmlinkage void sha512_transform_ssse3(struct sha512_state *state,
     42				       const u8 *data, int blocks);
     43
     44static int sha512_update(struct shash_desc *desc, const u8 *data,
     45		       unsigned int len, sha512_block_fn *sha512_xform)
     46{
     47	struct sha512_state *sctx = shash_desc_ctx(desc);
     48
     49	if (!crypto_simd_usable() ||
     50	    (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
     51		return crypto_sha512_update(desc, data, len);
     52
     53	/*
     54	 * Make sure struct sha512_state begins directly with the SHA512
     55	 * 512-bit internal state, as this is what the asm functions expect.
     56	 */
     57	BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
     58
     59	kernel_fpu_begin();
     60	sha512_base_do_update(desc, data, len, sha512_xform);
     61	kernel_fpu_end();
     62
     63	return 0;
     64}
     65
     66static int sha512_finup(struct shash_desc *desc, const u8 *data,
     67	      unsigned int len, u8 *out, sha512_block_fn *sha512_xform)
     68{
     69	if (!crypto_simd_usable())
     70		return crypto_sha512_finup(desc, data, len, out);
     71
     72	kernel_fpu_begin();
     73	if (len)
     74		sha512_base_do_update(desc, data, len, sha512_xform);
     75	sha512_base_do_finalize(desc, sha512_xform);
     76	kernel_fpu_end();
     77
     78	return sha512_base_finish(desc, out);
     79}
     80
     81static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
     82		       unsigned int len)
     83{
     84	return sha512_update(desc, data, len, sha512_transform_ssse3);
     85}
     86
     87static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
     88	      unsigned int len, u8 *out)
     89{
     90	return sha512_finup(desc, data, len, out, sha512_transform_ssse3);
     91}
     92
     93/* Add padding and return the message digest. */
     94static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
     95{
     96	return sha512_ssse3_finup(desc, NULL, 0, out);
     97}
     98
     99static struct shash_alg sha512_ssse3_algs[] = { {
    100	.digestsize	=	SHA512_DIGEST_SIZE,
    101	.init		=	sha512_base_init,
    102	.update		=	sha512_ssse3_update,
    103	.final		=	sha512_ssse3_final,
    104	.finup		=	sha512_ssse3_finup,
    105	.descsize	=	sizeof(struct sha512_state),
    106	.base		=	{
    107		.cra_name	=	"sha512",
    108		.cra_driver_name =	"sha512-ssse3",
    109		.cra_priority	=	150,
    110		.cra_blocksize	=	SHA512_BLOCK_SIZE,
    111		.cra_module	=	THIS_MODULE,
    112	}
    113},  {
    114	.digestsize	=	SHA384_DIGEST_SIZE,
    115	.init		=	sha384_base_init,
    116	.update		=	sha512_ssse3_update,
    117	.final		=	sha512_ssse3_final,
    118	.finup		=	sha512_ssse3_finup,
    119	.descsize	=	sizeof(struct sha512_state),
    120	.base		=	{
    121		.cra_name	=	"sha384",
    122		.cra_driver_name =	"sha384-ssse3",
    123		.cra_priority	=	150,
    124		.cra_blocksize	=	SHA384_BLOCK_SIZE,
    125		.cra_module	=	THIS_MODULE,
    126	}
    127} };
    128
    129static int register_sha512_ssse3(void)
    130{
    131	if (boot_cpu_has(X86_FEATURE_SSSE3))
    132		return crypto_register_shashes(sha512_ssse3_algs,
    133			ARRAY_SIZE(sha512_ssse3_algs));
    134	return 0;
    135}
    136
    137static void unregister_sha512_ssse3(void)
    138{
    139	if (boot_cpu_has(X86_FEATURE_SSSE3))
    140		crypto_unregister_shashes(sha512_ssse3_algs,
    141			ARRAY_SIZE(sha512_ssse3_algs));
    142}
    143
    144asmlinkage void sha512_transform_avx(struct sha512_state *state,
    145				     const u8 *data, int blocks);
    146static bool avx_usable(void)
    147{
    148	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
    149		if (boot_cpu_has(X86_FEATURE_AVX))
    150			pr_info("AVX detected but unusable.\n");
    151		return false;
    152	}
    153
    154	return true;
    155}
    156
    157static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
    158		       unsigned int len)
    159{
    160	return sha512_update(desc, data, len, sha512_transform_avx);
    161}
    162
    163static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
    164	      unsigned int len, u8 *out)
    165{
    166	return sha512_finup(desc, data, len, out, sha512_transform_avx);
    167}
    168
    169/* Add padding and return the message digest. */
    170static int sha512_avx_final(struct shash_desc *desc, u8 *out)
    171{
    172	return sha512_avx_finup(desc, NULL, 0, out);
    173}
    174
    175static struct shash_alg sha512_avx_algs[] = { {
    176	.digestsize	=	SHA512_DIGEST_SIZE,
    177	.init		=	sha512_base_init,
    178	.update		=	sha512_avx_update,
    179	.final		=	sha512_avx_final,
    180	.finup		=	sha512_avx_finup,
    181	.descsize	=	sizeof(struct sha512_state),
    182	.base		=	{
    183		.cra_name	=	"sha512",
    184		.cra_driver_name =	"sha512-avx",
    185		.cra_priority	=	160,
    186		.cra_blocksize	=	SHA512_BLOCK_SIZE,
    187		.cra_module	=	THIS_MODULE,
    188	}
    189},  {
    190	.digestsize	=	SHA384_DIGEST_SIZE,
    191	.init		=	sha384_base_init,
    192	.update		=	sha512_avx_update,
    193	.final		=	sha512_avx_final,
    194	.finup		=	sha512_avx_finup,
    195	.descsize	=	sizeof(struct sha512_state),
    196	.base		=	{
    197		.cra_name	=	"sha384",
    198		.cra_driver_name =	"sha384-avx",
    199		.cra_priority	=	160,
    200		.cra_blocksize	=	SHA384_BLOCK_SIZE,
    201		.cra_module	=	THIS_MODULE,
    202	}
    203} };
    204
    205static int register_sha512_avx(void)
    206{
    207	if (avx_usable())
    208		return crypto_register_shashes(sha512_avx_algs,
    209			ARRAY_SIZE(sha512_avx_algs));
    210	return 0;
    211}
    212
    213static void unregister_sha512_avx(void)
    214{
    215	if (avx_usable())
    216		crypto_unregister_shashes(sha512_avx_algs,
    217			ARRAY_SIZE(sha512_avx_algs));
    218}
    219
    220asmlinkage void sha512_transform_rorx(struct sha512_state *state,
    221				      const u8 *data, int blocks);
    222
    223static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
    224		       unsigned int len)
    225{
    226	return sha512_update(desc, data, len, sha512_transform_rorx);
    227}
    228
    229static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
    230	      unsigned int len, u8 *out)
    231{
    232	return sha512_finup(desc, data, len, out, sha512_transform_rorx);
    233}
    234
    235/* Add padding and return the message digest. */
    236static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
    237{
    238	return sha512_avx2_finup(desc, NULL, 0, out);
    239}
    240
    241static struct shash_alg sha512_avx2_algs[] = { {
    242	.digestsize	=	SHA512_DIGEST_SIZE,
    243	.init		=	sha512_base_init,
    244	.update		=	sha512_avx2_update,
    245	.final		=	sha512_avx2_final,
    246	.finup		=	sha512_avx2_finup,
    247	.descsize	=	sizeof(struct sha512_state),
    248	.base		=	{
    249		.cra_name	=	"sha512",
    250		.cra_driver_name =	"sha512-avx2",
    251		.cra_priority	=	170,
    252		.cra_blocksize	=	SHA512_BLOCK_SIZE,
    253		.cra_module	=	THIS_MODULE,
    254	}
    255},  {
    256	.digestsize	=	SHA384_DIGEST_SIZE,
    257	.init		=	sha384_base_init,
    258	.update		=	sha512_avx2_update,
    259	.final		=	sha512_avx2_final,
    260	.finup		=	sha512_avx2_finup,
    261	.descsize	=	sizeof(struct sha512_state),
    262	.base		=	{
    263		.cra_name	=	"sha384",
    264		.cra_driver_name =	"sha384-avx2",
    265		.cra_priority	=	170,
    266		.cra_blocksize	=	SHA384_BLOCK_SIZE,
    267		.cra_module	=	THIS_MODULE,
    268	}
    269} };
    270
    271static bool avx2_usable(void)
    272{
    273	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
    274		    boot_cpu_has(X86_FEATURE_BMI2))
    275		return true;
    276
    277	return false;
    278}
    279
    280static int register_sha512_avx2(void)
    281{
    282	if (avx2_usable())
    283		return crypto_register_shashes(sha512_avx2_algs,
    284			ARRAY_SIZE(sha512_avx2_algs));
    285	return 0;
    286}
    287
    288static void unregister_sha512_avx2(void)
    289{
    290	if (avx2_usable())
    291		crypto_unregister_shashes(sha512_avx2_algs,
    292			ARRAY_SIZE(sha512_avx2_algs));
    293}
    294
    295static int __init sha512_ssse3_mod_init(void)
    296{
    297
    298	if (register_sha512_ssse3())
    299		goto fail;
    300
    301	if (register_sha512_avx()) {
    302		unregister_sha512_ssse3();
    303		goto fail;
    304	}
    305
    306	if (register_sha512_avx2()) {
    307		unregister_sha512_avx();
    308		unregister_sha512_ssse3();
    309		goto fail;
    310	}
    311
    312	return 0;
    313fail:
    314	return -ENODEV;
    315}
    316
    317static void __exit sha512_ssse3_mod_fini(void)
    318{
    319	unregister_sha512_avx2();
    320	unregister_sha512_avx();
    321	unregister_sha512_ssse3();
    322}
    323
    324module_init(sha512_ssse3_mod_init);
    325module_exit(sha512_ssse3_mod_fini);
    326
    327MODULE_LICENSE("GPL");
    328MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");
    329
    330MODULE_ALIAS_CRYPTO("sha512");
    331MODULE_ALIAS_CRYPTO("sha512-ssse3");
    332MODULE_ALIAS_CRYPTO("sha512-avx");
    333MODULE_ALIAS_CRYPTO("sha512-avx2");
    334MODULE_ALIAS_CRYPTO("sha384");
    335MODULE_ALIAS_CRYPTO("sha384-ssse3");
    336MODULE_ALIAS_CRYPTO("sha384-avx");
    337MODULE_ALIAS_CRYPTO("sha384-avx2");