cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

sm3_avx_glue.c (3210B)

      1/* SPDX-License-Identifier: GPL-2.0-or-later */
      2/*
      3 * SM3 Secure Hash Algorithm, AVX assembler accelerated.
      4 * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02
      5 *
      6 * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
      7 */
      8
      9#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
     10
     11#include <crypto/internal/hash.h>
     12#include <crypto/internal/simd.h>
     13#include <linux/init.h>
     14#include <linux/module.h>
     15#include <linux/types.h>
     16#include <crypto/sm3.h>
     17#include <crypto/sm3_base.h>
     18#include <asm/simd.h>
     19
     20asmlinkage void sm3_transform_avx(struct sm3_state *state,
     21			const u8 *data, int nblocks);
     22
     23static int sm3_avx_update(struct shash_desc *desc, const u8 *data,
     24			 unsigned int len)
     25{
     26	struct sm3_state *sctx = shash_desc_ctx(desc);
     27
     28	if (!crypto_simd_usable() ||
     29			(sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) {
     30		sm3_update(sctx, data, len);
     31		return 0;
     32	}
     33
     34	/*
     35	 * Make sure struct sm3_state begins directly with the SM3
     36	 * 256-bit internal state, as this is what the asm functions expect.
     37	 */
     38	BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0);
     39
     40	kernel_fpu_begin();
     41	sm3_base_do_update(desc, data, len, sm3_transform_avx);
     42	kernel_fpu_end();
     43
     44	return 0;
     45}
     46
     47static int sm3_avx_finup(struct shash_desc *desc, const u8 *data,
     48		      unsigned int len, u8 *out)
     49{
     50	if (!crypto_simd_usable()) {
     51		struct sm3_state *sctx = shash_desc_ctx(desc);
     52
     53		if (len)
     54			sm3_update(sctx, data, len);
     55
     56		sm3_final(sctx, out);
     57		return 0;
     58	}
     59
     60	kernel_fpu_begin();
     61	if (len)
     62		sm3_base_do_update(desc, data, len, sm3_transform_avx);
     63	sm3_base_do_finalize(desc, sm3_transform_avx);
     64	kernel_fpu_end();
     65
     66	return sm3_base_finish(desc, out);
     67}
     68
     69static int sm3_avx_final(struct shash_desc *desc, u8 *out)
     70{
     71	if (!crypto_simd_usable()) {
     72		sm3_final(shash_desc_ctx(desc), out);
     73		return 0;
     74	}
     75
     76	kernel_fpu_begin();
     77	sm3_base_do_finalize(desc, sm3_transform_avx);
     78	kernel_fpu_end();
     79
     80	return sm3_base_finish(desc, out);
     81}
     82
     83static struct shash_alg sm3_avx_alg = {
     84	.digestsize	=	SM3_DIGEST_SIZE,
     85	.init		=	sm3_base_init,
     86	.update		=	sm3_avx_update,
     87	.final		=	sm3_avx_final,
     88	.finup		=	sm3_avx_finup,
     89	.descsize	=	sizeof(struct sm3_state),
     90	.base		=	{
     91		.cra_name	=	"sm3",
     92		.cra_driver_name =	"sm3-avx",
     93		.cra_priority	=	300,
     94		.cra_blocksize	=	SM3_BLOCK_SIZE,
     95		.cra_module	=	THIS_MODULE,
     96	}
     97};
     98
     99static int __init sm3_avx_mod_init(void)
    100{
    101	const char *feature_name;
    102
    103	if (!boot_cpu_has(X86_FEATURE_AVX)) {
    104		pr_info("AVX instruction are not detected.\n");
    105		return -ENODEV;
    106	}
    107
    108	if (!boot_cpu_has(X86_FEATURE_BMI2)) {
    109		pr_info("BMI2 instruction are not detected.\n");
    110		return -ENODEV;
    111	}
    112
    113	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
    114				&feature_name)) {
    115		pr_info("CPU feature '%s' is not supported.\n", feature_name);
    116		return -ENODEV;
    117	}
    118
    119	return crypto_register_shash(&sm3_avx_alg);
    120}
    121
    122static void __exit sm3_avx_mod_exit(void)
    123{
    124	crypto_unregister_shash(&sm3_avx_alg);
    125}
    126
    127module_init(sm3_avx_mod_init);
    128module_exit(sm3_avx_mod_exit);
    129
    130MODULE_LICENSE("GPL v2");
    131MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
    132MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated");
    133MODULE_ALIAS_CRYPTO("sm3");
    134MODULE_ALIAS_CRYPTO("sm3-avx");