cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

crct10dif-ce-glue.c (3554B)

      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
      4 *
      5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
      6 */
      7
      8#include <linux/cpufeature.h>
      9#include <linux/crc-t10dif.h>
     10#include <linux/init.h>
     11#include <linux/kernel.h>
     12#include <linux/module.h>
     13#include <linux/string.h>
     14
     15#include <crypto/internal/hash.h>
     16#include <crypto/internal/simd.h>
     17
     18#include <asm/neon.h>
     19#include <asm/simd.h>
     20
     21#define CRC_T10DIF_PMULL_CHUNK_SIZE	16U
     22
     23asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
     24asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
     25
     26static int crct10dif_init(struct shash_desc *desc)
     27{
     28	u16 *crc = shash_desc_ctx(desc);
     29
     30	*crc = 0;
     31	return 0;
     32}
     33
     34static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
     35			    unsigned int length)
     36{
     37	u16 *crc = shash_desc_ctx(desc);
     38
     39	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
     40		do {
     41			unsigned int chunk = length;
     42
     43			if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
     44				chunk = SZ_4K;
     45
     46			kernel_neon_begin();
     47			*crc = crc_t10dif_pmull_p8(*crc, data, chunk);
     48			kernel_neon_end();
     49			data += chunk;
     50			length -= chunk;
     51		} while (length);
     52	} else {
     53		*crc = crc_t10dif_generic(*crc, data, length);
     54	}
     55
     56	return 0;
     57}
     58
     59static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
     60			    unsigned int length)
     61{
     62	u16 *crc = shash_desc_ctx(desc);
     63
     64	if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
     65		do {
     66			unsigned int chunk = length;
     67
     68			if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
     69				chunk = SZ_4K;
     70
     71			kernel_neon_begin();
     72			*crc = crc_t10dif_pmull_p64(*crc, data, chunk);
     73			kernel_neon_end();
     74			data += chunk;
     75			length -= chunk;
     76		} while (length);
     77	} else {
     78		*crc = crc_t10dif_generic(*crc, data, length);
     79	}
     80
     81	return 0;
     82}
     83
     84static int crct10dif_final(struct shash_desc *desc, u8 *out)
     85{
     86	u16 *crc = shash_desc_ctx(desc);
     87
     88	*(u16 *)out = *crc;
     89	return 0;
     90}
     91
     92static struct shash_alg crc_t10dif_alg[] = {{
     93	.digestsize		= CRC_T10DIF_DIGEST_SIZE,
     94	.init			= crct10dif_init,
     95	.update			= crct10dif_update_pmull_p8,
     96	.final			= crct10dif_final,
     97	.descsize		= CRC_T10DIF_DIGEST_SIZE,
     98
     99	.base.cra_name		= "crct10dif",
    100	.base.cra_driver_name	= "crct10dif-arm64-neon",
    101	.base.cra_priority	= 100,
    102	.base.cra_blocksize	= CRC_T10DIF_BLOCK_SIZE,
    103	.base.cra_module	= THIS_MODULE,
    104}, {
    105	.digestsize		= CRC_T10DIF_DIGEST_SIZE,
    106	.init			= crct10dif_init,
    107	.update			= crct10dif_update_pmull_p64,
    108	.final			= crct10dif_final,
    109	.descsize		= CRC_T10DIF_DIGEST_SIZE,
    110
    111	.base.cra_name		= "crct10dif",
    112	.base.cra_driver_name	= "crct10dif-arm64-ce",
    113	.base.cra_priority	= 200,
    114	.base.cra_blocksize	= CRC_T10DIF_BLOCK_SIZE,
    115	.base.cra_module	= THIS_MODULE,
    116}};
    117
    118static int __init crc_t10dif_mod_init(void)
    119{
    120	if (cpu_have_named_feature(PMULL))
    121		return crypto_register_shashes(crc_t10dif_alg,
    122					       ARRAY_SIZE(crc_t10dif_alg));
    123	else
    124		/* only register the first array element */
    125		return crypto_register_shash(crc_t10dif_alg);
    126}
    127
    128static void __exit crc_t10dif_mod_exit(void)
    129{
    130	if (cpu_have_named_feature(PMULL))
    131		crypto_unregister_shashes(crc_t10dif_alg,
    132					  ARRAY_SIZE(crc_t10dif_alg));
    133	else
    134		crypto_unregister_shash(crc_t10dif_alg);
    135}
    136
    137module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
    138module_exit(crc_t10dif_mod_exit);
    139
    140MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
    141MODULE_LICENSE("GPL v2");
    142MODULE_ALIAS_CRYPTO("crct10dif");
    143MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");