sha512_ssse3_glue.c (8829B)
1/* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA512 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha512_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation 10 * Author: Tim Chen <tim.c.chen@linux.intel.com> 11 * 12 * This program is free software; you can redistribute it and/or modify it 13 * under the terms of the GNU General Public License as published by the Free 14 * Software Foundation; either version 2 of the License, or (at your option) 15 * any later version. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 * SOFTWARE. 25 * 26 */ 27 28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30#include <crypto/internal/hash.h> 31#include <crypto/internal/simd.h> 32#include <linux/init.h> 33#include <linux/module.h> 34#include <linux/mm.h> 35#include <linux/string.h> 36#include <linux/types.h> 37#include <crypto/sha2.h> 38#include <crypto/sha512_base.h> 39#include <asm/simd.h> 40 41asmlinkage void sha512_transform_ssse3(struct sha512_state *state, 42 const u8 *data, int blocks); 43 44static int sha512_update(struct shash_desc *desc, const u8 *data, 45 unsigned int len, sha512_block_fn *sha512_xform) 46{ 47 struct sha512_state *sctx = shash_desc_ctx(desc); 48 49 if (!crypto_simd_usable() || 50 (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) 51 return crypto_sha512_update(desc, data, len); 52 53 /* 54 * Make sure struct sha512_state begins directly with the SHA512 55 * 512-bit internal state, as this is what the asm functions expect. 56 */ 57 BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); 58 59 kernel_fpu_begin(); 60 sha512_base_do_update(desc, data, len, sha512_xform); 61 kernel_fpu_end(); 62 63 return 0; 64} 65 66static int sha512_finup(struct shash_desc *desc, const u8 *data, 67 unsigned int len, u8 *out, sha512_block_fn *sha512_xform) 68{ 69 if (!crypto_simd_usable()) 70 return crypto_sha512_finup(desc, data, len, out); 71 72 kernel_fpu_begin(); 73 if (len) 74 sha512_base_do_update(desc, data, len, sha512_xform); 75 sha512_base_do_finalize(desc, sha512_xform); 76 kernel_fpu_end(); 77 78 return sha512_base_finish(desc, out); 79} 80 81static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, 82 unsigned int len) 83{ 84 return sha512_update(desc, data, len, sha512_transform_ssse3); 85} 86 87static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, 88 unsigned int len, u8 *out) 89{ 90 return sha512_finup(desc, data, len, out, sha512_transform_ssse3); 91} 92 93/* Add padding and return the message digest. */ 94static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) 95{ 96 return sha512_ssse3_finup(desc, NULL, 0, out); 97} 98 99static struct shash_alg sha512_ssse3_algs[] = { { 100 .digestsize = SHA512_DIGEST_SIZE, 101 .init = sha512_base_init, 102 .update = sha512_ssse3_update, 103 .final = sha512_ssse3_final, 104 .finup = sha512_ssse3_finup, 105 .descsize = sizeof(struct sha512_state), 106 .base = { 107 .cra_name = "sha512", 108 .cra_driver_name = "sha512-ssse3", 109 .cra_priority = 150, 110 .cra_blocksize = SHA512_BLOCK_SIZE, 111 .cra_module = THIS_MODULE, 112 } 113}, { 114 .digestsize = SHA384_DIGEST_SIZE, 115 .init = sha384_base_init, 116 .update = sha512_ssse3_update, 117 .final = sha512_ssse3_final, 118 .finup = sha512_ssse3_finup, 119 .descsize = sizeof(struct sha512_state), 120 .base = { 121 .cra_name = "sha384", 122 .cra_driver_name = "sha384-ssse3", 123 .cra_priority = 150, 124 .cra_blocksize = SHA384_BLOCK_SIZE, 125 .cra_module = THIS_MODULE, 126 } 127} }; 128 129static int register_sha512_ssse3(void) 130{ 131 if (boot_cpu_has(X86_FEATURE_SSSE3)) 132 return crypto_register_shashes(sha512_ssse3_algs, 133 ARRAY_SIZE(sha512_ssse3_algs)); 134 return 0; 135} 136 137static void unregister_sha512_ssse3(void) 138{ 139 if (boot_cpu_has(X86_FEATURE_SSSE3)) 140 crypto_unregister_shashes(sha512_ssse3_algs, 141 ARRAY_SIZE(sha512_ssse3_algs)); 142} 143 144asmlinkage void sha512_transform_avx(struct sha512_state *state, 145 const u8 *data, int blocks); 146static bool avx_usable(void) 147{ 148 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { 149 if (boot_cpu_has(X86_FEATURE_AVX)) 150 pr_info("AVX detected but unusable.\n"); 151 return false; 152 } 153 154 return true; 155} 156 157static int sha512_avx_update(struct shash_desc *desc, const u8 *data, 158 unsigned int len) 159{ 160 return sha512_update(desc, data, len, sha512_transform_avx); 161} 162 163static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, 164 unsigned int len, u8 *out) 165{ 166 return sha512_finup(desc, data, len, out, sha512_transform_avx); 167} 168 169/* Add padding and return the message digest. */ 170static int sha512_avx_final(struct shash_desc *desc, u8 *out) 171{ 172 return sha512_avx_finup(desc, NULL, 0, out); 173} 174 175static struct shash_alg sha512_avx_algs[] = { { 176 .digestsize = SHA512_DIGEST_SIZE, 177 .init = sha512_base_init, 178 .update = sha512_avx_update, 179 .final = sha512_avx_final, 180 .finup = sha512_avx_finup, 181 .descsize = sizeof(struct sha512_state), 182 .base = { 183 .cra_name = "sha512", 184 .cra_driver_name = "sha512-avx", 185 .cra_priority = 160, 186 .cra_blocksize = SHA512_BLOCK_SIZE, 187 .cra_module = THIS_MODULE, 188 } 189}, { 190 .digestsize = SHA384_DIGEST_SIZE, 191 .init = sha384_base_init, 192 .update = sha512_avx_update, 193 .final = sha512_avx_final, 194 .finup = sha512_avx_finup, 195 .descsize = sizeof(struct sha512_state), 196 .base = { 197 .cra_name = "sha384", 198 .cra_driver_name = "sha384-avx", 199 .cra_priority = 160, 200 .cra_blocksize = SHA384_BLOCK_SIZE, 201 .cra_module = THIS_MODULE, 202 } 203} }; 204 205static int register_sha512_avx(void) 206{ 207 if (avx_usable()) 208 return crypto_register_shashes(sha512_avx_algs, 209 ARRAY_SIZE(sha512_avx_algs)); 210 return 0; 211} 212 213static void unregister_sha512_avx(void) 214{ 215 if (avx_usable()) 216 crypto_unregister_shashes(sha512_avx_algs, 217 ARRAY_SIZE(sha512_avx_algs)); 218} 219 220asmlinkage void sha512_transform_rorx(struct sha512_state *state, 221 const u8 *data, int blocks); 222 223static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, 224 unsigned int len) 225{ 226 return sha512_update(desc, data, len, sha512_transform_rorx); 227} 228 229static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, 230 unsigned int len, u8 *out) 231{ 232 return sha512_finup(desc, data, len, out, sha512_transform_rorx); 233} 234 235/* Add padding and return the message digest. */ 236static int sha512_avx2_final(struct shash_desc *desc, u8 *out) 237{ 238 return sha512_avx2_finup(desc, NULL, 0, out); 239} 240 241static struct shash_alg sha512_avx2_algs[] = { { 242 .digestsize = SHA512_DIGEST_SIZE, 243 .init = sha512_base_init, 244 .update = sha512_avx2_update, 245 .final = sha512_avx2_final, 246 .finup = sha512_avx2_finup, 247 .descsize = sizeof(struct sha512_state), 248 .base = { 249 .cra_name = "sha512", 250 .cra_driver_name = "sha512-avx2", 251 .cra_priority = 170, 252 .cra_blocksize = SHA512_BLOCK_SIZE, 253 .cra_module = THIS_MODULE, 254 } 255}, { 256 .digestsize = SHA384_DIGEST_SIZE, 257 .init = sha384_base_init, 258 .update = sha512_avx2_update, 259 .final = sha512_avx2_final, 260 .finup = sha512_avx2_finup, 261 .descsize = sizeof(struct sha512_state), 262 .base = { 263 .cra_name = "sha384", 264 .cra_driver_name = "sha384-avx2", 265 .cra_priority = 170, 266 .cra_blocksize = SHA384_BLOCK_SIZE, 267 .cra_module = THIS_MODULE, 268 } 269} }; 270 271static bool avx2_usable(void) 272{ 273 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && 274 boot_cpu_has(X86_FEATURE_BMI2)) 275 return true; 276 277 return false; 278} 279 280static int register_sha512_avx2(void) 281{ 282 if (avx2_usable()) 283 return crypto_register_shashes(sha512_avx2_algs, 284 ARRAY_SIZE(sha512_avx2_algs)); 285 return 0; 286} 287 288static void unregister_sha512_avx2(void) 289{ 290 if (avx2_usable()) 291 crypto_unregister_shashes(sha512_avx2_algs, 292 ARRAY_SIZE(sha512_avx2_algs)); 293} 294 295static int __init sha512_ssse3_mod_init(void) 296{ 297 298 if (register_sha512_ssse3()) 299 goto fail; 300 301 if (register_sha512_avx()) { 302 unregister_sha512_ssse3(); 303 goto fail; 304 } 305 306 if (register_sha512_avx2()) { 307 unregister_sha512_avx(); 308 unregister_sha512_ssse3(); 309 goto fail; 310 } 311 312 return 0; 313fail: 314 return -ENODEV; 315} 316 317static void __exit sha512_ssse3_mod_fini(void) 318{ 319 unregister_sha512_avx2(); 320 unregister_sha512_avx(); 321 unregister_sha512_ssse3(); 322} 323 324module_init(sha512_ssse3_mod_init); 325module_exit(sha512_ssse3_mod_fini); 326 327MODULE_LICENSE("GPL"); 328MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 329 330MODULE_ALIAS_CRYPTO("sha512"); 331MODULE_ALIAS_CRYPTO("sha512-ssse3"); 332MODULE_ALIAS_CRYPTO("sha512-avx"); 333MODULE_ALIAS_CRYPTO("sha512-avx2"); 334MODULE_ALIAS_CRYPTO("sha384"); 335MODULE_ALIAS_CRYPTO("sha384-ssse3"); 336MODULE_ALIAS_CRYPTO("sha384-avx"); 337MODULE_ALIAS_CRYPTO("sha384-avx2");