arch/x86/crypto/sha256_ssse3_glue.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Glue code for the SHA256 Secure Hash Algorithm assembler
   5  * implementation using supplemental SSE3 / AVX / AVX2 instructions.
   6  *
   7  * This file is based on sha256_generic.c
   8  *
   9  * Copyright (C) 2013 Intel Corporation.
  10  *
  11  * Author:
  12  *     Tim Chen <tim.c.chen@linux.intel.com>
  13  *
  14  * This program is free software; you can redistribute it and/or modify it
  15  * under the terms of the GNU General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option)
  17  * any later version.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  22  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  23  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  24  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  25  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  26  * SOFTWARE.
  27  */
  28
  29
  30 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  31
  32 #include <crypto/internal/hash.h>
  33 #include <linux/init.h>
  34 #include <linux/module.h>
  35 #include <linux/mm.h>
  36 #include <linux/cryptohash.h>
  37 #include <linux/types.h>
  38 #include <crypto/sha.h>
  39 #include <crypto/sha256_base.h>
  40 #include <asm/fpu/api.h>
  41 #include <linux/string.h>
  42
  43 asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data,
  44                                        u64 rounds);
  45 #ifdef CONFIG_AS_AVX
  46 asmlinkage void sha256_transform_avx(u32 *digest, const char *data,
  47                                      u64 rounds);
  48 #endif
  49 #ifdef CONFIG_AS_AVX2
  50 asmlinkage void sha256_transform_rorx(u32 *digest, const char *data,
  51                                       u64 rounds);
  52 #endif
  53 #ifdef CONFIG_AS_SHA256_NI
  54 asmlinkage void sha256_ni_transform(u32 *digest, const char *data,
  55                                    u64 rounds); /*unsigned int rounds);*/
  56 #endif
  57
  58 static void (*sha256_transform_asm)(u32 *, const char *, u64);
  59
  60 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
  61                              unsigned int len)
  62 {
  63         struct sha256_state *sctx = shash_desc_ctx(desc);
  64
  65         if (!irq_fpu_usable() ||
  66             (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
  67                 return crypto_sha256_update(desc, data, len);
  68
  69         /* make sure casting to sha256_block_fn() is safe */
  70         BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
  71
  72         kernel_fpu_begin();
  73         sha256_base_do_update(desc, data, len,
  74                               (sha256_block_fn *)sha256_transform_asm);
  75         kernel_fpu_end();
  76
  77         return 0;
  78 }
  79
  80 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
  81                               unsigned int len, u8 *out)
  82 {
  83         if (!irq_fpu_usable())
  84                 return crypto_sha256_finup(desc, data, len, out);
  85
  86         kernel_fpu_begin();
  87         if (len)
  88                 sha256_base_do_update(desc, data, len,
  89                                       (sha256_block_fn *)sha256_transform_asm);
  90         sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_transform_asm);
  91         kernel_fpu_end();
  92
  93         return sha256_base_finish(desc, out);
  94 }
  95
  96 /* Add padding and return the message digest. */
  97 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
  98 {
  99         return sha256_ssse3_finup(desc, NULL, 0, out);
 100 }
 101
 102 static struct shash_alg algs[] = { {
 103         .digestsize     =       SHA256_DIGEST_SIZE,
 104         .init           =       sha256_base_init,
 105         .update         =       sha256_ssse3_update,
 106         .final          =       sha256_ssse3_final,
 107         .finup          =       sha256_ssse3_finup,
 108         .descsize       =       sizeof(struct sha256_state),
 109         .base           =       {
 110                 .cra_name       =       "sha256",
 111                 .cra_driver_name =      "sha256-ssse3",
 112                 .cra_priority   =       150,
 113                 .cra_flags      =       CRYPTO_ALG_TYPE_SHASH,
 114                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 115                 .cra_module     =       THIS_MODULE,
 116         }
 117 }, {
 118         .digestsize     =       SHA224_DIGEST_SIZE,
 119         .init           =       sha224_base_init,
 120         .update         =       sha256_ssse3_update,
 121         .final          =       sha256_ssse3_final,
 122         .finup          =       sha256_ssse3_finup,
 123         .descsize       =       sizeof(struct sha256_state),
 124         .base           =       {
 125                 .cra_name       =       "sha224",
 126                 .cra_driver_name =      "sha224-ssse3",
 127                 .cra_priority   =       150,
 128                 .cra_flags      =       CRYPTO_ALG_TYPE_SHASH,
 129                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 130                 .cra_module     =       THIS_MODULE,
 131         }
 132 } };
 133
 134 #ifdef CONFIG_AS_AVX
 135 static bool __init avx_usable(void)
 136 {
 137         if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) {
 138                 if (cpu_has_avx)
 139                         pr_info("AVX detected but unusable.\n");
 140                 return false;
 141         }
 142
 143         return true;
 144 }
 145 #endif
 146
 147 static int __init sha256_ssse3_mod_init(void)
 148 {
 149         char *algo;
 150
 151         /* test for SSSE3 first */
 152         if (cpu_has_ssse3) {
 153                 sha256_transform_asm = sha256_transform_ssse3;
 154                 algo = "SSSE3";
 155         }
 156
 157 #ifdef CONFIG_AS_AVX
 158         /* allow AVX to override SSSE3, it's a little faster */
 159         if (avx_usable()) {
 160                 sha256_transform_asm = sha256_transform_avx;
 161                 algo = "AVX";
 162 #ifdef CONFIG_AS_AVX2
 163                 if (boot_cpu_has(X86_FEATURE_AVX2) &&
 164                     boot_cpu_has(X86_FEATURE_BMI2)) {
 165                         sha256_transform_asm = sha256_transform_rorx;
 166                         algo = "AVX2";
 167                 }
 168 #endif
 169         }
 170 #endif
 171 #ifdef CONFIG_AS_SHA256_NI
 172         if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
 173                 sha256_transform_asm = sha256_ni_transform;
 174                 algo = "SHA-256-NI";
 175         }
 176 #endif
 177
 178         if (sha256_transform_asm) {
 179                 pr_info("Using %s optimized SHA-256 implementation\n", algo);
 180                 return crypto_register_shashes(algs, ARRAY_SIZE(algs));
 181         }
 182         pr_info("Neither AVX nor SSSE3/SHA-NI is available/usable.\n");
 183
 184         return -ENODEV;
 185 }
 186
 187 static void __exit sha256_ssse3_mod_fini(void)
 188 {
 189         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 190 }
 191
 192 module_init(sha256_ssse3_mod_init);
 193 module_exit(sha256_ssse3_mod_fini);
 194
 195 MODULE_LICENSE("GPL");
 196 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 197
 198 MODULE_ALIAS_CRYPTO("sha256");
 199 MODULE_ALIAS_CRYPTO("sha224");