arch/arm64/crypto/sha256-glue.c

   1 /*
   2  * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
   3  *
   4  * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the Free
   8  * Software Foundation; either version 2 of the License, or (at your option)
   9  * any later version.
  10  *
  11  */
  12
  13 #include <asm/hwcap.h>
  14 #include <asm/neon.h>
  15 #include <asm/simd.h>
  16 #include <crypto/internal/hash.h>
  17 #include <crypto/sha.h>
  18 #include <crypto/sha256_base.h>
  19 #include <linux/cryptohash.h>
  20 #include <linux/types.h>
  21 #include <linux/string.h>
  22
  23 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
  24 MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
  25 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  26 MODULE_LICENSE("GPL v2");
  27 MODULE_ALIAS_CRYPTO("sha224");
  28 MODULE_ALIAS_CRYPTO("sha256");
  29
  30 asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
  31                                         unsigned int num_blks);
  32
  33 asmlinkage void sha256_block_neon(u32 *digest, const void *data,
  34                                   unsigned int num_blks);
  35
  36 static int sha256_update(struct shash_desc *desc, const u8 *data,
  37                          unsigned int len)
  38 {
  39         return sha256_base_do_update(desc, data, len,
  40                                 (sha256_block_fn *)sha256_block_data_order);
  41 }
  42
  43 static int sha256_finup(struct shash_desc *desc, const u8 *data,
  44                         unsigned int len, u8 *out)
  45 {
  46         if (len)
  47                 sha256_base_do_update(desc, data, len,
  48                                 (sha256_block_fn *)sha256_block_data_order);
  49         sha256_base_do_finalize(desc,
  50                                 (sha256_block_fn *)sha256_block_data_order);
  51
  52         return sha256_base_finish(desc, out);
  53 }
  54
  55 static int sha256_final(struct shash_desc *desc, u8 *out)
  56 {
  57         return sha256_finup(desc, NULL, 0, out);
  58 }
  59
  60 static struct shash_alg algs[] = { {
  61         .digestsize             = SHA256_DIGEST_SIZE,
  62         .init                   = sha256_base_init,
  63         .update                 = sha256_update,
  64         .final                  = sha256_final,
  65         .finup                  = sha256_finup,
  66         .descsize               = sizeof(struct sha256_state),
  67         .base.cra_name          = "sha256",
  68         .base.cra_driver_name   = "sha256-arm64",
  69         .base.cra_priority      = 100,
  70         .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
  71         .base.cra_blocksize     = SHA256_BLOCK_SIZE,
  72         .base.cra_module        = THIS_MODULE,
  73 }, {
  74         .digestsize             = SHA224_DIGEST_SIZE,
  75         .init                   = sha224_base_init,
  76         .update                 = sha256_update,
  77         .final                  = sha256_final,
  78         .finup                  = sha256_finup,
  79         .descsize               = sizeof(struct sha256_state),
  80         .base.cra_name          = "sha224",
  81         .base.cra_driver_name   = "sha224-arm64",
  82         .base.cra_priority      = 100,
  83         .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
  84         .base.cra_blocksize     = SHA224_BLOCK_SIZE,
  85         .base.cra_module        = THIS_MODULE,
  86 } };
  87
  88 static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
  89                               unsigned int len)
  90 {
  91         /*
  92          * Stacking and unstacking a substantial slice of the NEON register
  93          * file may significantly affect performance for small updates when
  94          * executing in interrupt context, so fall back to the scalar code
  95          * in that case.
  96          */
  97         if (!may_use_simd())
  98                 return sha256_base_do_update(desc, data, len,
  99                                 (sha256_block_fn *)sha256_block_data_order);
 100
 101         kernel_neon_begin();
 102         sha256_base_do_update(desc, data, len,
 103                                 (sha256_block_fn *)sha256_block_neon);
 104         kernel_neon_end();
 105
 106         return 0;
 107 }
 108
 109 static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
 110                              unsigned int len, u8 *out)
 111 {
 112         if (!may_use_simd()) {
 113                 if (len)
 114                         sha256_base_do_update(desc, data, len,
 115                                 (sha256_block_fn *)sha256_block_data_order);
 116                 sha256_base_do_finalize(desc,
 117                                 (sha256_block_fn *)sha256_block_data_order);
 118         } else {
 119                 kernel_neon_begin();
 120                 if (len)
 121                         sha256_base_do_update(desc, data, len,
 122                                 (sha256_block_fn *)sha256_block_neon);
 123                 sha256_base_do_finalize(desc,
 124                                 (sha256_block_fn *)sha256_block_neon);
 125                 kernel_neon_end();
 126         }
 127         return sha256_base_finish(desc, out);
 128 }
 129
 130 static int sha256_final_neon(struct shash_desc *desc, u8 *out)
 131 {
 132         return sha256_finup_neon(desc, NULL, 0, out);
 133 }
 134
 135 static struct shash_alg neon_algs[] = { {
 136         .digestsize             = SHA256_DIGEST_SIZE,
 137         .init                   = sha256_base_init,
 138         .update                 = sha256_update_neon,
 139         .final                  = sha256_final_neon,
 140         .finup                  = sha256_finup_neon,
 141         .descsize               = sizeof(struct sha256_state),
 142         .base.cra_name          = "sha256",
 143         .base.cra_driver_name   = "sha256-arm64-neon",
 144         .base.cra_priority      = 150,
 145         .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
 146         .base.cra_blocksize     = SHA256_BLOCK_SIZE,
 147         .base.cra_module        = THIS_MODULE,
 148 }, {
 149         .digestsize             = SHA224_DIGEST_SIZE,
 150         .init                   = sha224_base_init,
 151         .update                 = sha256_update_neon,
 152         .final                  = sha256_final_neon,
 153         .finup                  = sha256_finup_neon,
 154         .descsize               = sizeof(struct sha256_state),
 155         .base.cra_name          = "sha224",
 156         .base.cra_driver_name   = "sha224-arm64-neon",
 157         .base.cra_priority      = 150,
 158         .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
 159         .base.cra_blocksize     = SHA224_BLOCK_SIZE,
 160         .base.cra_module        = THIS_MODULE,
 161 } };
 162
 163 static int __init sha256_mod_init(void)
 164 {
 165         int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
 166         if (ret)
 167                 return ret;
 168
 169         if (elf_hwcap & HWCAP_ASIMD) {
 170                 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
 171                 if (ret)
 172                         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 173         }
 174         return ret;
 175 }
 176
 177 static void __exit sha256_mod_fini(void)
 178 {
 179         if (elf_hwcap & HWCAP_ASIMD)
 180                 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
 181         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 182 }
 183
 184 module_init(sha256_mod_init);
 185 module_exit(sha256_mod_fini);