1 /* 2 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 3 * 4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <linux/crc32.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 21 #include <asm/hwcap.h> 22 #include <asm/neon.h> 23 #include <asm/simd.h> 24 #include <asm/unaligned.h> 25 26 #define PMULL_MIN_LEN 64L /* minimum size of buffer 27 * for crc32_pmull_le_16 */ 28 #define SCALE_F 16L /* size of NEON register */ 29 30 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 31 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 32 33 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 34 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 35 36 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len); 37 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len); 38 39 static int crc32_cra_init(struct crypto_tfm *tfm) 40 { 41 u32 *key = crypto_tfm_ctx(tfm); 42 43 *key = 0; 44 return 0; 45 } 46 47 static int crc32c_cra_init(struct crypto_tfm *tfm) 48 { 49 u32 *key = crypto_tfm_ctx(tfm); 50 51 *key = ~0; 52 return 0; 53 } 54 55 static int crc32_setkey(struct crypto_shash *hash, const u8 *key, 56 unsigned int keylen) 57 { 58 u32 *mctx = crypto_shash_ctx(hash); 59 60 if (keylen != sizeof(u32)) { 61 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 62 return -EINVAL; 63 } 64 *mctx = le32_to_cpup((__le32 *)key); 65 return 0; 66 } 67 68 static int crc32_init(struct shash_desc *desc) 69 { 70 u32 *mctx = crypto_shash_ctx(desc->tfm); 71 u32 *crc = shash_desc_ctx(desc); 72 73 *crc = *mctx; 74 return 0; 75 } 76 77 static int crc32_update(struct shash_desc *desc, const u8 *data, 78 unsigned int length) 79 { 80 u32 *crc = shash_desc_ctx(desc); 81 82 *crc = crc32_armv8_le(*crc, data, length); 83 return 0; 84 } 85 86 static int crc32c_update(struct shash_desc *desc, const u8 *data, 87 unsigned int length) 88 { 89 u32 *crc = shash_desc_ctx(desc); 90 91 *crc = crc32c_armv8_le(*crc, data, length); 92 return 0; 93 } 94 95 static int crc32_final(struct shash_desc *desc, u8 *out) 96 { 97 u32 *crc = shash_desc_ctx(desc); 98 99 put_unaligned_le32(*crc, out); 100 return 0; 101 } 102 103 static int crc32c_final(struct shash_desc *desc, u8 *out) 104 { 105 u32 *crc = shash_desc_ctx(desc); 106 107 put_unaligned_le32(~*crc, out); 108 return 0; 109 } 110 111 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, 112 unsigned int length) 113 { 114 u32 *crc = shash_desc_ctx(desc); 115 unsigned int l; 116 117 if (crypto_simd_usable()) { 118 if ((u32)data % SCALE_F) { 119 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 120 121 *crc = fallback_crc32(*crc, data, l); 122 123 data += l; 124 length -= l; 125 } 126 127 if (length >= PMULL_MIN_LEN) { 128 l = round_down(length, SCALE_F); 129 130 kernel_neon_begin(); 131 *crc = crc32_pmull_le(data, l, *crc); 132 kernel_neon_end(); 133 134 data += l; 135 length -= l; 136 } 137 } 138 139 if (length > 0) 140 *crc = fallback_crc32(*crc, data, length); 141 142 return 0; 143 } 144 145 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data, 146 unsigned int length) 147 { 148 u32 *crc = shash_desc_ctx(desc); 149 unsigned int l; 150 151 if (crypto_simd_usable()) { 152 if ((u32)data % SCALE_F) { 153 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 154 155 *crc = fallback_crc32c(*crc, data, l); 156 157 data += l; 158 length -= l; 159 } 160 161 if (length >= PMULL_MIN_LEN) { 162 l = round_down(length, SCALE_F); 163 164 kernel_neon_begin(); 165 *crc = crc32c_pmull_le(data, l, *crc); 166 kernel_neon_end(); 167 168 data += l; 169 length -= l; 170 } 171 } 172 173 if (length > 0) 174 *crc = fallback_crc32c(*crc, data, length); 175 176 return 0; 177 } 178 179 static struct shash_alg crc32_pmull_algs[] = { { 180 .setkey = crc32_setkey, 181 .init = crc32_init, 182 .update = crc32_update, 183 .final = crc32_final, 184 .descsize = sizeof(u32), 185 .digestsize = sizeof(u32), 186 187 .base.cra_ctxsize = sizeof(u32), 188 .base.cra_init = crc32_cra_init, 189 .base.cra_name = "crc32", 190 .base.cra_driver_name = "crc32-arm-ce", 191 .base.cra_priority = 200, 192 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 193 .base.cra_blocksize = 1, 194 .base.cra_module = THIS_MODULE, 195 }, { 196 .setkey = crc32_setkey, 197 .init = crc32_init, 198 .update = crc32c_update, 199 .final = crc32c_final, 200 .descsize = sizeof(u32), 201 .digestsize = sizeof(u32), 202 203 .base.cra_ctxsize = sizeof(u32), 204 .base.cra_init = crc32c_cra_init, 205 .base.cra_name = "crc32c", 206 .base.cra_driver_name = "crc32c-arm-ce", 207 .base.cra_priority = 200, 208 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 209 .base.cra_blocksize = 1, 210 .base.cra_module = THIS_MODULE, 211 } }; 212 213 static int __init crc32_pmull_mod_init(void) 214 { 215 if (elf_hwcap2 & HWCAP2_PMULL) { 216 crc32_pmull_algs[0].update = crc32_pmull_update; 217 crc32_pmull_algs[1].update = crc32c_pmull_update; 218 219 if (elf_hwcap2 & HWCAP2_CRC32) { 220 fallback_crc32 = crc32_armv8_le; 221 fallback_crc32c = crc32c_armv8_le; 222 } else { 223 fallback_crc32 = crc32_le; 224 fallback_crc32c = __crc32c_le; 225 } 226 } else if (!(elf_hwcap2 & HWCAP2_CRC32)) { 227 return -ENODEV; 228 } 229 230 return crypto_register_shashes(crc32_pmull_algs, 231 ARRAY_SIZE(crc32_pmull_algs)); 232 } 233 234 static void __exit crc32_pmull_mod_exit(void) 235 { 236 crypto_unregister_shashes(crc32_pmull_algs, 237 ARRAY_SIZE(crc32_pmull_algs)); 238 } 239 240 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = { 241 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { } 242 }; 243 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature); 244 245 module_init(crc32_pmull_mod_init); 246 module_exit(crc32_pmull_mod_exit); 247 248 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 249 MODULE_LICENSE("GPL v2"); 250 MODULE_ALIAS_CRYPTO("crc32"); 251 MODULE_ALIAS_CRYPTO("crc32c"); 252