1 /* 2 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 3 * 4 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <linux/crc32.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 18 #include <crypto/internal/hash.h> 19 20 #include <asm/hwcap.h> 21 #include <asm/neon.h> 22 #include <asm/simd.h> 23 #include <asm/unaligned.h> 24 25 #define PMULL_MIN_LEN 64L /* minimum size of buffer 26 * for crc32_pmull_le_16 */ 27 #define SCALE_F 16L /* size of NEON register */ 28 29 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 30 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 31 32 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 33 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 34 35 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len); 36 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len); 37 38 static int crc32_cra_init(struct crypto_tfm *tfm) 39 { 40 u32 *key = crypto_tfm_ctx(tfm); 41 42 *key = 0; 43 return 0; 44 } 45 46 static int crc32c_cra_init(struct crypto_tfm *tfm) 47 { 48 u32 *key = crypto_tfm_ctx(tfm); 49 50 *key = ~0; 51 return 0; 52 } 53 54 static int crc32_setkey(struct crypto_shash *hash, const u8 *key, 55 unsigned int keylen) 56 { 57 u32 *mctx = crypto_shash_ctx(hash); 58 59 if (keylen != sizeof(u32)) { 60 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 61 return -EINVAL; 62 } 63 *mctx = le32_to_cpup((__le32 *)key); 64 return 0; 65 } 66 67 static int crc32_init(struct shash_desc *desc) 68 { 69 u32 *mctx = crypto_shash_ctx(desc->tfm); 70 u32 *crc = shash_desc_ctx(desc); 71 72 *crc = *mctx; 73 return 0; 74 } 75 76 static int crc32_update(struct shash_desc *desc, const u8 *data, 77 unsigned int length) 78 { 79 u32 *crc = shash_desc_ctx(desc); 80 81 *crc = crc32_armv8_le(*crc, data, length); 82 return 0; 83 } 84 85 static int crc32c_update(struct shash_desc *desc, const u8 *data, 86 unsigned int length) 87 { 88 u32 *crc = shash_desc_ctx(desc); 89 90 *crc = crc32c_armv8_le(*crc, data, length); 91 return 0; 92 } 93 94 static int crc32_final(struct shash_desc *desc, u8 *out) 95 { 96 u32 *crc = shash_desc_ctx(desc); 97 98 put_unaligned_le32(*crc, out); 99 return 0; 100 } 101 102 static int crc32c_final(struct shash_desc *desc, u8 *out) 103 { 104 u32 *crc = shash_desc_ctx(desc); 105 106 put_unaligned_le32(~*crc, out); 107 return 0; 108 } 109 110 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, 111 unsigned int length) 112 { 113 u32 *crc = shash_desc_ctx(desc); 114 unsigned int l; 115 116 if (may_use_simd()) { 117 if ((u32)data % SCALE_F) { 118 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 119 120 *crc = fallback_crc32(*crc, data, l); 121 122 data += l; 123 length -= l; 124 } 125 126 if (length >= PMULL_MIN_LEN) { 127 l = round_down(length, SCALE_F); 128 129 kernel_neon_begin(); 130 *crc = crc32_pmull_le(data, l, *crc); 131 kernel_neon_end(); 132 133 data += l; 134 length -= l; 135 } 136 } 137 138 if (length > 0) 139 *crc = fallback_crc32(*crc, data, length); 140 141 return 0; 142 } 143 144 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data, 145 unsigned int length) 146 { 147 u32 *crc = shash_desc_ctx(desc); 148 unsigned int l; 149 150 if (may_use_simd()) { 151 if ((u32)data % SCALE_F) { 152 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 153 154 *crc = fallback_crc32c(*crc, data, l); 155 156 data += l; 157 length -= l; 158 } 159 160 if (length >= PMULL_MIN_LEN) { 161 l = round_down(length, SCALE_F); 162 163 kernel_neon_begin(); 164 *crc = crc32c_pmull_le(data, l, *crc); 165 kernel_neon_end(); 166 167 data += l; 168 length -= l; 169 } 170 } 171 172 if (length > 0) 173 *crc = fallback_crc32c(*crc, data, length); 174 175 return 0; 176 } 177 178 static struct shash_alg crc32_pmull_algs[] = { { 179 .setkey = crc32_setkey, 180 .init = crc32_init, 181 .update = crc32_update, 182 .final = crc32_final, 183 .descsize = sizeof(u32), 184 .digestsize = sizeof(u32), 185 186 .base.cra_ctxsize = sizeof(u32), 187 .base.cra_init = crc32_cra_init, 188 .base.cra_name = "crc32", 189 .base.cra_driver_name = "crc32-arm-ce", 190 .base.cra_priority = 200, 191 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 192 .base.cra_blocksize = 1, 193 .base.cra_module = THIS_MODULE, 194 }, { 195 .setkey = crc32_setkey, 196 .init = crc32_init, 197 .update = crc32c_update, 198 .final = crc32c_final, 199 .descsize = sizeof(u32), 200 .digestsize = sizeof(u32), 201 202 .base.cra_ctxsize = sizeof(u32), 203 .base.cra_init = crc32c_cra_init, 204 .base.cra_name = "crc32c", 205 .base.cra_driver_name = "crc32c-arm-ce", 206 .base.cra_priority = 200, 207 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 208 .base.cra_blocksize = 1, 209 .base.cra_module = THIS_MODULE, 210 } }; 211 212 static int __init crc32_pmull_mod_init(void) 213 { 214 if (elf_hwcap2 & HWCAP2_PMULL) { 215 crc32_pmull_algs[0].update = crc32_pmull_update; 216 crc32_pmull_algs[1].update = crc32c_pmull_update; 217 218 if (elf_hwcap2 & HWCAP2_CRC32) { 219 fallback_crc32 = crc32_armv8_le; 220 fallback_crc32c = crc32c_armv8_le; 221 } else { 222 fallback_crc32 = crc32_le; 223 fallback_crc32c = __crc32c_le; 224 } 225 } else if (!(elf_hwcap2 & HWCAP2_CRC32)) { 226 return -ENODEV; 227 } 228 229 return crypto_register_shashes(crc32_pmull_algs, 230 ARRAY_SIZE(crc32_pmull_algs)); 231 } 232 233 static void __exit crc32_pmull_mod_exit(void) 234 { 235 crypto_unregister_shashes(crc32_pmull_algs, 236 ARRAY_SIZE(crc32_pmull_algs)); 237 } 238 239 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = { 240 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { } 241 }; 242 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature); 243 244 module_init(crc32_pmull_mod_init); 245 module_exit(crc32_pmull_mod_exit); 246 247 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 248 MODULE_LICENSE("GPL v2"); 249 MODULE_ALIAS_CRYPTO("crc32"); 250 MODULE_ALIAS_CRYPTO("crc32c"); 251