1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 4 * 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/cpufeature.h> 9 #include <linux/crc32.h> 10 #include <linux/init.h> 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/string.h> 14 15 #include <crypto/internal/hash.h> 16 #include <crypto/internal/simd.h> 17 18 #include <asm/hwcap.h> 19 #include <asm/neon.h> 20 #include <asm/simd.h> 21 #include <asm/unaligned.h> 22 23 #define PMULL_MIN_LEN 64L /* minimum size of buffer 24 * for crc32_pmull_le_16 */ 25 #define SCALE_F 16L /* size of NEON register */ 26 27 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 28 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 29 30 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 31 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 32 33 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len); 34 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len); 35 36 static int crc32_cra_init(struct crypto_tfm *tfm) 37 { 38 u32 *key = crypto_tfm_ctx(tfm); 39 40 *key = 0; 41 return 0; 42 } 43 44 static int crc32c_cra_init(struct crypto_tfm *tfm) 45 { 46 u32 *key = crypto_tfm_ctx(tfm); 47 48 *key = ~0; 49 return 0; 50 } 51 52 static int crc32_setkey(struct crypto_shash *hash, const u8 *key, 53 unsigned int keylen) 54 { 55 u32 *mctx = crypto_shash_ctx(hash); 56 57 if (keylen != sizeof(u32)) { 58 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 59 return -EINVAL; 60 } 61 *mctx = le32_to_cpup((__le32 *)key); 62 return 0; 63 } 64 65 static int crc32_init(struct shash_desc *desc) 66 { 67 u32 *mctx = crypto_shash_ctx(desc->tfm); 68 u32 *crc = shash_desc_ctx(desc); 69 70 *crc = *mctx; 71 return 0; 72 } 73 74 static int crc32_update(struct shash_desc *desc, const u8 *data, 75 unsigned int length) 76 { 77 u32 *crc = shash_desc_ctx(desc); 78 79 *crc = crc32_armv8_le(*crc, data, length); 80 return 0; 81 } 82 83 static int crc32c_update(struct shash_desc *desc, const u8 *data, 84 unsigned int length) 85 { 86 u32 *crc = shash_desc_ctx(desc); 87 88 *crc = crc32c_armv8_le(*crc, data, length); 89 return 0; 90 } 91 92 static int crc32_final(struct shash_desc *desc, u8 *out) 93 { 94 u32 *crc = shash_desc_ctx(desc); 95 96 put_unaligned_le32(*crc, out); 97 return 0; 98 } 99 100 static int crc32c_final(struct shash_desc *desc, u8 *out) 101 { 102 u32 *crc = shash_desc_ctx(desc); 103 104 put_unaligned_le32(~*crc, out); 105 return 0; 106 } 107 108 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, 109 unsigned int length) 110 { 111 u32 *crc = shash_desc_ctx(desc); 112 unsigned int l; 113 114 if (crypto_simd_usable()) { 115 if ((u32)data % SCALE_F) { 116 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 117 118 *crc = fallback_crc32(*crc, data, l); 119 120 data += l; 121 length -= l; 122 } 123 124 if (length >= PMULL_MIN_LEN) { 125 l = round_down(length, SCALE_F); 126 127 kernel_neon_begin(); 128 *crc = crc32_pmull_le(data, l, *crc); 129 kernel_neon_end(); 130 131 data += l; 132 length -= l; 133 } 134 } 135 136 if (length > 0) 137 *crc = fallback_crc32(*crc, data, length); 138 139 return 0; 140 } 141 142 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data, 143 unsigned int length) 144 { 145 u32 *crc = shash_desc_ctx(desc); 146 unsigned int l; 147 148 if (crypto_simd_usable()) { 149 if ((u32)data % SCALE_F) { 150 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 151 152 *crc = fallback_crc32c(*crc, data, l); 153 154 data += l; 155 length -= l; 156 } 157 158 if (length >= PMULL_MIN_LEN) { 159 l = round_down(length, SCALE_F); 160 161 kernel_neon_begin(); 162 *crc = crc32c_pmull_le(data, l, *crc); 163 kernel_neon_end(); 164 165 data += l; 166 length -= l; 167 } 168 } 169 170 if (length > 0) 171 *crc = fallback_crc32c(*crc, data, length); 172 173 return 0; 174 } 175 176 static struct shash_alg crc32_pmull_algs[] = { { 177 .setkey = crc32_setkey, 178 .init = crc32_init, 179 .update = crc32_update, 180 .final = crc32_final, 181 .descsize = sizeof(u32), 182 .digestsize = sizeof(u32), 183 184 .base.cra_ctxsize = sizeof(u32), 185 .base.cra_init = crc32_cra_init, 186 .base.cra_name = "crc32", 187 .base.cra_driver_name = "crc32-arm-ce", 188 .base.cra_priority = 200, 189 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 190 .base.cra_blocksize = 1, 191 .base.cra_module = THIS_MODULE, 192 }, { 193 .setkey = crc32_setkey, 194 .init = crc32_init, 195 .update = crc32c_update, 196 .final = crc32c_final, 197 .descsize = sizeof(u32), 198 .digestsize = sizeof(u32), 199 200 .base.cra_ctxsize = sizeof(u32), 201 .base.cra_init = crc32c_cra_init, 202 .base.cra_name = "crc32c", 203 .base.cra_driver_name = "crc32c-arm-ce", 204 .base.cra_priority = 200, 205 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 206 .base.cra_blocksize = 1, 207 .base.cra_module = THIS_MODULE, 208 } }; 209 210 static int __init crc32_pmull_mod_init(void) 211 { 212 if (elf_hwcap2 & HWCAP2_PMULL) { 213 crc32_pmull_algs[0].update = crc32_pmull_update; 214 crc32_pmull_algs[1].update = crc32c_pmull_update; 215 216 if (elf_hwcap2 & HWCAP2_CRC32) { 217 fallback_crc32 = crc32_armv8_le; 218 fallback_crc32c = crc32c_armv8_le; 219 } else { 220 fallback_crc32 = crc32_le; 221 fallback_crc32c = __crc32c_le; 222 } 223 } else if (!(elf_hwcap2 & HWCAP2_CRC32)) { 224 return -ENODEV; 225 } 226 227 return crypto_register_shashes(crc32_pmull_algs, 228 ARRAY_SIZE(crc32_pmull_algs)); 229 } 230 231 static void __exit crc32_pmull_mod_exit(void) 232 { 233 crypto_unregister_shashes(crc32_pmull_algs, 234 ARRAY_SIZE(crc32_pmull_algs)); 235 } 236 237 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = { 238 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { } 239 }; 240 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature); 241 242 module_init(crc32_pmull_mod_init); 243 module_exit(crc32_pmull_mod_exit); 244 245 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 246 MODULE_LICENSE("GPL v2"); 247 MODULE_ALIAS_CRYPTO("crc32"); 248 MODULE_ALIAS_CRYPTO("crc32c"); 249