1 /* 2 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 3 * 4 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/cpufeature.h> 12 #include <linux/crc-t10dif.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 18 #include <crypto/internal/hash.h> 19 20 #include <asm/neon.h> 21 #include <asm/simd.h> 22 23 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 24 25 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); 26 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 27 28 static int crct10dif_init(struct shash_desc *desc) 29 { 30 u16 *crc = shash_desc_ctx(desc); 31 32 *crc = 0; 33 return 0; 34 } 35 36 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, 37 unsigned int length) 38 { 39 u16 *crc = shash_desc_ctx(desc); 40 41 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { 42 kernel_neon_begin(); 43 *crc = crc_t10dif_pmull_p8(*crc, data, length); 44 kernel_neon_end(); 45 } else { 46 *crc = crc_t10dif_generic(*crc, data, length); 47 } 48 49 return 0; 50 } 51 52 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, 53 unsigned int length) 54 { 55 u16 *crc = shash_desc_ctx(desc); 56 57 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && may_use_simd()) { 58 kernel_neon_begin(); 59 *crc = crc_t10dif_pmull_p64(*crc, data, length); 60 kernel_neon_end(); 61 } else { 62 *crc = crc_t10dif_generic(*crc, data, length); 63 } 64 65 return 0; 66 } 67 68 static int crct10dif_final(struct shash_desc *desc, u8 *out) 69 { 70 u16 *crc = shash_desc_ctx(desc); 71 72 *(u16 *)out = *crc; 73 return 0; 74 } 75 76 static struct shash_alg crc_t10dif_alg[] = {{ 77 .digestsize = CRC_T10DIF_DIGEST_SIZE, 78 .init = crct10dif_init, 79 .update = crct10dif_update_pmull_p8, 80 .final = crct10dif_final, 81 .descsize = CRC_T10DIF_DIGEST_SIZE, 82 83 .base.cra_name = "crct10dif", 84 .base.cra_driver_name = "crct10dif-arm64-neon", 85 .base.cra_priority = 100, 86 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 87 .base.cra_module = THIS_MODULE, 88 }, { 89 .digestsize = CRC_T10DIF_DIGEST_SIZE, 90 .init = crct10dif_init, 91 .update = crct10dif_update_pmull_p64, 92 .final = crct10dif_final, 93 .descsize = CRC_T10DIF_DIGEST_SIZE, 94 95 .base.cra_name = "crct10dif", 96 .base.cra_driver_name = "crct10dif-arm64-ce", 97 .base.cra_priority = 200, 98 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, 99 .base.cra_module = THIS_MODULE, 100 }}; 101 102 static int __init crc_t10dif_mod_init(void) 103 { 104 if (elf_hwcap & HWCAP_PMULL) 105 return crypto_register_shashes(crc_t10dif_alg, 106 ARRAY_SIZE(crc_t10dif_alg)); 107 else 108 /* only register the first array element */ 109 return crypto_register_shash(crc_t10dif_alg); 110 } 111 112 static void __exit crc_t10dif_mod_exit(void) 113 { 114 if (elf_hwcap & HWCAP_PMULL) 115 crypto_unregister_shashes(crc_t10dif_alg, 116 ARRAY_SIZE(crc_t10dif_alg)); 117 else 118 crypto_unregister_shash(crc_t10dif_alg); 119 } 120 121 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); 122 module_exit(crc_t10dif_mod_exit); 123 124 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 125 MODULE_LICENSE("GPL v2"); 126 MODULE_ALIAS_CRYPTO("crct10dif"); 127 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce"); 128