1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/crc32.h> 3 #include <crypto/internal/hash.h> 4 #include <crypto/internal/simd.h> 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/string.h> 8 #include <linux/kernel.h> 9 #include <linux/cpufeature.h> 10 #include <asm/simd.h> 11 #include <asm/switch_to.h> 12 13 #define CHKSUM_BLOCK_SIZE 1 14 #define CHKSUM_DIGEST_SIZE 4 15 16 #define VMX_ALIGN 16 17 #define VMX_ALIGN_MASK (VMX_ALIGN-1) 18 19 #define VECTOR_BREAKPOINT 512 20 21 u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len); 22 23 static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len) 24 { 25 unsigned int prealign; 26 unsigned int tail; 27 28 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable()) 29 return __crc32c_le(crc, p, len); 30 31 if ((unsigned long)p & VMX_ALIGN_MASK) { 32 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 33 crc = __crc32c_le(crc, p, prealign); 34 len -= prealign; 35 p += prealign; 36 } 37 38 if (len & ~VMX_ALIGN_MASK) { 39 preempt_disable(); 40 pagefault_disable(); 41 enable_kernel_altivec(); 42 crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 43 disable_kernel_altivec(); 44 pagefault_enable(); 45 preempt_enable(); 46 } 47 48 tail = len & VMX_ALIGN_MASK; 49 if (tail) { 50 p += len & ~VMX_ALIGN_MASK; 51 crc = __crc32c_le(crc, p, tail); 52 } 53 54 return crc; 55 } 56 57 static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm) 58 { 59 u32 *key = crypto_tfm_ctx(tfm); 60 61 *key = ~0; 62 63 return 0; 64 } 65 66 /* 67 * Setting the seed allows arbitrary accumulators and flexible XOR policy 68 * If your algorithm starts with ~0, then XOR with ~0 before you set 69 * the seed. 70 */ 71 static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key, 72 unsigned int keylen) 73 { 74 u32 *mctx = crypto_shash_ctx(hash); 75 76 if (keylen != sizeof(u32)) { 77 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 78 return -EINVAL; 79 } 80 *mctx = le32_to_cpup((__le32 *)key); 81 return 0; 82 } 83 84 static int crc32c_vpmsum_init(struct shash_desc *desc) 85 { 86 u32 *mctx = crypto_shash_ctx(desc->tfm); 87 u32 *crcp = shash_desc_ctx(desc); 88 89 *crcp = *mctx; 90 91 return 0; 92 } 93 94 static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data, 95 unsigned int len) 96 { 97 u32 *crcp = shash_desc_ctx(desc); 98 99 *crcp = crc32c_vpmsum(*crcp, data, len); 100 101 return 0; 102 } 103 104 static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len, 105 u8 *out) 106 { 107 *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len)); 108 109 return 0; 110 } 111 112 static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data, 113 unsigned int len, u8 *out) 114 { 115 return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out); 116 } 117 118 static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out) 119 { 120 u32 *crcp = shash_desc_ctx(desc); 121 122 *(__le32 *)out = ~cpu_to_le32p(crcp); 123 124 return 0; 125 } 126 127 static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data, 128 unsigned int len, u8 *out) 129 { 130 return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len, 131 out); 132 } 133 134 static struct shash_alg alg = { 135 .setkey = crc32c_vpmsum_setkey, 136 .init = crc32c_vpmsum_init, 137 .update = crc32c_vpmsum_update, 138 .final = crc32c_vpmsum_final, 139 .finup = crc32c_vpmsum_finup, 140 .digest = crc32c_vpmsum_digest, 141 .descsize = sizeof(u32), 142 .digestsize = CHKSUM_DIGEST_SIZE, 143 .base = { 144 .cra_name = "crc32c", 145 .cra_driver_name = "crc32c-vpmsum", 146 .cra_priority = 200, 147 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 148 .cra_blocksize = CHKSUM_BLOCK_SIZE, 149 .cra_ctxsize = sizeof(u32), 150 .cra_module = THIS_MODULE, 151 .cra_init = crc32c_vpmsum_cra_init, 152 } 153 }; 154 155 static int __init crc32c_vpmsum_mod_init(void) 156 { 157 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 158 return -ENODEV; 159 160 return crypto_register_shash(&alg); 161 } 162 163 static void __exit crc32c_vpmsum_mod_fini(void) 164 { 165 crypto_unregister_shash(&alg); 166 } 167 168 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init); 169 module_exit(crc32c_vpmsum_mod_fini); 170 171 MODULE_AUTHOR("Anton Blanchard <anton@samba.org>"); 172 MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions"); 173 MODULE_LICENSE("GPL"); 174 MODULE_ALIAS_CRYPTO("crc32c"); 175 MODULE_ALIAS_CRYPTO("crc32c-vpmsum"); 176