1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
4 *
5 * Module based on arm64/crypto/crc32-arm.c
6 *
7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8 * Copyright (C) 2018 MIPS Tech, LLC
9 */
10
11 #include <linux/cpufeature.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/string.h>
16 #include <asm/mipsregs.h>
17 #include <asm/unaligned.h>
18
19 #include <crypto/internal/hash.h>
20
21 enum crc_op_size {
22 b, h, w, d,
23 };
24
25 enum crc_type {
26 crc32,
27 crc32c,
28 };
29
30 #ifndef TOOLCHAIN_SUPPORTS_CRC
31 #define _ASM_SET_CRC(OP, SZ, TYPE) \
32 _ASM_MACRO_3R(OP, rt, rs, rt2, \
33 ".ifnc \\rt, \\rt2\n\t" \
34 ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
35 ".endif\n\t" \
36 _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \
37 ((SZ) << 6) | ((TYPE) << 8)) \
38 _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \
39 ((SZ) << 14) | ((TYPE) << 3)))
40 #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
41 #else /* !TOOLCHAIN_SUPPORTS_CRC */
42 #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
43 #define _ASM_UNSET_CRC(op, SZ, TYPE)
44 #endif
45
46 #define __CRC32(crc, value, op, SZ, TYPE) \
47 do { \
48 __asm__ __volatile__( \
49 ".set push\n\t" \
50 _ASM_SET_CRC(op, SZ, TYPE) \
51 #op " %0, %1, %0\n\t" \
52 _ASM_UNSET_CRC(op, SZ, TYPE) \
53 ".set pop" \
54 : "+r" (crc) \
55 : "r" (value)); \
56 } while (0)
57
58 #define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0)
59 #define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0)
60 #define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0)
61 #define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0)
62 #define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1)
63 #define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1)
64 #define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1)
65 #define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1)
66
67 #define _CRC32(crc, value, size, op) \
68 _CRC32_##op##size(crc, value)
69
70 #define CRC32(crc, value, size) \
71 _CRC32(crc, value, size, crc32)
72
73 #define CRC32C(crc, value, size) \
74 _CRC32(crc, value, size, crc32c)
75
crc32_mips_le_hw(u32 crc_,const u8 * p,unsigned int len)76 static u32 crc32_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
77 {
78 u32 crc = crc_;
79
80 #ifdef CONFIG_64BIT
81 while (len >= sizeof(u64)) {
82 u64 value = get_unaligned_le64(p);
83
84 CRC32(crc, value, d);
85 p += sizeof(u64);
86 len -= sizeof(u64);
87 }
88
89 if (len & sizeof(u32)) {
90 #else /* !CONFIG_64BIT */
91 while (len >= sizeof(u32)) {
92 #endif
93 u32 value = get_unaligned_le32(p);
94
95 CRC32(crc, value, w);
96 p += sizeof(u32);
97 len -= sizeof(u32);
98 }
99
100 if (len & sizeof(u16)) {
101 u16 value = get_unaligned_le16(p);
102
103 CRC32(crc, value, h);
104 p += sizeof(u16);
105 }
106
107 if (len & sizeof(u8)) {
108 u8 value = *p++;
109
110 CRC32(crc, value, b);
111 }
112
113 return crc;
114 }
115
116 static u32 crc32c_mips_le_hw(u32 crc_, const u8 *p, unsigned int len)
117 {
118 u32 crc = crc_;
119
120 #ifdef CONFIG_64BIT
121 while (len >= sizeof(u64)) {
122 u64 value = get_unaligned_le64(p);
123
124 CRC32C(crc, value, d);
125 p += sizeof(u64);
126 len -= sizeof(u64);
127 }
128
129 if (len & sizeof(u32)) {
130 #else /* !CONFIG_64BIT */
131 while (len >= sizeof(u32)) {
132 #endif
133 u32 value = get_unaligned_le32(p);
134
135 CRC32C(crc, value, w);
136 p += sizeof(u32);
137 len -= sizeof(u32);
138 }
139
140 if (len & sizeof(u16)) {
141 u16 value = get_unaligned_le16(p);
142
143 CRC32C(crc, value, h);
144 p += sizeof(u16);
145 }
146
147 if (len & sizeof(u8)) {
148 u8 value = *p++;
149
150 CRC32C(crc, value, b);
151 }
152 return crc;
153 }
154
155 #define CHKSUM_BLOCK_SIZE 1
156 #define CHKSUM_DIGEST_SIZE 4
157
158 struct chksum_ctx {
159 u32 key;
160 };
161
162 struct chksum_desc_ctx {
163 u32 crc;
164 };
165
166 static int chksum_init(struct shash_desc *desc)
167 {
168 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
169 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
170
171 ctx->crc = mctx->key;
172
173 return 0;
174 }
175
176 /*
177 * Setting the seed allows arbitrary accumulators and flexible XOR policy
178 * If your algorithm starts with ~0, then XOR with ~0 before you set
179 * the seed.
180 */
181 static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
182 unsigned int keylen)
183 {
184 struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
185
186 if (keylen != sizeof(mctx->key))
187 return -EINVAL;
188 mctx->key = get_unaligned_le32(key);
189 return 0;
190 }
191
192 static int chksum_update(struct shash_desc *desc, const u8 *data,
193 unsigned int length)
194 {
195 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
196
197 ctx->crc = crc32_mips_le_hw(ctx->crc, data, length);
198 return 0;
199 }
200
201 static int chksumc_update(struct shash_desc *desc, const u8 *data,
202 unsigned int length)
203 {
204 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
205
206 ctx->crc = crc32c_mips_le_hw(ctx->crc, data, length);
207 return 0;
208 }
209
210 static int chksum_final(struct shash_desc *desc, u8 *out)
211 {
212 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
213
214 put_unaligned_le32(ctx->crc, out);
215 return 0;
216 }
217
218 static int chksumc_final(struct shash_desc *desc, u8 *out)
219 {
220 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
221
222 put_unaligned_le32(~ctx->crc, out);
223 return 0;
224 }
225
226 static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
227 {
228 put_unaligned_le32(crc32_mips_le_hw(crc, data, len), out);
229 return 0;
230 }
231
232 static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
233 {
234 put_unaligned_le32(~crc32c_mips_le_hw(crc, data, len), out);
235 return 0;
236 }
237
238 static int chksum_finup(struct shash_desc *desc, const u8 *data,
239 unsigned int len, u8 *out)
240 {
241 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
242
243 return __chksum_finup(ctx->crc, data, len, out);
244 }
245
246 static int chksumc_finup(struct shash_desc *desc, const u8 *data,
247 unsigned int len, u8 *out)
248 {
249 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
250
251 return __chksumc_finup(ctx->crc, data, len, out);
252 }
253
254 static int chksum_digest(struct shash_desc *desc, const u8 *data,
255 unsigned int length, u8 *out)
256 {
257 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
258
259 return __chksum_finup(mctx->key, data, length, out);
260 }
261
262 static int chksumc_digest(struct shash_desc *desc, const u8 *data,
263 unsigned int length, u8 *out)
264 {
265 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
266
267 return __chksumc_finup(mctx->key, data, length, out);
268 }
269
270 static int chksum_cra_init(struct crypto_tfm *tfm)
271 {
272 struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
273
274 mctx->key = ~0;
275 return 0;
276 }
277
278 static struct shash_alg crc32_alg = {
279 .digestsize = CHKSUM_DIGEST_SIZE,
280 .setkey = chksum_setkey,
281 .init = chksum_init,
282 .update = chksum_update,
283 .final = chksum_final,
284 .finup = chksum_finup,
285 .digest = chksum_digest,
286 .descsize = sizeof(struct chksum_desc_ctx),
287 .base = {
288 .cra_name = "crc32",
289 .cra_driver_name = "crc32-mips-hw",
290 .cra_priority = 300,
291 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
292 .cra_blocksize = CHKSUM_BLOCK_SIZE,
293 .cra_alignmask = 0,
294 .cra_ctxsize = sizeof(struct chksum_ctx),
295 .cra_module = THIS_MODULE,
296 .cra_init = chksum_cra_init,
297 }
298 };
299
300 static struct shash_alg crc32c_alg = {
301 .digestsize = CHKSUM_DIGEST_SIZE,
302 .setkey = chksum_setkey,
303 .init = chksum_init,
304 .update = chksumc_update,
305 .final = chksumc_final,
306 .finup = chksumc_finup,
307 .digest = chksumc_digest,
308 .descsize = sizeof(struct chksum_desc_ctx),
309 .base = {
310 .cra_name = "crc32c",
311 .cra_driver_name = "crc32c-mips-hw",
312 .cra_priority = 300,
313 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
314 .cra_blocksize = CHKSUM_BLOCK_SIZE,
315 .cra_alignmask = 0,
316 .cra_ctxsize = sizeof(struct chksum_ctx),
317 .cra_module = THIS_MODULE,
318 .cra_init = chksum_cra_init,
319 }
320 };
321
322 static int __init crc32_mod_init(void)
323 {
324 int err;
325
326 err = crypto_register_shash(&crc32_alg);
327
328 if (err)
329 return err;
330
331 err = crypto_register_shash(&crc32c_alg);
332
333 if (err) {
334 crypto_unregister_shash(&crc32_alg);
335 return err;
336 }
337
338 return 0;
339 }
340
341 static void __exit crc32_mod_exit(void)
342 {
343 crypto_unregister_shash(&crc32_alg);
344 crypto_unregister_shash(&crc32c_alg);
345 }
346
347 MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com");
348 MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions");
349 MODULE_LICENSE("GPL v2");
350
351 module_cpu_feature_match(MIPS_CRC32, crc32_mod_init);
352 module_exit(crc32_mod_exit);
353