1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * OpenSSL/Cryptogams accelerated Poly1305 transform for MIPS 4 * 5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/unaligned.h> 9 #include <crypto/algapi.h> 10 #include <crypto/internal/hash.h> 11 #include <crypto/internal/poly1305.h> 12 #include <linux/cpufeature.h> 13 #include <linux/crypto.h> 14 #include <linux/module.h> 15 16 asmlinkage void poly1305_init_mips(void *state, const u8 *key); 17 asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); 18 asmlinkage void poly1305_emit_mips(void *state, __le32 *digest, const u32 *nonce); 19 20 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) 21 { 22 poly1305_init_mips(&dctx->h, key); 23 dctx->s[0] = get_unaligned_le32(key + 16); 24 dctx->s[1] = get_unaligned_le32(key + 20); 25 dctx->s[2] = get_unaligned_le32(key + 24); 26 dctx->s[3] = get_unaligned_le32(key + 28); 27 dctx->buflen = 0; 28 } 29 EXPORT_SYMBOL(poly1305_init_arch); 30 31 static int mips_poly1305_init(struct shash_desc *desc) 32 { 33 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 34 35 dctx->buflen = 0; 36 dctx->rset = 0; 37 dctx->sset = false; 38 39 return 0; 40 } 41 42 static void mips_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, 43 u32 len, u32 hibit) 44 { 45 if (unlikely(!dctx->sset)) { 46 if (!dctx->rset) { 47 poly1305_init_mips(&dctx->h, src); 48 src += POLY1305_BLOCK_SIZE; 49 len -= POLY1305_BLOCK_SIZE; 50 dctx->rset = 1; 51 } 52 if (len >= POLY1305_BLOCK_SIZE) { 53 dctx->s[0] = get_unaligned_le32(src + 0); 54 dctx->s[1] = get_unaligned_le32(src + 4); 55 dctx->s[2] = get_unaligned_le32(src + 8); 56 dctx->s[3] = get_unaligned_le32(src + 12); 57 src += POLY1305_BLOCK_SIZE; 58 len -= POLY1305_BLOCK_SIZE; 59 dctx->sset = true; 60 } 61 if (len < POLY1305_BLOCK_SIZE) 62 return; 63 } 64 65 len &= ~(POLY1305_BLOCK_SIZE - 1); 66 67 poly1305_blocks_mips(&dctx->h, src, len, hibit); 68 } 69 70 static int mips_poly1305_update(struct shash_desc *desc, const u8 *src, 71 unsigned int len) 72 { 73 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 74 75 if (unlikely(dctx->buflen)) { 76 u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); 77 78 memcpy(dctx->buf + dctx->buflen, src, bytes); 79 src += bytes; 80 len -= bytes; 81 dctx->buflen += bytes; 82 83 if (dctx->buflen == POLY1305_BLOCK_SIZE) { 84 mips_poly1305_blocks(dctx, dctx->buf, POLY1305_BLOCK_SIZE, 1); 85 dctx->buflen = 0; 86 } 87 } 88 89 if (likely(len >= POLY1305_BLOCK_SIZE)) { 90 mips_poly1305_blocks(dctx, src, len, 1); 91 src += round_down(len, POLY1305_BLOCK_SIZE); 92 len %= POLY1305_BLOCK_SIZE; 93 } 94 95 if (unlikely(len)) { 96 dctx->buflen = len; 97 memcpy(dctx->buf, src, len); 98 } 99 return 0; 100 } 101 102 void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, 103 unsigned int nbytes) 104 { 105 if (unlikely(dctx->buflen)) { 106 u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); 107 108 memcpy(dctx->buf + dctx->buflen, src, bytes); 109 src += bytes; 110 nbytes -= bytes; 111 dctx->buflen += bytes; 112 113 if (dctx->buflen == POLY1305_BLOCK_SIZE) { 114 poly1305_blocks_mips(&dctx->h, dctx->buf, 115 POLY1305_BLOCK_SIZE, 1); 116 dctx->buflen = 0; 117 } 118 } 119 120 if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { 121 unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); 122 123 poly1305_blocks_mips(&dctx->h, src, len, 1); 124 src += len; 125 nbytes %= POLY1305_BLOCK_SIZE; 126 } 127 128 if (unlikely(nbytes)) { 129 dctx->buflen = nbytes; 130 memcpy(dctx->buf, src, nbytes); 131 } 132 } 133 EXPORT_SYMBOL(poly1305_update_arch); 134 135 void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) 136 { 137 __le32 digest[4]; 138 u64 f = 0; 139 140 if (unlikely(dctx->buflen)) { 141 dctx->buf[dctx->buflen++] = 1; 142 memset(dctx->buf + dctx->buflen, 0, 143 POLY1305_BLOCK_SIZE - dctx->buflen); 144 poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); 145 } 146 147 poly1305_emit_mips(&dctx->h, digest, dctx->s); 148 149 /* mac = (h + s) % (2^128) */ 150 f = (f >> 32) + le32_to_cpu(digest[0]); 151 put_unaligned_le32(f, dst); 152 f = (f >> 32) + le32_to_cpu(digest[1]); 153 put_unaligned_le32(f, dst + 4); 154 f = (f >> 32) + le32_to_cpu(digest[2]); 155 put_unaligned_le32(f, dst + 8); 156 f = (f >> 32) + le32_to_cpu(digest[3]); 157 put_unaligned_le32(f, dst + 12); 158 159 *dctx = (struct poly1305_desc_ctx){}; 160 } 161 EXPORT_SYMBOL(poly1305_final_arch); 162 163 static int mips_poly1305_final(struct shash_desc *desc, u8 *dst) 164 { 165 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 166 167 if (unlikely(!dctx->sset)) 168 return -ENOKEY; 169 170 poly1305_final_arch(dctx, dst); 171 return 0; 172 } 173 174 static struct shash_alg mips_poly1305_alg = { 175 .init = mips_poly1305_init, 176 .update = mips_poly1305_update, 177 .final = mips_poly1305_final, 178 .digestsize = POLY1305_DIGEST_SIZE, 179 .descsize = sizeof(struct poly1305_desc_ctx), 180 181 .base.cra_name = "poly1305", 182 .base.cra_driver_name = "poly1305-mips", 183 .base.cra_priority = 200, 184 .base.cra_blocksize = POLY1305_BLOCK_SIZE, 185 .base.cra_module = THIS_MODULE, 186 }; 187 188 static int __init mips_poly1305_mod_init(void) 189 { 190 return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? 191 crypto_register_shash(&mips_poly1305_alg) : 0; 192 } 193 194 static void __exit mips_poly1305_mod_exit(void) 195 { 196 if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) 197 crypto_unregister_shash(&mips_poly1305_alg); 198 } 199 200 module_init(mips_poly1305_mod_init); 201 module_exit(mips_poly1305_mod_exit); 202 203 MODULE_LICENSE("GPL v2"); 204 MODULE_ALIAS_CRYPTO("poly1305"); 205 MODULE_ALIAS_CRYPTO("poly1305-mips"); 206