1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * OpenSSL/Cryptogams accelerated Poly1305 transform for MIPS 4 * 5 * Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <asm/unaligned.h> 9 #include <crypto/algapi.h> 10 #include <crypto/internal/hash.h> 11 #include <crypto/internal/poly1305.h> 12 #include <linux/cpufeature.h> 13 #include <linux/crypto.h> 14 #include <linux/module.h> 15 16 asmlinkage void poly1305_init_mips(void *state, const u8 *key); 17 asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); 18 asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); 19 20 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) 21 { 22 poly1305_init_mips(&dctx->h, key); 23 dctx->s[0] = get_unaligned_le32(key + 16); 24 dctx->s[1] = get_unaligned_le32(key + 20); 25 dctx->s[2] = get_unaligned_le32(key + 24); 26 dctx->s[3] = get_unaligned_le32(key + 28); 27 dctx->buflen = 0; 28 } 29 EXPORT_SYMBOL(poly1305_init_arch); 30 31 static int mips_poly1305_init(struct shash_desc *desc) 32 { 33 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 34 35 dctx->buflen = 0; 36 dctx->rset = 0; 37 dctx->sset = false; 38 39 return 0; 40 } 41 42 static void mips_poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, 43 u32 len, u32 hibit) 44 { 45 if (unlikely(!dctx->sset)) { 46 if (!dctx->rset) { 47 poly1305_init_mips(&dctx->h, src); 48 src += POLY1305_BLOCK_SIZE; 49 len -= POLY1305_BLOCK_SIZE; 50 dctx->rset = 1; 51 } 52 if (len >= POLY1305_BLOCK_SIZE) { 53 dctx->s[0] = get_unaligned_le32(src + 0); 54 dctx->s[1] = get_unaligned_le32(src + 4); 55 dctx->s[2] = get_unaligned_le32(src + 8); 56 dctx->s[3] = get_unaligned_le32(src + 12); 57 src += POLY1305_BLOCK_SIZE; 58 len -= POLY1305_BLOCK_SIZE; 59 dctx->sset = true; 60 } 61 if (len < POLY1305_BLOCK_SIZE) 62 return; 63 } 64 65 len &= ~(POLY1305_BLOCK_SIZE - 1); 66 67 poly1305_blocks_mips(&dctx->h, src, len, hibit); 68 } 69 70 static int mips_poly1305_update(struct shash_desc *desc, const u8 *src, 71 unsigned int len) 72 { 73 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 74 75 if (unlikely(dctx->buflen)) { 76 u32 bytes = min(len, POLY1305_BLOCK_SIZE - dctx->buflen); 77 78 memcpy(dctx->buf + dctx->buflen, src, bytes); 79 src += bytes; 80 len -= bytes; 81 dctx->buflen += bytes; 82 83 if (dctx->buflen == POLY1305_BLOCK_SIZE) { 84 mips_poly1305_blocks(dctx, dctx->buf, POLY1305_BLOCK_SIZE, 1); 85 dctx->buflen = 0; 86 } 87 } 88 89 if (likely(len >= POLY1305_BLOCK_SIZE)) { 90 mips_poly1305_blocks(dctx, src, len, 1); 91 src += round_down(len, POLY1305_BLOCK_SIZE); 92 len %= POLY1305_BLOCK_SIZE; 93 } 94 95 if (unlikely(len)) { 96 dctx->buflen = len; 97 memcpy(dctx->buf, src, len); 98 } 99 return 0; 100 } 101 102 void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, 103 unsigned int nbytes) 104 { 105 if (unlikely(dctx->buflen)) { 106 u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); 107 108 memcpy(dctx->buf + dctx->buflen, src, bytes); 109 src += bytes; 110 nbytes -= bytes; 111 dctx->buflen += bytes; 112 113 if (dctx->buflen == POLY1305_BLOCK_SIZE) { 114 poly1305_blocks_mips(&dctx->h, dctx->buf, 115 POLY1305_BLOCK_SIZE, 1); 116 dctx->buflen = 0; 117 } 118 } 119 120 if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { 121 unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); 122 123 poly1305_blocks_mips(&dctx->h, src, len, 1); 124 src += len; 125 nbytes %= POLY1305_BLOCK_SIZE; 126 } 127 128 if (unlikely(nbytes)) { 129 dctx->buflen = nbytes; 130 memcpy(dctx->buf, src, nbytes); 131 } 132 } 133 EXPORT_SYMBOL(poly1305_update_arch); 134 135 void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) 136 { 137 if (unlikely(dctx->buflen)) { 138 dctx->buf[dctx->buflen++] = 1; 139 memset(dctx->buf + dctx->buflen, 0, 140 POLY1305_BLOCK_SIZE - dctx->buflen); 141 poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); 142 } 143 144 poly1305_emit_mips(&dctx->h, dst, dctx->s); 145 *dctx = (struct poly1305_desc_ctx){}; 146 } 147 EXPORT_SYMBOL(poly1305_final_arch); 148 149 static int mips_poly1305_final(struct shash_desc *desc, u8 *dst) 150 { 151 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 152 153 if (unlikely(!dctx->sset)) 154 return -ENOKEY; 155 156 poly1305_final_arch(dctx, dst); 157 return 0; 158 } 159 160 static struct shash_alg mips_poly1305_alg = { 161 .init = mips_poly1305_init, 162 .update = mips_poly1305_update, 163 .final = mips_poly1305_final, 164 .digestsize = POLY1305_DIGEST_SIZE, 165 .descsize = sizeof(struct poly1305_desc_ctx), 166 167 .base.cra_name = "poly1305", 168 .base.cra_driver_name = "poly1305-mips", 169 .base.cra_priority = 200, 170 .base.cra_blocksize = POLY1305_BLOCK_SIZE, 171 .base.cra_module = THIS_MODULE, 172 }; 173 174 static int __init mips_poly1305_mod_init(void) 175 { 176 return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? 177 crypto_register_shash(&mips_poly1305_alg) : 0; 178 } 179 180 static void __exit mips_poly1305_mod_exit(void) 181 { 182 if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) 183 crypto_unregister_shash(&mips_poly1305_alg); 184 } 185 186 module_init(mips_poly1305_mod_init); 187 module_exit(mips_poly1305_mod_exit); 188 189 MODULE_LICENSE("GPL v2"); 190 MODULE_ALIAS_CRYPTO("poly1305"); 191 MODULE_ALIAS_CRYPTO("poly1305-mips"); 192