1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Poly1305 authenticator algorithm, RFC7539. 4 * 5 * Copyright 2023- IBM Corp. All rights reserved. 6 */ 7 8 #include <crypto/algapi.h> 9 #include <linux/crypto.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/jump_label.h> 13 #include <crypto/internal/hash.h> 14 #include <crypto/internal/poly1305.h> 15 #include <crypto/internal/simd.h> 16 #include <linux/cpufeature.h> 17 #include <asm/unaligned.h> 18 #include <asm/simd.h> 19 #include <asm/switch_to.h> 20 21 asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); 22 asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); 23 asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); 24 25 static void vsx_begin(void) 26 { 27 preempt_disable(); 28 enable_kernel_vsx(); 29 } 30 31 static void vsx_end(void) 32 { 33 disable_kernel_vsx(); 34 preempt_enable(); 35 } 36 37 static int crypto_poly1305_p10_init(struct shash_desc *desc) 38 { 39 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 40 41 poly1305_core_init(&dctx->h); 42 dctx->buflen = 0; 43 dctx->rset = 0; 44 dctx->sset = false; 45 46 return 0; 47 } 48 49 static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, 50 const u8 *inp, unsigned int len) 51 { 52 unsigned int acc = 0; 53 54 if (unlikely(!dctx->sset)) { 55 if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { 56 struct poly1305_core_key *key = &dctx->core_r; 57 58 key->key.r64[0] = get_unaligned_le64(&inp[0]); 59 key->key.r64[1] = get_unaligned_le64(&inp[8]); 60 inp += POLY1305_BLOCK_SIZE; 61 len -= POLY1305_BLOCK_SIZE; 62 acc += POLY1305_BLOCK_SIZE; 63 dctx->rset = 1; 64 } 65 if (len >= POLY1305_BLOCK_SIZE) { 66 dctx->s[0] = get_unaligned_le32(&inp[0]); 67 dctx->s[1] = get_unaligned_le32(&inp[4]); 68 dctx->s[2] = get_unaligned_le32(&inp[8]); 69 dctx->s[3] = get_unaligned_le32(&inp[12]); 70 acc += POLY1305_BLOCK_SIZE; 71 dctx->sset = true; 72 } 73 } 74 return acc; 75 } 76 77 static int crypto_poly1305_p10_update(struct shash_desc *desc, 78 const u8 *src, unsigned int srclen) 79 { 80 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 81 unsigned int bytes, used; 82 83 if (unlikely(dctx->buflen)) { 84 bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); 85 memcpy(dctx->buf + dctx->buflen, src, bytes); 86 src += bytes; 87 srclen -= bytes; 88 dctx->buflen += bytes; 89 90 if (dctx->buflen == POLY1305_BLOCK_SIZE) { 91 if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, 92 POLY1305_BLOCK_SIZE))) { 93 vsx_begin(); 94 poly1305_64s(&dctx->h, dctx->buf, 95 POLY1305_BLOCK_SIZE, 1); 96 vsx_end(); 97 } 98 dctx->buflen = 0; 99 } 100 } 101 102 if (likely(srclen >= POLY1305_BLOCK_SIZE)) { 103 bytes = round_down(srclen, POLY1305_BLOCK_SIZE); 104 used = crypto_poly1305_setdctxkey(dctx, src, bytes); 105 if (likely(used)) { 106 srclen -= used; 107 src += used; 108 } 109 if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { 110 vsx_begin(); 111 poly1305_p10le_4blocks(&dctx->h, src, srclen); 112 vsx_end(); 113 src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4)); 114 srclen %= POLY1305_BLOCK_SIZE * 4; 115 } 116 while (srclen >= POLY1305_BLOCK_SIZE) { 117 vsx_begin(); 118 poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1); 119 vsx_end(); 120 srclen -= POLY1305_BLOCK_SIZE; 121 src += POLY1305_BLOCK_SIZE; 122 } 123 } 124 125 if (unlikely(srclen)) { 126 dctx->buflen = srclen; 127 memcpy(dctx->buf, src, srclen); 128 } 129 130 return 0; 131 } 132 133 static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst) 134 { 135 struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); 136 137 if (unlikely(!dctx->sset)) 138 return -ENOKEY; 139 140 if ((dctx->buflen)) { 141 dctx->buf[dctx->buflen++] = 1; 142 memset(dctx->buf + dctx->buflen, 0, 143 POLY1305_BLOCK_SIZE - dctx->buflen); 144 vsx_begin(); 145 poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); 146 vsx_end(); 147 dctx->buflen = 0; 148 } 149 150 poly1305_emit_64(&dctx->h, &dctx->s, dst); 151 return 0; 152 } 153 154 static struct shash_alg poly1305_alg = { 155 .digestsize = POLY1305_DIGEST_SIZE, 156 .init = crypto_poly1305_p10_init, 157 .update = crypto_poly1305_p10_update, 158 .final = crypto_poly1305_p10_final, 159 .descsize = sizeof(struct poly1305_desc_ctx), 160 .base = { 161 .cra_name = "poly1305", 162 .cra_driver_name = "poly1305-p10", 163 .cra_priority = 300, 164 .cra_blocksize = POLY1305_BLOCK_SIZE, 165 .cra_module = THIS_MODULE, 166 }, 167 }; 168 169 static int __init poly1305_p10_init(void) 170 { 171 return crypto_register_shash(&poly1305_alg); 172 } 173 174 static void __exit poly1305_p10_exit(void) 175 { 176 crypto_unregister_shash(&poly1305_alg); 177 } 178 179 module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init); 180 module_exit(poly1305_p10_exit); 181 182 MODULE_LICENSE("GPL"); 183 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); 184 MODULE_DESCRIPTION("Optimized Poly1305 for P10"); 185 MODULE_ALIAS_CRYPTO("poly1305"); 186 MODULE_ALIAS_CRYPTO("poly1305-p10"); 187