1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is based in part on Andrew Moon's poly1305-donna, which is in the 6 * public domain. 7 */ 8 9 #include <linux/kernel.h> 10 #include <asm/unaligned.h> 11 #include <crypto/internal/poly1305.h> 12 13 void poly1305_core_setkey(struct poly1305_core_key *key, 14 const u8 raw_key[POLY1305_BLOCK_SIZE]) 15 { 16 u64 t0, t1; 17 18 /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ 19 t0 = get_unaligned_le64(&raw_key[0]); 20 t1 = get_unaligned_le64(&raw_key[8]); 21 22 key->key.r64[0] = t0 & 0xffc0fffffffULL; 23 key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; 24 key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; 25 26 /* s = 20*r */ 27 key->precomputed_s.r64[0] = key->key.r64[1] * 20; 28 key->precomputed_s.r64[1] = key->key.r64[2] * 20; 29 } 30 EXPORT_SYMBOL(poly1305_core_setkey); 31 32 void poly1305_core_blocks(struct poly1305_state *state, 33 const struct poly1305_core_key *key, const void *src, 34 unsigned int nblocks, u32 hibit) 35 { 36 const u8 *input = src; 37 u64 hibit64; 38 u64 r0, r1, r2; 39 u64 s1, s2; 40 u64 h0, h1, h2; 41 u64 c; 42 u128 d0, d1, d2, d; 43 44 if (!nblocks) 45 return; 46 47 hibit64 = ((u64)hibit) << 40; 48 49 r0 = key->key.r64[0]; 50 r1 = key->key.r64[1]; 51 r2 = key->key.r64[2]; 52 53 h0 = state->h64[0]; 54 h1 = state->h64[1]; 55 h2 = state->h64[2]; 56 57 s1 = key->precomputed_s.r64[0]; 58 s2 = key->precomputed_s.r64[1]; 59 60 do { 61 u64 t0, t1; 62 63 /* h += m[i] */ 64 t0 = get_unaligned_le64(&input[0]); 65 t1 = get_unaligned_le64(&input[8]); 66 67 h0 += t0 & 0xfffffffffffULL; 68 h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; 69 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; 70 71 /* h *= r */ 72 d0 = (u128)h0 * r0; 73 d = (u128)h1 * s2; 74 d0 += d; 75 d = (u128)h2 * s1; 76 d0 += d; 77 d1 = (u128)h0 * r1; 78 d = (u128)h1 * r0; 79 d1 += d; 80 d = (u128)h2 * s2; 81 d1 += d; 82 d2 = (u128)h0 * r2; 83 d = (u128)h1 * r1; 84 d2 += d; 85 d = (u128)h2 * r0; 86 d2 += d; 87 88 /* (partial) h %= p */ 89 c = (u64)(d0 >> 44); 90 h0 = (u64)d0 & 0xfffffffffffULL; 91 d1 += c; 92 c = (u64)(d1 >> 44); 93 h1 = (u64)d1 & 0xfffffffffffULL; 94 d2 += c; 95 c = (u64)(d2 >> 42); 96 h2 = (u64)d2 & 0x3ffffffffffULL; 97 h0 += c * 5; 98 c = h0 >> 44; 99 h0 = h0 & 0xfffffffffffULL; 100 h1 += c; 101 102 input += POLY1305_BLOCK_SIZE; 103 } while (--nblocks); 104 105 state->h64[0] = h0; 106 state->h64[1] = h1; 107 state->h64[2] = h2; 108 } 109 EXPORT_SYMBOL(poly1305_core_blocks); 110 111 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], 112 void *dst) 113 { 114 u8 *mac = dst; 115 u64 h0, h1, h2, c; 116 u64 g0, g1, g2; 117 u64 t0, t1; 118 119 /* fully carry h */ 120 h0 = state->h64[0]; 121 h1 = state->h64[1]; 122 h2 = state->h64[2]; 123 124 c = h1 >> 44; 125 h1 &= 0xfffffffffffULL; 126 h2 += c; 127 c = h2 >> 42; 128 h2 &= 0x3ffffffffffULL; 129 h0 += c * 5; 130 c = h0 >> 44; 131 h0 &= 0xfffffffffffULL; 132 h1 += c; 133 c = h1 >> 44; 134 h1 &= 0xfffffffffffULL; 135 h2 += c; 136 c = h2 >> 42; 137 h2 &= 0x3ffffffffffULL; 138 h0 += c * 5; 139 c = h0 >> 44; 140 h0 &= 0xfffffffffffULL; 141 h1 += c; 142 143 /* compute h + -p */ 144 g0 = h0 + 5; 145 c = g0 >> 44; 146 g0 &= 0xfffffffffffULL; 147 g1 = h1 + c; 148 c = g1 >> 44; 149 g1 &= 0xfffffffffffULL; 150 g2 = h2 + c - (1ULL << 42); 151 152 /* select h if h < p, or h + -p if h >= p */ 153 c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; 154 g0 &= c; 155 g1 &= c; 156 g2 &= c; 157 c = ~c; 158 h0 = (h0 & c) | g0; 159 h1 = (h1 & c) | g1; 160 h2 = (h2 & c) | g2; 161 162 if (likely(nonce)) { 163 /* h = (h + nonce) */ 164 t0 = ((u64)nonce[1] << 32) | nonce[0]; 165 t1 = ((u64)nonce[3] << 32) | nonce[2]; 166 167 h0 += t0 & 0xfffffffffffULL; 168 c = h0 >> 44; 169 h0 &= 0xfffffffffffULL; 170 h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; 171 c = h1 >> 44; 172 h1 &= 0xfffffffffffULL; 173 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; 174 h2 &= 0x3ffffffffffULL; 175 } 176 177 /* mac = h % (2^128) */ 178 h0 = h0 | (h1 << 44); 179 h1 = (h1 >> 20) | (h2 << 24); 180 181 put_unaligned_le64(h0, &mac[0]); 182 put_unaligned_le64(h1, &mac[8]); 183 } 184 EXPORT_SYMBOL(poly1305_core_emit); 185