1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 4 * 5 * This is based in part on Andrew Moon's poly1305-donna, which is in the 6 * public domain. 7 */ 8 9 #include <linux/kernel.h> 10 #include <asm/unaligned.h> 11 #include <crypto/internal/poly1305.h> 12 13 typedef __uint128_t u128; 14 15 void poly1305_core_setkey(struct poly1305_core_key *key, 16 const u8 raw_key[POLY1305_BLOCK_SIZE]) 17 { 18 u64 t0, t1; 19 20 /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ 21 t0 = get_unaligned_le64(&raw_key[0]); 22 t1 = get_unaligned_le64(&raw_key[8]); 23 24 key->key.r64[0] = t0 & 0xffc0fffffffULL; 25 key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; 26 key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; 27 28 /* s = 20*r */ 29 key->precomputed_s.r64[0] = key->key.r64[1] * 20; 30 key->precomputed_s.r64[1] = key->key.r64[2] * 20; 31 } 32 EXPORT_SYMBOL(poly1305_core_setkey); 33 34 void poly1305_core_blocks(struct poly1305_state *state, 35 const struct poly1305_core_key *key, const void *src, 36 unsigned int nblocks, u32 hibit) 37 { 38 const u8 *input = src; 39 u64 hibit64; 40 u64 r0, r1, r2; 41 u64 s1, s2; 42 u64 h0, h1, h2; 43 u64 c; 44 u128 d0, d1, d2, d; 45 46 if (!nblocks) 47 return; 48 49 hibit64 = ((u64)hibit) << 40; 50 51 r0 = key->key.r64[0]; 52 r1 = key->key.r64[1]; 53 r2 = key->key.r64[2]; 54 55 h0 = state->h64[0]; 56 h1 = state->h64[1]; 57 h2 = state->h64[2]; 58 59 s1 = key->precomputed_s.r64[0]; 60 s2 = key->precomputed_s.r64[1]; 61 62 do { 63 u64 t0, t1; 64 65 /* h += m[i] */ 66 t0 = get_unaligned_le64(&input[0]); 67 t1 = get_unaligned_le64(&input[8]); 68 69 h0 += t0 & 0xfffffffffffULL; 70 h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; 71 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; 72 73 /* h *= r */ 74 d0 = (u128)h0 * r0; 75 d = (u128)h1 * s2; 76 d0 += d; 77 d = (u128)h2 * s1; 78 d0 += d; 79 d1 = (u128)h0 * r1; 80 d = (u128)h1 * r0; 81 d1 += d; 82 d = (u128)h2 * s2; 83 d1 += d; 84 d2 = (u128)h0 * r2; 85 d = (u128)h1 * r1; 86 d2 += d; 87 d = (u128)h2 * r0; 88 d2 += d; 89 90 /* (partial) h %= p */ 91 c = (u64)(d0 >> 44); 92 h0 = (u64)d0 & 0xfffffffffffULL; 93 d1 += c; 94 c = (u64)(d1 >> 44); 95 h1 = (u64)d1 & 0xfffffffffffULL; 96 d2 += c; 97 c = (u64)(d2 >> 42); 98 h2 = (u64)d2 & 0x3ffffffffffULL; 99 h0 += c * 5; 100 c = h0 >> 44; 101 h0 = h0 & 0xfffffffffffULL; 102 h1 += c; 103 104 input += POLY1305_BLOCK_SIZE; 105 } while (--nblocks); 106 107 state->h64[0] = h0; 108 state->h64[1] = h1; 109 state->h64[2] = h2; 110 } 111 EXPORT_SYMBOL(poly1305_core_blocks); 112 113 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], 114 void *dst) 115 { 116 u8 *mac = dst; 117 u64 h0, h1, h2, c; 118 u64 g0, g1, g2; 119 u64 t0, t1; 120 121 /* fully carry h */ 122 h0 = state->h64[0]; 123 h1 = state->h64[1]; 124 h2 = state->h64[2]; 125 126 c = h1 >> 44; 127 h1 &= 0xfffffffffffULL; 128 h2 += c; 129 c = h2 >> 42; 130 h2 &= 0x3ffffffffffULL; 131 h0 += c * 5; 132 c = h0 >> 44; 133 h0 &= 0xfffffffffffULL; 134 h1 += c; 135 c = h1 >> 44; 136 h1 &= 0xfffffffffffULL; 137 h2 += c; 138 c = h2 >> 42; 139 h2 &= 0x3ffffffffffULL; 140 h0 += c * 5; 141 c = h0 >> 44; 142 h0 &= 0xfffffffffffULL; 143 h1 += c; 144 145 /* compute h + -p */ 146 g0 = h0 + 5; 147 c = g0 >> 44; 148 g0 &= 0xfffffffffffULL; 149 g1 = h1 + c; 150 c = g1 >> 44; 151 g1 &= 0xfffffffffffULL; 152 g2 = h2 + c - (1ULL << 42); 153 154 /* select h if h < p, or h + -p if h >= p */ 155 c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; 156 g0 &= c; 157 g1 &= c; 158 g2 &= c; 159 c = ~c; 160 h0 = (h0 & c) | g0; 161 h1 = (h1 & c) | g1; 162 h2 = (h2 & c) | g2; 163 164 if (likely(nonce)) { 165 /* h = (h + nonce) */ 166 t0 = ((u64)nonce[1] << 32) | nonce[0]; 167 t1 = ((u64)nonce[3] << 32) | nonce[2]; 168 169 h0 += t0 & 0xfffffffffffULL; 170 c = h0 >> 44; 171 h0 &= 0xfffffffffffULL; 172 h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; 173 c = h1 >> 44; 174 h1 &= 0xfffffffffffULL; 175 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; 176 h2 &= 0x3ffffffffffULL; 177 } 178 179 /* mac = h % (2^128) */ 180 h0 = h0 | (h1 << 44); 181 h1 = (h1 >> 20) | (h2 << 24); 182 183 put_unaligned_le64(h0, &mac[0]); 184 put_unaligned_le64(h1, &mac[8]); 185 } 186 EXPORT_SYMBOL(poly1305_core_emit); 187