1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/*
  3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4 *
  5 * This is based in part on Andrew Moon's poly1305-donna, which is in the
  6 * public domain.
  7 */
  8
  9#include <linux/kernel.h>
 10#include <asm/unaligned.h>
 11#include <crypto/internal/poly1305.h>
 12
 13typedef __uint128_t u128;
 14
 15void poly1305_core_setkey(struct poly1305_core_key *key,
 16			  const u8 raw_key[POLY1305_BLOCK_SIZE])
 17{
 18	u64 t0, t1;
 19
 20	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
 21	t0 = get_unaligned_le64(&raw_key[0]);
 22	t1 = get_unaligned_le64(&raw_key[8]);
 23
 24	key->key.r64[0] = t0 & 0xffc0fffffffULL;
 25	key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
 26	key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
 27
 28	/* s = 20*r */
 29	key->precomputed_s.r64[0] = key->key.r64[1] * 20;
 30	key->precomputed_s.r64[1] = key->key.r64[2] * 20;
 31}
 32EXPORT_SYMBOL(poly1305_core_setkey);
 33
 34void poly1305_core_blocks(struct poly1305_state *state,
 35			  const struct poly1305_core_key *key, const void *src,
 36			  unsigned int nblocks, u32 hibit)
 37{
 38	const u8 *input = src;
 39	u64 hibit64;
 40	u64 r0, r1, r2;
 41	u64 s1, s2;
 42	u64 h0, h1, h2;
 43	u64 c;
 44	u128 d0, d1, d2, d;
 45
 46	if (!nblocks)
 47		return;
 48
 49	hibit64 = ((u64)hibit) << 40;
 50
 51	r0 = key->key.r64[0];
 52	r1 = key->key.r64[1];
 53	r2 = key->key.r64[2];
 54
 55	h0 = state->h64[0];
 56	h1 = state->h64[1];
 57	h2 = state->h64[2];
 58
 59	s1 = key->precomputed_s.r64[0];
 60	s2 = key->precomputed_s.r64[1];
 61
 62	do {
 63		u64 t0, t1;
 64
 65		/* h += m[i] */
 66		t0 = get_unaligned_le64(&input[0]);
 67		t1 = get_unaligned_le64(&input[8]);
 68
 69		h0 += t0 & 0xfffffffffffULL;
 70		h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
 71		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
 72
 73		/* h *= r */
 74		d0 = (u128)h0 * r0;
 75		d = (u128)h1 * s2;
 76		d0 += d;
 77		d = (u128)h2 * s1;
 78		d0 += d;
 79		d1 = (u128)h0 * r1;
 80		d = (u128)h1 * r0;
 81		d1 += d;
 82		d = (u128)h2 * s2;
 83		d1 += d;
 84		d2 = (u128)h0 * r2;
 85		d = (u128)h1 * r1;
 86		d2 += d;
 87		d = (u128)h2 * r0;
 88		d2 += d;
 89
 90		/* (partial) h %= p */
 91		c = (u64)(d0 >> 44);
 92		h0 = (u64)d0 & 0xfffffffffffULL;
 93		d1 += c;
 94		c = (u64)(d1 >> 44);
 95		h1 = (u64)d1 & 0xfffffffffffULL;
 96		d2 += c;
 97		c = (u64)(d2 >> 42);
 98		h2 = (u64)d2 & 0x3ffffffffffULL;
 99		h0 += c * 5;
100		c = h0 >> 44;
101		h0 = h0 & 0xfffffffffffULL;
102		h1 += c;
103
104		input += POLY1305_BLOCK_SIZE;
105	} while (--nblocks);
106
107	state->h64[0] = h0;
108	state->h64[1] = h1;
109	state->h64[2] = h2;
110}
111EXPORT_SYMBOL(poly1305_core_blocks);
112
113void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
114			void *dst)
115{
116	u8 *mac = dst;
117	u64 h0, h1, h2, c;
118	u64 g0, g1, g2;
119	u64 t0, t1;
120
121	/* fully carry h */
122	h0 = state->h64[0];
123	h1 = state->h64[1];
124	h2 = state->h64[2];
125
126	c = h1 >> 44;
127	h1 &= 0xfffffffffffULL;
128	h2 += c;
129	c = h2 >> 42;
130	h2 &= 0x3ffffffffffULL;
131	h0 += c * 5;
132	c = h0 >> 44;
133	h0 &= 0xfffffffffffULL;
134	h1 += c;
135	c = h1 >> 44;
136	h1 &= 0xfffffffffffULL;
137	h2 += c;
138	c = h2 >> 42;
139	h2 &= 0x3ffffffffffULL;
140	h0 += c * 5;
141	c = h0 >> 44;
142	h0 &= 0xfffffffffffULL;
143	h1 += c;
144
145	/* compute h + -p */
146	g0 = h0 + 5;
147	c  = g0 >> 44;
148	g0 &= 0xfffffffffffULL;
149	g1 = h1 + c;
150	c  = g1 >> 44;
151	g1 &= 0xfffffffffffULL;
152	g2 = h2 + c - (1ULL << 42);
153
154	/* select h if h < p, or h + -p if h >= p */
155	c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
156	g0 &= c;
157	g1 &= c;
158	g2 &= c;
159	c  = ~c;
160	h0 = (h0 & c) | g0;
161	h1 = (h1 & c) | g1;
162	h2 = (h2 & c) | g2;
163
164	if (likely(nonce)) {
165		/* h = (h + nonce) */
166		t0 = ((u64)nonce[1] << 32) | nonce[0];
167		t1 = ((u64)nonce[3] << 32) | nonce[2];
168
169		h0 += t0 & 0xfffffffffffULL;
170		c = h0 >> 44;
171		h0 &= 0xfffffffffffULL;
172		h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
173		c = h1 >> 44;
174		h1 &= 0xfffffffffffULL;
175		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
176		h2 &= 0x3ffffffffffULL;
177	}
178
179	/* mac = h % (2^128) */
180	h0 = h0 | (h1 << 44);
181	h1 = (h1 >> 20) | (h2 << 24);
182
183	put_unaligned_le64(h0, &mac[0]);
184	put_unaligned_le64(h1, &mac[8]);
185}
186EXPORT_SYMBOL(poly1305_core_emit);