Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4 *
5 * This is an implementation of the ChaCha20Poly1305 AEAD construction.
6 *
7 * Information: https://tools.ietf.org/html/rfc8439
8 */
9
10#include <crypto/algapi.h>
11#include <crypto/chacha20poly1305.h>
12#include <crypto/chacha.h>
13#include <crypto/poly1305.h>
14#include <crypto/scatterwalk.h>
15
16#include <asm/unaligned.h>
17#include <linux/kernel.h>
18#include <linux/init.h>
19#include <linux/mm.h>
20#include <linux/module.h>
21
22#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
23
24static void chacha_load_key(u32 *k, const u8 *in)
25{
26 k[0] = get_unaligned_le32(in);
27 k[1] = get_unaligned_le32(in + 4);
28 k[2] = get_unaligned_le32(in + 8);
29 k[3] = get_unaligned_le32(in + 12);
30 k[4] = get_unaligned_le32(in + 16);
31 k[5] = get_unaligned_le32(in + 20);
32 k[6] = get_unaligned_le32(in + 24);
33 k[7] = get_unaligned_le32(in + 28);
34}
35
36static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
37{
38 u32 k[CHACHA_KEY_WORDS];
39 u8 iv[CHACHA_IV_SIZE];
40
41 memset(iv, 0, 8);
42 memcpy(iv + 8, nonce + 16, 8);
43
44 chacha_load_key(k, key);
45
46 /* Compute the subkey given the original key and first 128 nonce bits */
47 chacha_init(chacha_state, k, nonce);
48 hchacha_block(chacha_state, k, 20);
49
50 chacha_init(chacha_state, k, iv);
51
52 memzero_explicit(k, sizeof(k));
53 memzero_explicit(iv, sizeof(iv));
54}
55
56static void
57__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
58 const u8 *ad, const size_t ad_len, u32 *chacha_state)
59{
60 const u8 *pad0 = page_address(ZERO_PAGE(0));
61 struct poly1305_desc_ctx poly1305_state;
62 union {
63 u8 block0[POLY1305_KEY_SIZE];
64 __le64 lens[2];
65 } b;
66
67 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
68 poly1305_init(&poly1305_state, b.block0);
69
70 poly1305_update(&poly1305_state, ad, ad_len);
71 if (ad_len & 0xf)
72 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
73
74 chacha20_crypt(chacha_state, dst, src, src_len);
75
76 poly1305_update(&poly1305_state, dst, src_len);
77 if (src_len & 0xf)
78 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
79
80 b.lens[0] = cpu_to_le64(ad_len);
81 b.lens[1] = cpu_to_le64(src_len);
82 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
83
84 poly1305_final(&poly1305_state, dst + src_len);
85
86 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
87 memzero_explicit(&b, sizeof(b));
88}
89
90void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
91 const u8 *ad, const size_t ad_len,
92 const u64 nonce,
93 const u8 key[CHACHA20POLY1305_KEY_SIZE])
94{
95 u32 chacha_state[CHACHA_STATE_WORDS];
96 u32 k[CHACHA_KEY_WORDS];
97 __le64 iv[2];
98
99 chacha_load_key(k, key);
100
101 iv[0] = 0;
102 iv[1] = cpu_to_le64(nonce);
103
104 chacha_init(chacha_state, k, (u8 *)iv);
105 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
106
107 memzero_explicit(iv, sizeof(iv));
108 memzero_explicit(k, sizeof(k));
109}
110EXPORT_SYMBOL(chacha20poly1305_encrypt);
111
112void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
113 const u8 *ad, const size_t ad_len,
114 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
115 const u8 key[CHACHA20POLY1305_KEY_SIZE])
116{
117 u32 chacha_state[CHACHA_STATE_WORDS];
118
119 xchacha_init(chacha_state, key, nonce);
120 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
121}
122EXPORT_SYMBOL(xchacha20poly1305_encrypt);
123
124static bool
125__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
126 const u8 *ad, const size_t ad_len, u32 *chacha_state)
127{
128 const u8 *pad0 = page_address(ZERO_PAGE(0));
129 struct poly1305_desc_ctx poly1305_state;
130 size_t dst_len;
131 int ret;
132 union {
133 u8 block0[POLY1305_KEY_SIZE];
134 u8 mac[POLY1305_DIGEST_SIZE];
135 __le64 lens[2];
136 } b;
137
138 if (unlikely(src_len < POLY1305_DIGEST_SIZE))
139 return false;
140
141 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
142 poly1305_init(&poly1305_state, b.block0);
143
144 poly1305_update(&poly1305_state, ad, ad_len);
145 if (ad_len & 0xf)
146 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
147
148 dst_len = src_len - POLY1305_DIGEST_SIZE;
149 poly1305_update(&poly1305_state, src, dst_len);
150 if (dst_len & 0xf)
151 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
152
153 b.lens[0] = cpu_to_le64(ad_len);
154 b.lens[1] = cpu_to_le64(dst_len);
155 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
156
157 poly1305_final(&poly1305_state, b.mac);
158
159 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
160 if (likely(!ret))
161 chacha20_crypt(chacha_state, dst, src, dst_len);
162
163 memzero_explicit(&b, sizeof(b));
164
165 return !ret;
166}
167
168bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
169 const u8 *ad, const size_t ad_len,
170 const u64 nonce,
171 const u8 key[CHACHA20POLY1305_KEY_SIZE])
172{
173 u32 chacha_state[CHACHA_STATE_WORDS];
174 u32 k[CHACHA_KEY_WORDS];
175 __le64 iv[2];
176 bool ret;
177
178 chacha_load_key(k, key);
179
180 iv[0] = 0;
181 iv[1] = cpu_to_le64(nonce);
182
183 chacha_init(chacha_state, k, (u8 *)iv);
184 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
185 chacha_state);
186
187 memzero_explicit(chacha_state, sizeof(chacha_state));
188 memzero_explicit(iv, sizeof(iv));
189 memzero_explicit(k, sizeof(k));
190 return ret;
191}
192EXPORT_SYMBOL(chacha20poly1305_decrypt);
193
194bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
195 const u8 *ad, const size_t ad_len,
196 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
197 const u8 key[CHACHA20POLY1305_KEY_SIZE])
198{
199 u32 chacha_state[CHACHA_STATE_WORDS];
200
201 xchacha_init(chacha_state, key, nonce);
202 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
203 chacha_state);
204}
205EXPORT_SYMBOL(xchacha20poly1305_decrypt);
206
207static
208bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
209 const size_t src_len,
210 const u8 *ad, const size_t ad_len,
211 const u64 nonce,
212 const u8 key[CHACHA20POLY1305_KEY_SIZE],
213 int encrypt)
214{
215 const u8 *pad0 = page_address(ZERO_PAGE(0));
216 struct poly1305_desc_ctx poly1305_state;
217 u32 chacha_state[CHACHA_STATE_WORDS];
218 struct sg_mapping_iter miter;
219 size_t partial = 0;
220 unsigned int flags;
221 bool ret = true;
222 int sl;
223 union {
224 struct {
225 u32 k[CHACHA_KEY_WORDS];
226 __le64 iv[2];
227 };
228 u8 block0[POLY1305_KEY_SIZE];
229 u8 chacha_stream[CHACHA_BLOCK_SIZE];
230 struct {
231 u8 mac[2][POLY1305_DIGEST_SIZE];
232 };
233 __le64 lens[2];
234 } b __aligned(16);
235
236 if (WARN_ON(src_len > INT_MAX))
237 return false;
238
239 chacha_load_key(b.k, key);
240
241 b.iv[0] = 0;
242 b.iv[1] = cpu_to_le64(nonce);
243
244 chacha_init(chacha_state, b.k, (u8 *)b.iv);
245 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
246 poly1305_init(&poly1305_state, b.block0);
247
248 if (unlikely(ad_len)) {
249 poly1305_update(&poly1305_state, ad, ad_len);
250 if (ad_len & 0xf)
251 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
252 }
253
254 flags = SG_MITER_TO_SG;
255 if (!preemptible())
256 flags |= SG_MITER_ATOMIC;
257
258 sg_miter_start(&miter, src, sg_nents(src), flags);
259
260 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
261 u8 *addr = miter.addr;
262 size_t length = min_t(size_t, sl, miter.length);
263
264 if (!encrypt)
265 poly1305_update(&poly1305_state, addr, length);
266
267 if (unlikely(partial)) {
268 size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
269
270 crypto_xor(addr, b.chacha_stream + partial, l);
271 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
272
273 addr += l;
274 length -= l;
275 }
276
277 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
278 size_t l = length;
279
280 if (unlikely(length < sl))
281 l &= ~(CHACHA_BLOCK_SIZE - 1);
282 chacha20_crypt(chacha_state, addr, addr, l);
283 addr += l;
284 length -= l;
285 }
286
287 if (unlikely(length > 0)) {
288 chacha20_crypt(chacha_state, b.chacha_stream, pad0,
289 CHACHA_BLOCK_SIZE);
290 crypto_xor(addr, b.chacha_stream, length);
291 partial = length;
292 }
293
294 if (encrypt)
295 poly1305_update(&poly1305_state, miter.addr,
296 min_t(size_t, sl, miter.length));
297 }
298
299 if (src_len & 0xf)
300 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
301
302 b.lens[0] = cpu_to_le64(ad_len);
303 b.lens[1] = cpu_to_le64(src_len);
304 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
305
306 if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
307 if (encrypt) {
308 poly1305_final(&poly1305_state,
309 miter.addr + miter.length + sl);
310 ret = true;
311 } else {
312 poly1305_final(&poly1305_state, b.mac[0]);
313 ret = !crypto_memneq(b.mac[0],
314 miter.addr + miter.length + sl,
315 POLY1305_DIGEST_SIZE);
316 }
317 }
318
319 sg_miter_stop(&miter);
320
321 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
322 poly1305_final(&poly1305_state, b.mac[1]);
323 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
324 sizeof(b.mac[1]), encrypt);
325 ret = encrypt ||
326 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
327 }
328
329 memzero_explicit(chacha_state, sizeof(chacha_state));
330 memzero_explicit(&b, sizeof(b));
331
332 return ret;
333}
334
335bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
336 const u8 *ad, const size_t ad_len,
337 const u64 nonce,
338 const u8 key[CHACHA20POLY1305_KEY_SIZE])
339{
340 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
341 nonce, key, 1);
342}
343EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
344
345bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
346 const u8 *ad, const size_t ad_len,
347 const u64 nonce,
348 const u8 key[CHACHA20POLY1305_KEY_SIZE])
349{
350 if (unlikely(src_len < POLY1305_DIGEST_SIZE))
351 return false;
352
353 return chacha20poly1305_crypt_sg_inplace(src,
354 src_len - POLY1305_DIGEST_SIZE,
355 ad, ad_len, nonce, key, 0);
356}
357EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
358
359static int __init mod_init(void)
360{
361 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
362 WARN_ON(!chacha20poly1305_selftest()))
363 return -ENODEV;
364 return 0;
365}
366
367module_init(mod_init);
368MODULE_LICENSE("GPL v2");
369MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
370MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");