Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * AES-NI + SSE4.1 implementation of AEGIS-128
4 *
5 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
6 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
7 * Copyright 2024 Google LLC
8 */
9
10#include <linux/linkage.h>
11
12#define STATE0 %xmm0
13#define STATE1 %xmm1
14#define STATE2 %xmm2
15#define STATE3 %xmm3
16#define STATE4 %xmm4
17#define KEY %xmm5
18#define MSG %xmm5
19#define T0 %xmm6
20#define T1 %xmm7
21
22.section .rodata.cst16.aegis128_const, "aM", @progbits, 32
23.align 16
24.Laegis128_const_0:
25 .byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
26 .byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
27.Laegis128_const_1:
28 .byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
29 .byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
30
31.section .rodata.cst32.zeropad_mask, "aM", @progbits, 32
32.align 32
33.Lzeropad_mask:
34 .octa 0xffffffffffffffffffffffffffffffff
35 .octa 0
36
37.text
38
39/*
40 * aegis128_update
41 * input:
42 * STATE[0-4] - input state
43 * output:
44 * STATE[0-4] - output state (shifted positions)
45 * changed:
46 * T0
47 */
48.macro aegis128_update
49 movdqa STATE4, T0
50 aesenc STATE0, STATE4
51 aesenc STATE1, STATE0
52 aesenc STATE2, STATE1
53 aesenc STATE3, STATE2
54 aesenc T0, STATE3
55.endm
56
57/*
58 * Load 1 <= LEN (%ecx) <= 15 bytes from the pointer SRC into the xmm register
59 * MSG and zeroize any remaining bytes. Clobbers %rax, %rcx, and %r8.
60 */
61.macro load_partial
62 sub $8, %ecx /* LEN - 8 */
63 jle .Lle8\@
64
65 /* Load 9 <= LEN <= 15 bytes: */
66 movq (SRC), MSG /* Load first 8 bytes */
67 mov (SRC, %rcx), %rax /* Load last 8 bytes */
68 neg %ecx
69 shl $3, %ecx
70 shr %cl, %rax /* Discard overlapping bytes */
71 pinsrq $1, %rax, MSG
72 jmp .Ldone\@
73
74.Lle8\@:
75 add $4, %ecx /* LEN - 4 */
76 jl .Llt4\@
77
78 /* Load 4 <= LEN <= 8 bytes: */
79 mov (SRC), %eax /* Load first 4 bytes */
80 mov (SRC, %rcx), %r8d /* Load last 4 bytes */
81 jmp .Lcombine\@
82
83.Llt4\@:
84 /* Load 1 <= LEN <= 3 bytes: */
85 add $2, %ecx /* LEN - 2 */
86 movzbl (SRC), %eax /* Load first byte */
87 jl .Lmovq\@
88 movzwl (SRC, %rcx), %r8d /* Load last 2 bytes */
89.Lcombine\@:
90 shl $3, %ecx
91 shl %cl, %r8
92 or %r8, %rax /* Combine the two parts */
93.Lmovq\@:
94 movq %rax, MSG
95.Ldone\@:
96.endm
97
98/*
99 * Store 1 <= LEN (%ecx) <= 15 bytes from the xmm register \msg to the pointer
100 * DST. Clobbers %rax, %rcx, and %r8.
101 */
102.macro store_partial msg
103 sub $8, %ecx /* LEN - 8 */
104 jl .Llt8\@
105
106 /* Store 8 <= LEN <= 15 bytes: */
107 pextrq $1, \msg, %rax
108 mov %ecx, %r8d
109 shl $3, %ecx
110 ror %cl, %rax
111 mov %rax, (DST, %r8) /* Store last LEN - 8 bytes */
112 movq \msg, (DST) /* Store first 8 bytes */
113 jmp .Ldone\@
114
115.Llt8\@:
116 add $4, %ecx /* LEN - 4 */
117 jl .Llt4\@
118
119 /* Store 4 <= LEN <= 7 bytes: */
120 pextrd $1, \msg, %eax
121 mov %ecx, %r8d
122 shl $3, %ecx
123 ror %cl, %eax
124 mov %eax, (DST, %r8) /* Store last LEN - 4 bytes */
125 movd \msg, (DST) /* Store first 4 bytes */
126 jmp .Ldone\@
127
128.Llt4\@:
129 /* Store 1 <= LEN <= 3 bytes: */
130 pextrb $0, \msg, 0(DST)
131 cmp $-2, %ecx /* LEN - 4 == -2, i.e. LEN == 2? */
132 jl .Ldone\@
133 pextrb $1, \msg, 1(DST)
134 je .Ldone\@
135 pextrb $2, \msg, 2(DST)
136.Ldone\@:
137.endm
138
139/*
140 * void aegis128_aesni_init(struct aegis_state *state,
141 * const struct aegis_block *key,
142 * const u8 iv[AEGIS128_NONCE_SIZE]);
143 */
144SYM_FUNC_START(aegis128_aesni_init)
145 .set STATEP, %rdi
146 .set KEYP, %rsi
147 .set IVP, %rdx
148
149 /* load IV: */
150 movdqu (IVP), T1
151
152 /* load key: */
153 movdqa (KEYP), KEY
154 pxor KEY, T1
155 movdqa T1, STATE0
156 movdqa KEY, STATE3
157 movdqa KEY, STATE4
158
159 /* load the constants: */
160 movdqa .Laegis128_const_0(%rip), STATE2
161 movdqa .Laegis128_const_1(%rip), STATE1
162 pxor STATE2, STATE3
163 pxor STATE1, STATE4
164
165 /* update 10 times with KEY / KEY xor IV: */
166 aegis128_update; pxor KEY, STATE4
167 aegis128_update; pxor T1, STATE3
168 aegis128_update; pxor KEY, STATE2
169 aegis128_update; pxor T1, STATE1
170 aegis128_update; pxor KEY, STATE0
171 aegis128_update; pxor T1, STATE4
172 aegis128_update; pxor KEY, STATE3
173 aegis128_update; pxor T1, STATE2
174 aegis128_update; pxor KEY, STATE1
175 aegis128_update; pxor T1, STATE0
176
177 /* store the state: */
178 movdqu STATE0, 0x00(STATEP)
179 movdqu STATE1, 0x10(STATEP)
180 movdqu STATE2, 0x20(STATEP)
181 movdqu STATE3, 0x30(STATEP)
182 movdqu STATE4, 0x40(STATEP)
183 RET
184SYM_FUNC_END(aegis128_aesni_init)
185
186/*
187 * void aegis128_aesni_ad(struct aegis_state *state, const u8 *data,
188 * unsigned int len);
189 *
190 * len must be a multiple of 16.
191 */
192SYM_FUNC_START(aegis128_aesni_ad)
193 .set STATEP, %rdi
194 .set SRC, %rsi
195 .set LEN, %edx
196
197 test LEN, LEN
198 jz .Lad_out
199
200 /* load the state: */
201 movdqu 0x00(STATEP), STATE0
202 movdqu 0x10(STATEP), STATE1
203 movdqu 0x20(STATEP), STATE2
204 movdqu 0x30(STATEP), STATE3
205 movdqu 0x40(STATEP), STATE4
206
207.align 8
208.Lad_loop:
209 movdqu 0x00(SRC), MSG
210 aegis128_update
211 pxor MSG, STATE4
212 sub $0x10, LEN
213 jz .Lad_out_1
214
215 movdqu 0x10(SRC), MSG
216 aegis128_update
217 pxor MSG, STATE3
218 sub $0x10, LEN
219 jz .Lad_out_2
220
221 movdqu 0x20(SRC), MSG
222 aegis128_update
223 pxor MSG, STATE2
224 sub $0x10, LEN
225 jz .Lad_out_3
226
227 movdqu 0x30(SRC), MSG
228 aegis128_update
229 pxor MSG, STATE1
230 sub $0x10, LEN
231 jz .Lad_out_4
232
233 movdqu 0x40(SRC), MSG
234 aegis128_update
235 pxor MSG, STATE0
236 sub $0x10, LEN
237 jz .Lad_out_0
238
239 add $0x50, SRC
240 jmp .Lad_loop
241
242 /* store the state: */
243.Lad_out_0:
244 movdqu STATE0, 0x00(STATEP)
245 movdqu STATE1, 0x10(STATEP)
246 movdqu STATE2, 0x20(STATEP)
247 movdqu STATE3, 0x30(STATEP)
248 movdqu STATE4, 0x40(STATEP)
249 RET
250
251.Lad_out_1:
252 movdqu STATE4, 0x00(STATEP)
253 movdqu STATE0, 0x10(STATEP)
254 movdqu STATE1, 0x20(STATEP)
255 movdqu STATE2, 0x30(STATEP)
256 movdqu STATE3, 0x40(STATEP)
257 RET
258
259.Lad_out_2:
260 movdqu STATE3, 0x00(STATEP)
261 movdqu STATE4, 0x10(STATEP)
262 movdqu STATE0, 0x20(STATEP)
263 movdqu STATE1, 0x30(STATEP)
264 movdqu STATE2, 0x40(STATEP)
265 RET
266
267.Lad_out_3:
268 movdqu STATE2, 0x00(STATEP)
269 movdqu STATE3, 0x10(STATEP)
270 movdqu STATE4, 0x20(STATEP)
271 movdqu STATE0, 0x30(STATEP)
272 movdqu STATE1, 0x40(STATEP)
273 RET
274
275.Lad_out_4:
276 movdqu STATE1, 0x00(STATEP)
277 movdqu STATE2, 0x10(STATEP)
278 movdqu STATE3, 0x20(STATEP)
279 movdqu STATE4, 0x30(STATEP)
280 movdqu STATE0, 0x40(STATEP)
281.Lad_out:
282 RET
283SYM_FUNC_END(aegis128_aesni_ad)
284
285.macro encrypt_block s0 s1 s2 s3 s4 i
286 movdqu (\i * 0x10)(SRC), MSG
287 movdqa MSG, T0
288 pxor \s1, T0
289 pxor \s4, T0
290 movdqa \s2, T1
291 pand \s3, T1
292 pxor T1, T0
293 movdqu T0, (\i * 0x10)(DST)
294
295 aegis128_update
296 pxor MSG, \s4
297
298 sub $0x10, LEN
299 jz .Lenc_out_\i
300.endm
301
302/*
303 * void aegis128_aesni_enc(struct aegis_state *state, const u8 *src, u8 *dst,
304 * unsigned int len);
305 *
306 * len must be nonzero and a multiple of 16.
307 */
308SYM_FUNC_START(aegis128_aesni_enc)
309 .set STATEP, %rdi
310 .set SRC, %rsi
311 .set DST, %rdx
312 .set LEN, %ecx
313
314 /* load the state: */
315 movdqu 0x00(STATEP), STATE0
316 movdqu 0x10(STATEP), STATE1
317 movdqu 0x20(STATEP), STATE2
318 movdqu 0x30(STATEP), STATE3
319 movdqu 0x40(STATEP), STATE4
320
321.align 8
322.Lenc_loop:
323 encrypt_block STATE0 STATE1 STATE2 STATE3 STATE4 0
324 encrypt_block STATE4 STATE0 STATE1 STATE2 STATE3 1
325 encrypt_block STATE3 STATE4 STATE0 STATE1 STATE2 2
326 encrypt_block STATE2 STATE3 STATE4 STATE0 STATE1 3
327 encrypt_block STATE1 STATE2 STATE3 STATE4 STATE0 4
328
329 add $0x50, SRC
330 add $0x50, DST
331 jmp .Lenc_loop
332
333 /* store the state: */
334.Lenc_out_0:
335 movdqu STATE4, 0x00(STATEP)
336 movdqu STATE0, 0x10(STATEP)
337 movdqu STATE1, 0x20(STATEP)
338 movdqu STATE2, 0x30(STATEP)
339 movdqu STATE3, 0x40(STATEP)
340 RET
341
342.Lenc_out_1:
343 movdqu STATE3, 0x00(STATEP)
344 movdqu STATE4, 0x10(STATEP)
345 movdqu STATE0, 0x20(STATEP)
346 movdqu STATE1, 0x30(STATEP)
347 movdqu STATE2, 0x40(STATEP)
348 RET
349
350.Lenc_out_2:
351 movdqu STATE2, 0x00(STATEP)
352 movdqu STATE3, 0x10(STATEP)
353 movdqu STATE4, 0x20(STATEP)
354 movdqu STATE0, 0x30(STATEP)
355 movdqu STATE1, 0x40(STATEP)
356 RET
357
358.Lenc_out_3:
359 movdqu STATE1, 0x00(STATEP)
360 movdqu STATE2, 0x10(STATEP)
361 movdqu STATE3, 0x20(STATEP)
362 movdqu STATE4, 0x30(STATEP)
363 movdqu STATE0, 0x40(STATEP)
364 RET
365
366.Lenc_out_4:
367 movdqu STATE0, 0x00(STATEP)
368 movdqu STATE1, 0x10(STATEP)
369 movdqu STATE2, 0x20(STATEP)
370 movdqu STATE3, 0x30(STATEP)
371 movdqu STATE4, 0x40(STATEP)
372.Lenc_out:
373 RET
374SYM_FUNC_END(aegis128_aesni_enc)
375
376/*
377 * void aegis128_aesni_enc_tail(struct aegis_state *state, const u8 *src,
378 * u8 *dst, unsigned int len);
379 */
380SYM_FUNC_START(aegis128_aesni_enc_tail)
381 .set STATEP, %rdi
382 .set SRC, %rsi
383 .set DST, %rdx
384 .set LEN, %ecx /* {load,store}_partial rely on this being %ecx */
385
386 /* load the state: */
387 movdqu 0x00(STATEP), STATE0
388 movdqu 0x10(STATEP), STATE1
389 movdqu 0x20(STATEP), STATE2
390 movdqu 0x30(STATEP), STATE3
391 movdqu 0x40(STATEP), STATE4
392
393 /* encrypt message: */
394 mov LEN, %r9d
395 load_partial
396
397 movdqa MSG, T0
398 pxor STATE1, T0
399 pxor STATE4, T0
400 movdqa STATE2, T1
401 pand STATE3, T1
402 pxor T1, T0
403
404 mov %r9d, LEN
405 store_partial T0
406
407 aegis128_update
408 pxor MSG, STATE4
409
410 /* store the state: */
411 movdqu STATE4, 0x00(STATEP)
412 movdqu STATE0, 0x10(STATEP)
413 movdqu STATE1, 0x20(STATEP)
414 movdqu STATE2, 0x30(STATEP)
415 movdqu STATE3, 0x40(STATEP)
416 RET
417SYM_FUNC_END(aegis128_aesni_enc_tail)
418
419.macro decrypt_block s0 s1 s2 s3 s4 i
420 movdqu (\i * 0x10)(SRC), MSG
421 pxor \s1, MSG
422 pxor \s4, MSG
423 movdqa \s2, T1
424 pand \s3, T1
425 pxor T1, MSG
426 movdqu MSG, (\i * 0x10)(DST)
427
428 aegis128_update
429 pxor MSG, \s4
430
431 sub $0x10, LEN
432 jz .Ldec_out_\i
433.endm
434
435/*
436 * void aegis128_aesni_dec(struct aegis_state *state, const u8 *src, u8 *dst,
437 * unsigned int len);
438 *
439 * len must be nonzero and a multiple of 16.
440 */
441SYM_FUNC_START(aegis128_aesni_dec)
442 .set STATEP, %rdi
443 .set SRC, %rsi
444 .set DST, %rdx
445 .set LEN, %ecx
446
447 /* load the state: */
448 movdqu 0x00(STATEP), STATE0
449 movdqu 0x10(STATEP), STATE1
450 movdqu 0x20(STATEP), STATE2
451 movdqu 0x30(STATEP), STATE3
452 movdqu 0x40(STATEP), STATE4
453
454.align 8
455.Ldec_loop:
456 decrypt_block STATE0 STATE1 STATE2 STATE3 STATE4 0
457 decrypt_block STATE4 STATE0 STATE1 STATE2 STATE3 1
458 decrypt_block STATE3 STATE4 STATE0 STATE1 STATE2 2
459 decrypt_block STATE2 STATE3 STATE4 STATE0 STATE1 3
460 decrypt_block STATE1 STATE2 STATE3 STATE4 STATE0 4
461
462 add $0x50, SRC
463 add $0x50, DST
464 jmp .Ldec_loop
465
466 /* store the state: */
467.Ldec_out_0:
468 movdqu STATE4, 0x00(STATEP)
469 movdqu STATE0, 0x10(STATEP)
470 movdqu STATE1, 0x20(STATEP)
471 movdqu STATE2, 0x30(STATEP)
472 movdqu STATE3, 0x40(STATEP)
473 RET
474
475.Ldec_out_1:
476 movdqu STATE3, 0x00(STATEP)
477 movdqu STATE4, 0x10(STATEP)
478 movdqu STATE0, 0x20(STATEP)
479 movdqu STATE1, 0x30(STATEP)
480 movdqu STATE2, 0x40(STATEP)
481 RET
482
483.Ldec_out_2:
484 movdqu STATE2, 0x00(STATEP)
485 movdqu STATE3, 0x10(STATEP)
486 movdqu STATE4, 0x20(STATEP)
487 movdqu STATE0, 0x30(STATEP)
488 movdqu STATE1, 0x40(STATEP)
489 RET
490
491.Ldec_out_3:
492 movdqu STATE1, 0x00(STATEP)
493 movdqu STATE2, 0x10(STATEP)
494 movdqu STATE3, 0x20(STATEP)
495 movdqu STATE4, 0x30(STATEP)
496 movdqu STATE0, 0x40(STATEP)
497 RET
498
499.Ldec_out_4:
500 movdqu STATE0, 0x00(STATEP)
501 movdqu STATE1, 0x10(STATEP)
502 movdqu STATE2, 0x20(STATEP)
503 movdqu STATE3, 0x30(STATEP)
504 movdqu STATE4, 0x40(STATEP)
505.Ldec_out:
506 RET
507SYM_FUNC_END(aegis128_aesni_dec)
508
509/*
510 * void aegis128_aesni_dec_tail(struct aegis_state *state, const u8 *src,
511 * u8 *dst, unsigned int len);
512 */
513SYM_FUNC_START(aegis128_aesni_dec_tail)
514 .set STATEP, %rdi
515 .set SRC, %rsi
516 .set DST, %rdx
517 .set LEN, %ecx /* {load,store}_partial rely on this being %ecx */
518
519 /* load the state: */
520 movdqu 0x00(STATEP), STATE0
521 movdqu 0x10(STATEP), STATE1
522 movdqu 0x20(STATEP), STATE2
523 movdqu 0x30(STATEP), STATE3
524 movdqu 0x40(STATEP), STATE4
525
526 /* decrypt message: */
527 mov LEN, %r9d
528 load_partial
529
530 pxor STATE1, MSG
531 pxor STATE4, MSG
532 movdqa STATE2, T1
533 pand STATE3, T1
534 pxor T1, MSG
535
536 mov %r9d, LEN
537 store_partial MSG
538
539 /* mask with byte count: */
540 lea .Lzeropad_mask+16(%rip), %rax
541 sub %r9, %rax
542 movdqu (%rax), T0
543 pand T0, MSG
544
545 aegis128_update
546 pxor MSG, STATE4
547
548 /* store the state: */
549 movdqu STATE4, 0x00(STATEP)
550 movdqu STATE0, 0x10(STATEP)
551 movdqu STATE1, 0x20(STATEP)
552 movdqu STATE2, 0x30(STATEP)
553 movdqu STATE3, 0x40(STATEP)
554 RET
555SYM_FUNC_END(aegis128_aesni_dec_tail)
556
557/*
558 * void aegis128_aesni_final(struct aegis_state *state,
559 * struct aegis_block *tag_xor,
560 * unsigned int assoclen, unsigned int cryptlen);
561 */
562SYM_FUNC_START(aegis128_aesni_final)
563 .set STATEP, %rdi
564 .set TAG_XOR, %rsi
565 .set ASSOCLEN, %edx
566 .set CRYPTLEN, %ecx
567
568 /* load the state: */
569 movdqu 0x00(STATEP), STATE0
570 movdqu 0x10(STATEP), STATE1
571 movdqu 0x20(STATEP), STATE2
572 movdqu 0x30(STATEP), STATE3
573 movdqu 0x40(STATEP), STATE4
574
575 /* prepare length block: */
576 movd ASSOCLEN, MSG
577 pinsrd $2, CRYPTLEN, MSG
578 psllq $3, MSG /* multiply by 8 (to get bit count) */
579
580 pxor STATE3, MSG
581
582 /* update state: */
583 aegis128_update; pxor MSG, STATE4
584 aegis128_update; pxor MSG, STATE3
585 aegis128_update; pxor MSG, STATE2
586 aegis128_update; pxor MSG, STATE1
587 aegis128_update; pxor MSG, STATE0
588 aegis128_update; pxor MSG, STATE4
589 aegis128_update; pxor MSG, STATE3
590
591 /* xor tag: */
592 movdqu (TAG_XOR), MSG
593
594 pxor STATE0, MSG
595 pxor STATE1, MSG
596 pxor STATE2, MSG
597 pxor STATE3, MSG
598 pxor STATE4, MSG
599
600 movdqu MSG, (TAG_XOR)
601 RET
602SYM_FUNC_END(aegis128_aesni_final)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * AES-NI + SSE2 implementation of AEGIS-128
4 *
5 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
6 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
7 */
8
9#include <linux/linkage.h>
10#include <asm/frame.h>
11
12#define STATE0 %xmm0
13#define STATE1 %xmm1
14#define STATE2 %xmm2
15#define STATE3 %xmm3
16#define STATE4 %xmm4
17#define KEY %xmm5
18#define MSG %xmm5
19#define T0 %xmm6
20#define T1 %xmm7
21
22#define STATEP %rdi
23#define LEN %rsi
24#define SRC %rdx
25#define DST %rcx
26
27.section .rodata.cst16.aegis128_const, "aM", @progbits, 32
28.align 16
29.Laegis128_const_0:
30 .byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
31 .byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
32.Laegis128_const_1:
33 .byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
34 .byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
35
36.section .rodata.cst16.aegis128_counter, "aM", @progbits, 16
37.align 16
38.Laegis128_counter:
39 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
40 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
41
42.text
43
44/*
45 * aegis128_update
46 * input:
47 * STATE[0-4] - input state
48 * output:
49 * STATE[0-4] - output state (shifted positions)
50 * changed:
51 * T0
52 */
53.macro aegis128_update
54 movdqa STATE4, T0
55 aesenc STATE0, STATE4
56 aesenc STATE1, STATE0
57 aesenc STATE2, STATE1
58 aesenc STATE3, STATE2
59 aesenc T0, STATE3
60.endm
61
62/*
63 * __load_partial: internal ABI
64 * input:
65 * LEN - bytes
66 * SRC - src
67 * output:
68 * MSG - message block
69 * changed:
70 * T0
71 * %r8
72 * %r9
73 */
74SYM_FUNC_START_LOCAL(__load_partial)
75 xor %r9d, %r9d
76 pxor MSG, MSG
77
78 mov LEN, %r8
79 and $0x1, %r8
80 jz .Lld_partial_1
81
82 mov LEN, %r8
83 and $0x1E, %r8
84 add SRC, %r8
85 mov (%r8), %r9b
86
87.Lld_partial_1:
88 mov LEN, %r8
89 and $0x2, %r8
90 jz .Lld_partial_2
91
92 mov LEN, %r8
93 and $0x1C, %r8
94 add SRC, %r8
95 shl $0x10, %r9
96 mov (%r8), %r9w
97
98.Lld_partial_2:
99 mov LEN, %r8
100 and $0x4, %r8
101 jz .Lld_partial_4
102
103 mov LEN, %r8
104 and $0x18, %r8
105 add SRC, %r8
106 shl $32, %r9
107 mov (%r8), %r8d
108 xor %r8, %r9
109
110.Lld_partial_4:
111 movq %r9, MSG
112
113 mov LEN, %r8
114 and $0x8, %r8
115 jz .Lld_partial_8
116
117 mov LEN, %r8
118 and $0x10, %r8
119 add SRC, %r8
120 pslldq $8, MSG
121 movq (%r8), T0
122 pxor T0, MSG
123
124.Lld_partial_8:
125 ret
126SYM_FUNC_END(__load_partial)
127
128/*
129 * __store_partial: internal ABI
130 * input:
131 * LEN - bytes
132 * DST - dst
133 * output:
134 * T0 - message block
135 * changed:
136 * %r8
137 * %r9
138 * %r10
139 */
140SYM_FUNC_START_LOCAL(__store_partial)
141 mov LEN, %r8
142 mov DST, %r9
143
144 movq T0, %r10
145
146 cmp $8, %r8
147 jl .Lst_partial_8
148
149 mov %r10, (%r9)
150 psrldq $8, T0
151 movq T0, %r10
152
153 sub $8, %r8
154 add $8, %r9
155
156.Lst_partial_8:
157 cmp $4, %r8
158 jl .Lst_partial_4
159
160 mov %r10d, (%r9)
161 shr $32, %r10
162
163 sub $4, %r8
164 add $4, %r9
165
166.Lst_partial_4:
167 cmp $2, %r8
168 jl .Lst_partial_2
169
170 mov %r10w, (%r9)
171 shr $0x10, %r10
172
173 sub $2, %r8
174 add $2, %r9
175
176.Lst_partial_2:
177 cmp $1, %r8
178 jl .Lst_partial_1
179
180 mov %r10b, (%r9)
181
182.Lst_partial_1:
183 ret
184SYM_FUNC_END(__store_partial)
185
186/*
187 * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv);
188 */
189SYM_FUNC_START(crypto_aegis128_aesni_init)
190 FRAME_BEGIN
191
192 /* load IV: */
193 movdqu (%rdx), T1
194
195 /* load key: */
196 movdqa (%rsi), KEY
197 pxor KEY, T1
198 movdqa T1, STATE0
199 movdqa KEY, STATE3
200 movdqa KEY, STATE4
201
202 /* load the constants: */
203 movdqa .Laegis128_const_0, STATE2
204 movdqa .Laegis128_const_1, STATE1
205 pxor STATE2, STATE3
206 pxor STATE1, STATE4
207
208 /* update 10 times with KEY / KEY xor IV: */
209 aegis128_update; pxor KEY, STATE4
210 aegis128_update; pxor T1, STATE3
211 aegis128_update; pxor KEY, STATE2
212 aegis128_update; pxor T1, STATE1
213 aegis128_update; pxor KEY, STATE0
214 aegis128_update; pxor T1, STATE4
215 aegis128_update; pxor KEY, STATE3
216 aegis128_update; pxor T1, STATE2
217 aegis128_update; pxor KEY, STATE1
218 aegis128_update; pxor T1, STATE0
219
220 /* store the state: */
221 movdqu STATE0, 0x00(STATEP)
222 movdqu STATE1, 0x10(STATEP)
223 movdqu STATE2, 0x20(STATEP)
224 movdqu STATE3, 0x30(STATEP)
225 movdqu STATE4, 0x40(STATEP)
226
227 FRAME_END
228 ret
229SYM_FUNC_END(crypto_aegis128_aesni_init)
230
231/*
232 * void crypto_aegis128_aesni_ad(void *state, unsigned int length,
233 * const void *data);
234 */
235SYM_FUNC_START(crypto_aegis128_aesni_ad)
236 FRAME_BEGIN
237
238 cmp $0x10, LEN
239 jb .Lad_out
240
241 /* load the state: */
242 movdqu 0x00(STATEP), STATE0
243 movdqu 0x10(STATEP), STATE1
244 movdqu 0x20(STATEP), STATE2
245 movdqu 0x30(STATEP), STATE3
246 movdqu 0x40(STATEP), STATE4
247
248 mov SRC, %r8
249 and $0xF, %r8
250 jnz .Lad_u_loop
251
252.align 8
253.Lad_a_loop:
254 movdqa 0x00(SRC), MSG
255 aegis128_update
256 pxor MSG, STATE4
257 sub $0x10, LEN
258 cmp $0x10, LEN
259 jl .Lad_out_1
260
261 movdqa 0x10(SRC), MSG
262 aegis128_update
263 pxor MSG, STATE3
264 sub $0x10, LEN
265 cmp $0x10, LEN
266 jl .Lad_out_2
267
268 movdqa 0x20(SRC), MSG
269 aegis128_update
270 pxor MSG, STATE2
271 sub $0x10, LEN
272 cmp $0x10, LEN
273 jl .Lad_out_3
274
275 movdqa 0x30(SRC), MSG
276 aegis128_update
277 pxor MSG, STATE1
278 sub $0x10, LEN
279 cmp $0x10, LEN
280 jl .Lad_out_4
281
282 movdqa 0x40(SRC), MSG
283 aegis128_update
284 pxor MSG, STATE0
285 sub $0x10, LEN
286 cmp $0x10, LEN
287 jl .Lad_out_0
288
289 add $0x50, SRC
290 jmp .Lad_a_loop
291
292.align 8
293.Lad_u_loop:
294 movdqu 0x00(SRC), MSG
295 aegis128_update
296 pxor MSG, STATE4
297 sub $0x10, LEN
298 cmp $0x10, LEN
299 jl .Lad_out_1
300
301 movdqu 0x10(SRC), MSG
302 aegis128_update
303 pxor MSG, STATE3
304 sub $0x10, LEN
305 cmp $0x10, LEN
306 jl .Lad_out_2
307
308 movdqu 0x20(SRC), MSG
309 aegis128_update
310 pxor MSG, STATE2
311 sub $0x10, LEN
312 cmp $0x10, LEN
313 jl .Lad_out_3
314
315 movdqu 0x30(SRC), MSG
316 aegis128_update
317 pxor MSG, STATE1
318 sub $0x10, LEN
319 cmp $0x10, LEN
320 jl .Lad_out_4
321
322 movdqu 0x40(SRC), MSG
323 aegis128_update
324 pxor MSG, STATE0
325 sub $0x10, LEN
326 cmp $0x10, LEN
327 jl .Lad_out_0
328
329 add $0x50, SRC
330 jmp .Lad_u_loop
331
332 /* store the state: */
333.Lad_out_0:
334 movdqu STATE0, 0x00(STATEP)
335 movdqu STATE1, 0x10(STATEP)
336 movdqu STATE2, 0x20(STATEP)
337 movdqu STATE3, 0x30(STATEP)
338 movdqu STATE4, 0x40(STATEP)
339 FRAME_END
340 ret
341
342.Lad_out_1:
343 movdqu STATE4, 0x00(STATEP)
344 movdqu STATE0, 0x10(STATEP)
345 movdqu STATE1, 0x20(STATEP)
346 movdqu STATE2, 0x30(STATEP)
347 movdqu STATE3, 0x40(STATEP)
348 FRAME_END
349 ret
350
351.Lad_out_2:
352 movdqu STATE3, 0x00(STATEP)
353 movdqu STATE4, 0x10(STATEP)
354 movdqu STATE0, 0x20(STATEP)
355 movdqu STATE1, 0x30(STATEP)
356 movdqu STATE2, 0x40(STATEP)
357 FRAME_END
358 ret
359
360.Lad_out_3:
361 movdqu STATE2, 0x00(STATEP)
362 movdqu STATE3, 0x10(STATEP)
363 movdqu STATE4, 0x20(STATEP)
364 movdqu STATE0, 0x30(STATEP)
365 movdqu STATE1, 0x40(STATEP)
366 FRAME_END
367 ret
368
369.Lad_out_4:
370 movdqu STATE1, 0x00(STATEP)
371 movdqu STATE2, 0x10(STATEP)
372 movdqu STATE3, 0x20(STATEP)
373 movdqu STATE4, 0x30(STATEP)
374 movdqu STATE0, 0x40(STATEP)
375 FRAME_END
376 ret
377
378.Lad_out:
379 FRAME_END
380 ret
381SYM_FUNC_END(crypto_aegis128_aesni_ad)
382
383.macro encrypt_block a s0 s1 s2 s3 s4 i
384 movdq\a (\i * 0x10)(SRC), MSG
385 movdqa MSG, T0
386 pxor \s1, T0
387 pxor \s4, T0
388 movdqa \s2, T1
389 pand \s3, T1
390 pxor T1, T0
391 movdq\a T0, (\i * 0x10)(DST)
392
393 aegis128_update
394 pxor MSG, \s4
395
396 sub $0x10, LEN
397 cmp $0x10, LEN
398 jl .Lenc_out_\i
399.endm
400
401/*
402 * void crypto_aegis128_aesni_enc(void *state, unsigned int length,
403 * const void *src, void *dst);
404 */
405SYM_FUNC_START(crypto_aegis128_aesni_enc)
406 FRAME_BEGIN
407
408 cmp $0x10, LEN
409 jb .Lenc_out
410
411 /* load the state: */
412 movdqu 0x00(STATEP), STATE0
413 movdqu 0x10(STATEP), STATE1
414 movdqu 0x20(STATEP), STATE2
415 movdqu 0x30(STATEP), STATE3
416 movdqu 0x40(STATEP), STATE4
417
418 mov SRC, %r8
419 or DST, %r8
420 and $0xF, %r8
421 jnz .Lenc_u_loop
422
423.align 8
424.Lenc_a_loop:
425 encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
426 encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
427 encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
428 encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
429 encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
430
431 add $0x50, SRC
432 add $0x50, DST
433 jmp .Lenc_a_loop
434
435.align 8
436.Lenc_u_loop:
437 encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
438 encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
439 encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
440 encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
441 encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
442
443 add $0x50, SRC
444 add $0x50, DST
445 jmp .Lenc_u_loop
446
447 /* store the state: */
448.Lenc_out_0:
449 movdqu STATE4, 0x00(STATEP)
450 movdqu STATE0, 0x10(STATEP)
451 movdqu STATE1, 0x20(STATEP)
452 movdqu STATE2, 0x30(STATEP)
453 movdqu STATE3, 0x40(STATEP)
454 FRAME_END
455 ret
456
457.Lenc_out_1:
458 movdqu STATE3, 0x00(STATEP)
459 movdqu STATE4, 0x10(STATEP)
460 movdqu STATE0, 0x20(STATEP)
461 movdqu STATE1, 0x30(STATEP)
462 movdqu STATE2, 0x40(STATEP)
463 FRAME_END
464 ret
465
466.Lenc_out_2:
467 movdqu STATE2, 0x00(STATEP)
468 movdqu STATE3, 0x10(STATEP)
469 movdqu STATE4, 0x20(STATEP)
470 movdqu STATE0, 0x30(STATEP)
471 movdqu STATE1, 0x40(STATEP)
472 FRAME_END
473 ret
474
475.Lenc_out_3:
476 movdqu STATE1, 0x00(STATEP)
477 movdqu STATE2, 0x10(STATEP)
478 movdqu STATE3, 0x20(STATEP)
479 movdqu STATE4, 0x30(STATEP)
480 movdqu STATE0, 0x40(STATEP)
481 FRAME_END
482 ret
483
484.Lenc_out_4:
485 movdqu STATE0, 0x00(STATEP)
486 movdqu STATE1, 0x10(STATEP)
487 movdqu STATE2, 0x20(STATEP)
488 movdqu STATE3, 0x30(STATEP)
489 movdqu STATE4, 0x40(STATEP)
490 FRAME_END
491 ret
492
493.Lenc_out:
494 FRAME_END
495 ret
496SYM_FUNC_END(crypto_aegis128_aesni_enc)
497
498/*
499 * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
500 * const void *src, void *dst);
501 */
502SYM_FUNC_START(crypto_aegis128_aesni_enc_tail)
503 FRAME_BEGIN
504
505 /* load the state: */
506 movdqu 0x00(STATEP), STATE0
507 movdqu 0x10(STATEP), STATE1
508 movdqu 0x20(STATEP), STATE2
509 movdqu 0x30(STATEP), STATE3
510 movdqu 0x40(STATEP), STATE4
511
512 /* encrypt message: */
513 call __load_partial
514
515 movdqa MSG, T0
516 pxor STATE1, T0
517 pxor STATE4, T0
518 movdqa STATE2, T1
519 pand STATE3, T1
520 pxor T1, T0
521
522 call __store_partial
523
524 aegis128_update
525 pxor MSG, STATE4
526
527 /* store the state: */
528 movdqu STATE4, 0x00(STATEP)
529 movdqu STATE0, 0x10(STATEP)
530 movdqu STATE1, 0x20(STATEP)
531 movdqu STATE2, 0x30(STATEP)
532 movdqu STATE3, 0x40(STATEP)
533
534 FRAME_END
535 ret
536SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
537
538.macro decrypt_block a s0 s1 s2 s3 s4 i
539 movdq\a (\i * 0x10)(SRC), MSG
540 pxor \s1, MSG
541 pxor \s4, MSG
542 movdqa \s2, T1
543 pand \s3, T1
544 pxor T1, MSG
545 movdq\a MSG, (\i * 0x10)(DST)
546
547 aegis128_update
548 pxor MSG, \s4
549
550 sub $0x10, LEN
551 cmp $0x10, LEN
552 jl .Ldec_out_\i
553.endm
554
555/*
556 * void crypto_aegis128_aesni_dec(void *state, unsigned int length,
557 * const void *src, void *dst);
558 */
559SYM_FUNC_START(crypto_aegis128_aesni_dec)
560 FRAME_BEGIN
561
562 cmp $0x10, LEN
563 jb .Ldec_out
564
565 /* load the state: */
566 movdqu 0x00(STATEP), STATE0
567 movdqu 0x10(STATEP), STATE1
568 movdqu 0x20(STATEP), STATE2
569 movdqu 0x30(STATEP), STATE3
570 movdqu 0x40(STATEP), STATE4
571
572 mov SRC, %r8
573 or DST, %r8
574 and $0xF, %r8
575 jnz .Ldec_u_loop
576
577.align 8
578.Ldec_a_loop:
579 decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
580 decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
581 decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
582 decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
583 decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
584
585 add $0x50, SRC
586 add $0x50, DST
587 jmp .Ldec_a_loop
588
589.align 8
590.Ldec_u_loop:
591 decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
592 decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
593 decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
594 decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
595 decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
596
597 add $0x50, SRC
598 add $0x50, DST
599 jmp .Ldec_u_loop
600
601 /* store the state: */
602.Ldec_out_0:
603 movdqu STATE4, 0x00(STATEP)
604 movdqu STATE0, 0x10(STATEP)
605 movdqu STATE1, 0x20(STATEP)
606 movdqu STATE2, 0x30(STATEP)
607 movdqu STATE3, 0x40(STATEP)
608 FRAME_END
609 ret
610
611.Ldec_out_1:
612 movdqu STATE3, 0x00(STATEP)
613 movdqu STATE4, 0x10(STATEP)
614 movdqu STATE0, 0x20(STATEP)
615 movdqu STATE1, 0x30(STATEP)
616 movdqu STATE2, 0x40(STATEP)
617 FRAME_END
618 ret
619
620.Ldec_out_2:
621 movdqu STATE2, 0x00(STATEP)
622 movdqu STATE3, 0x10(STATEP)
623 movdqu STATE4, 0x20(STATEP)
624 movdqu STATE0, 0x30(STATEP)
625 movdqu STATE1, 0x40(STATEP)
626 FRAME_END
627 ret
628
629.Ldec_out_3:
630 movdqu STATE1, 0x00(STATEP)
631 movdqu STATE2, 0x10(STATEP)
632 movdqu STATE3, 0x20(STATEP)
633 movdqu STATE4, 0x30(STATEP)
634 movdqu STATE0, 0x40(STATEP)
635 FRAME_END
636 ret
637
638.Ldec_out_4:
639 movdqu STATE0, 0x00(STATEP)
640 movdqu STATE1, 0x10(STATEP)
641 movdqu STATE2, 0x20(STATEP)
642 movdqu STATE3, 0x30(STATEP)
643 movdqu STATE4, 0x40(STATEP)
644 FRAME_END
645 ret
646
647.Ldec_out:
648 FRAME_END
649 ret
650SYM_FUNC_END(crypto_aegis128_aesni_dec)
651
652/*
653 * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
654 * const void *src, void *dst);
655 */
656SYM_FUNC_START(crypto_aegis128_aesni_dec_tail)
657 FRAME_BEGIN
658
659 /* load the state: */
660 movdqu 0x00(STATEP), STATE0
661 movdqu 0x10(STATEP), STATE1
662 movdqu 0x20(STATEP), STATE2
663 movdqu 0x30(STATEP), STATE3
664 movdqu 0x40(STATEP), STATE4
665
666 /* decrypt message: */
667 call __load_partial
668
669 pxor STATE1, MSG
670 pxor STATE4, MSG
671 movdqa STATE2, T1
672 pand STATE3, T1
673 pxor T1, MSG
674
675 movdqa MSG, T0
676 call __store_partial
677
678 /* mask with byte count: */
679 movq LEN, T0
680 punpcklbw T0, T0
681 punpcklbw T0, T0
682 punpcklbw T0, T0
683 punpcklbw T0, T0
684 movdqa .Laegis128_counter, T1
685 pcmpgtb T1, T0
686 pand T0, MSG
687
688 aegis128_update
689 pxor MSG, STATE4
690
691 /* store the state: */
692 movdqu STATE4, 0x00(STATEP)
693 movdqu STATE0, 0x10(STATEP)
694 movdqu STATE1, 0x20(STATEP)
695 movdqu STATE2, 0x30(STATEP)
696 movdqu STATE3, 0x40(STATEP)
697
698 FRAME_END
699 ret
700SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
701
702/*
703 * void crypto_aegis128_aesni_final(void *state, void *tag_xor,
704 * u64 assoclen, u64 cryptlen);
705 */
706SYM_FUNC_START(crypto_aegis128_aesni_final)
707 FRAME_BEGIN
708
709 /* load the state: */
710 movdqu 0x00(STATEP), STATE0
711 movdqu 0x10(STATEP), STATE1
712 movdqu 0x20(STATEP), STATE2
713 movdqu 0x30(STATEP), STATE3
714 movdqu 0x40(STATEP), STATE4
715
716 /* prepare length block: */
717 movq %rdx, MSG
718 movq %rcx, T0
719 pslldq $8, T0
720 pxor T0, MSG
721 psllq $3, MSG /* multiply by 8 (to get bit count) */
722
723 pxor STATE3, MSG
724
725 /* update state: */
726 aegis128_update; pxor MSG, STATE4
727 aegis128_update; pxor MSG, STATE3
728 aegis128_update; pxor MSG, STATE2
729 aegis128_update; pxor MSG, STATE1
730 aegis128_update; pxor MSG, STATE0
731 aegis128_update; pxor MSG, STATE4
732 aegis128_update; pxor MSG, STATE3
733
734 /* xor tag: */
735 movdqu (%rsi), MSG
736
737 pxor STATE0, MSG
738 pxor STATE1, MSG
739 pxor STATE2, MSG
740 pxor STATE3, MSG
741 pxor STATE4, MSG
742
743 movdqu MSG, (%rsi)
744
745 FRAME_END
746 ret
747SYM_FUNC_END(crypto_aegis128_aesni_final)