Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Implement AES algorithm in Intel AES-NI instructions.
4 *
5 * The white paper of AES-NI instructions can be downloaded from:
6 * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
7 *
8 * Copyright (C) 2008, Intel Corp.
9 * Author: Huang Ying <ying.huang@intel.com>
10 * Vinodh Gopal <vinodh.gopal@intel.com>
11 * Kahraman Akdemir
12 *
13 * Copyright (c) 2010, Intel Corporation.
14 *
15 * Ported x86_64 version to x86:
16 * Author: Mathias Krause <minipli@googlemail.com>
17 */
18
19#include <linux/linkage.h>
20#include <asm/frame.h>
21
22#define STATE1 %xmm0
23#define STATE2 %xmm4
24#define STATE3 %xmm5
25#define STATE4 %xmm6
26#define STATE STATE1
27#define IN1 %xmm1
28#define IN2 %xmm7
29#define IN3 %xmm8
30#define IN4 %xmm9
31#define IN IN1
32#define KEY %xmm2
33#define IV %xmm3
34
35#define BSWAP_MASK %xmm10
36#define CTR %xmm11
37#define INC %xmm12
38
39#define GF128MUL_MASK %xmm7
40
41#ifdef __x86_64__
42#define AREG %rax
43#define KEYP %rdi
44#define OUTP %rsi
45#define UKEYP OUTP
46#define INP %rdx
47#define LEN %rcx
48#define IVP %r8
49#define KLEN %r9d
50#define T1 %r10
51#define TKEYP T1
52#define T2 %r11
53#define TCTR_LOW T2
54#else
55#define AREG %eax
56#define KEYP %edi
57#define OUTP AREG
58#define UKEYP OUTP
59#define INP %edx
60#define LEN %esi
61#define IVP %ebp
62#define KLEN %ebx
63#define T1 %ecx
64#define TKEYP T1
65#endif
66
67SYM_FUNC_START_LOCAL(_key_expansion_256a)
68 pshufd $0b11111111, %xmm1, %xmm1
69 shufps $0b00010000, %xmm0, %xmm4
70 pxor %xmm4, %xmm0
71 shufps $0b10001100, %xmm0, %xmm4
72 pxor %xmm4, %xmm0
73 pxor %xmm1, %xmm0
74 movaps %xmm0, (TKEYP)
75 add $0x10, TKEYP
76 RET
77SYM_FUNC_END(_key_expansion_256a)
78SYM_FUNC_ALIAS_LOCAL(_key_expansion_128, _key_expansion_256a)
79
80SYM_FUNC_START_LOCAL(_key_expansion_192a)
81 pshufd $0b01010101, %xmm1, %xmm1
82 shufps $0b00010000, %xmm0, %xmm4
83 pxor %xmm4, %xmm0
84 shufps $0b10001100, %xmm0, %xmm4
85 pxor %xmm4, %xmm0
86 pxor %xmm1, %xmm0
87
88 movaps %xmm2, %xmm5
89 movaps %xmm2, %xmm6
90 pslldq $4, %xmm5
91 pshufd $0b11111111, %xmm0, %xmm3
92 pxor %xmm3, %xmm2
93 pxor %xmm5, %xmm2
94
95 movaps %xmm0, %xmm1
96 shufps $0b01000100, %xmm0, %xmm6
97 movaps %xmm6, (TKEYP)
98 shufps $0b01001110, %xmm2, %xmm1
99 movaps %xmm1, 0x10(TKEYP)
100 add $0x20, TKEYP
101 RET
102SYM_FUNC_END(_key_expansion_192a)
103
104SYM_FUNC_START_LOCAL(_key_expansion_192b)
105 pshufd $0b01010101, %xmm1, %xmm1
106 shufps $0b00010000, %xmm0, %xmm4
107 pxor %xmm4, %xmm0
108 shufps $0b10001100, %xmm0, %xmm4
109 pxor %xmm4, %xmm0
110 pxor %xmm1, %xmm0
111
112 movaps %xmm2, %xmm5
113 pslldq $4, %xmm5
114 pshufd $0b11111111, %xmm0, %xmm3
115 pxor %xmm3, %xmm2
116 pxor %xmm5, %xmm2
117
118 movaps %xmm0, (TKEYP)
119 add $0x10, TKEYP
120 RET
121SYM_FUNC_END(_key_expansion_192b)
122
123SYM_FUNC_START_LOCAL(_key_expansion_256b)
124 pshufd $0b10101010, %xmm1, %xmm1
125 shufps $0b00010000, %xmm2, %xmm4
126 pxor %xmm4, %xmm2
127 shufps $0b10001100, %xmm2, %xmm4
128 pxor %xmm4, %xmm2
129 pxor %xmm1, %xmm2
130 movaps %xmm2, (TKEYP)
131 add $0x10, TKEYP
132 RET
133SYM_FUNC_END(_key_expansion_256b)
134
135/*
136 * void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
137 * unsigned int key_len)
138 */
139SYM_FUNC_START(aesni_set_key)
140 FRAME_BEGIN
141#ifndef __x86_64__
142 pushl KEYP
143 movl (FRAME_OFFSET+8)(%esp), KEYP # ctx
144 movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key
145 movl (FRAME_OFFSET+16)(%esp), %edx # key_len
146#endif
147 movups (UKEYP), %xmm0 # user key (first 16 bytes)
148 movaps %xmm0, (KEYP)
149 lea 0x10(KEYP), TKEYP # key addr
150 movl %edx, 480(KEYP)
151 pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x
152 cmp $24, %dl
153 jb .Lenc_key128
154 je .Lenc_key192
155 movups 0x10(UKEYP), %xmm2 # other user key
156 movaps %xmm2, (TKEYP)
157 add $0x10, TKEYP
158 aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
159 call _key_expansion_256a
160 aeskeygenassist $0x1, %xmm0, %xmm1
161 call _key_expansion_256b
162 aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
163 call _key_expansion_256a
164 aeskeygenassist $0x2, %xmm0, %xmm1
165 call _key_expansion_256b
166 aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
167 call _key_expansion_256a
168 aeskeygenassist $0x4, %xmm0, %xmm1
169 call _key_expansion_256b
170 aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
171 call _key_expansion_256a
172 aeskeygenassist $0x8, %xmm0, %xmm1
173 call _key_expansion_256b
174 aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
175 call _key_expansion_256a
176 aeskeygenassist $0x10, %xmm0, %xmm1
177 call _key_expansion_256b
178 aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
179 call _key_expansion_256a
180 aeskeygenassist $0x20, %xmm0, %xmm1
181 call _key_expansion_256b
182 aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
183 call _key_expansion_256a
184 jmp .Ldec_key
185.Lenc_key192:
186 movq 0x10(UKEYP), %xmm2 # other user key
187 aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
188 call _key_expansion_192a
189 aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
190 call _key_expansion_192b
191 aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
192 call _key_expansion_192a
193 aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
194 call _key_expansion_192b
195 aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
196 call _key_expansion_192a
197 aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
198 call _key_expansion_192b
199 aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
200 call _key_expansion_192a
201 aeskeygenassist $0x80, %xmm2, %xmm1 # round 8
202 call _key_expansion_192b
203 jmp .Ldec_key
204.Lenc_key128:
205 aeskeygenassist $0x1, %xmm0, %xmm1 # round 1
206 call _key_expansion_128
207 aeskeygenassist $0x2, %xmm0, %xmm1 # round 2
208 call _key_expansion_128
209 aeskeygenassist $0x4, %xmm0, %xmm1 # round 3
210 call _key_expansion_128
211 aeskeygenassist $0x8, %xmm0, %xmm1 # round 4
212 call _key_expansion_128
213 aeskeygenassist $0x10, %xmm0, %xmm1 # round 5
214 call _key_expansion_128
215 aeskeygenassist $0x20, %xmm0, %xmm1 # round 6
216 call _key_expansion_128
217 aeskeygenassist $0x40, %xmm0, %xmm1 # round 7
218 call _key_expansion_128
219 aeskeygenassist $0x80, %xmm0, %xmm1 # round 8
220 call _key_expansion_128
221 aeskeygenassist $0x1b, %xmm0, %xmm1 # round 9
222 call _key_expansion_128
223 aeskeygenassist $0x36, %xmm0, %xmm1 # round 10
224 call _key_expansion_128
225.Ldec_key:
226 sub $0x10, TKEYP
227 movaps (KEYP), %xmm0
228 movaps (TKEYP), %xmm1
229 movaps %xmm0, 240(TKEYP)
230 movaps %xmm1, 240(KEYP)
231 add $0x10, KEYP
232 lea 240-16(TKEYP), UKEYP
233.align 4
234.Ldec_key_loop:
235 movaps (KEYP), %xmm0
236 aesimc %xmm0, %xmm1
237 movaps %xmm1, (UKEYP)
238 add $0x10, KEYP
239 sub $0x10, UKEYP
240 cmp TKEYP, KEYP
241 jb .Ldec_key_loop
242#ifndef __x86_64__
243 popl KEYP
244#endif
245 FRAME_END
246 RET
247SYM_FUNC_END(aesni_set_key)
248
249/*
250 * void aesni_enc(const void *ctx, u8 *dst, const u8 *src)
251 */
252SYM_FUNC_START(aesni_enc)
253 FRAME_BEGIN
254#ifndef __x86_64__
255 pushl KEYP
256 pushl KLEN
257 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
258 movl (FRAME_OFFSET+16)(%esp), OUTP # dst
259 movl (FRAME_OFFSET+20)(%esp), INP # src
260#endif
261 movl 480(KEYP), KLEN # key length
262 movups (INP), STATE # input
263 call _aesni_enc1
264 movups STATE, (OUTP) # output
265#ifndef __x86_64__
266 popl KLEN
267 popl KEYP
268#endif
269 FRAME_END
270 RET
271SYM_FUNC_END(aesni_enc)
272
273/*
274 * _aesni_enc1: internal ABI
275 * input:
276 * KEYP: key struct pointer
277 * KLEN: round count
278 * STATE: initial state (input)
279 * output:
280 * STATE: finial state (output)
281 * changed:
282 * KEY
283 * TKEYP (T1)
284 */
285SYM_FUNC_START_LOCAL(_aesni_enc1)
286 movaps (KEYP), KEY # key
287 mov KEYP, TKEYP
288 pxor KEY, STATE # round 0
289 add $0x30, TKEYP
290 cmp $24, KLEN
291 jb .Lenc128
292 lea 0x20(TKEYP), TKEYP
293 je .Lenc192
294 add $0x20, TKEYP
295 movaps -0x60(TKEYP), KEY
296 aesenc KEY, STATE
297 movaps -0x50(TKEYP), KEY
298 aesenc KEY, STATE
299.align 4
300.Lenc192:
301 movaps -0x40(TKEYP), KEY
302 aesenc KEY, STATE
303 movaps -0x30(TKEYP), KEY
304 aesenc KEY, STATE
305.align 4
306.Lenc128:
307 movaps -0x20(TKEYP), KEY
308 aesenc KEY, STATE
309 movaps -0x10(TKEYP), KEY
310 aesenc KEY, STATE
311 movaps (TKEYP), KEY
312 aesenc KEY, STATE
313 movaps 0x10(TKEYP), KEY
314 aesenc KEY, STATE
315 movaps 0x20(TKEYP), KEY
316 aesenc KEY, STATE
317 movaps 0x30(TKEYP), KEY
318 aesenc KEY, STATE
319 movaps 0x40(TKEYP), KEY
320 aesenc KEY, STATE
321 movaps 0x50(TKEYP), KEY
322 aesenc KEY, STATE
323 movaps 0x60(TKEYP), KEY
324 aesenc KEY, STATE
325 movaps 0x70(TKEYP), KEY
326 aesenclast KEY, STATE
327 RET
328SYM_FUNC_END(_aesni_enc1)
329
330/*
331 * _aesni_enc4: internal ABI
332 * input:
333 * KEYP: key struct pointer
334 * KLEN: round count
335 * STATE1: initial state (input)
336 * STATE2
337 * STATE3
338 * STATE4
339 * output:
340 * STATE1: finial state (output)
341 * STATE2
342 * STATE3
343 * STATE4
344 * changed:
345 * KEY
346 * TKEYP (T1)
347 */
348SYM_FUNC_START_LOCAL(_aesni_enc4)
349 movaps (KEYP), KEY # key
350 mov KEYP, TKEYP
351 pxor KEY, STATE1 # round 0
352 pxor KEY, STATE2
353 pxor KEY, STATE3
354 pxor KEY, STATE4
355 add $0x30, TKEYP
356 cmp $24, KLEN
357 jb .L4enc128
358 lea 0x20(TKEYP), TKEYP
359 je .L4enc192
360 add $0x20, TKEYP
361 movaps -0x60(TKEYP), KEY
362 aesenc KEY, STATE1
363 aesenc KEY, STATE2
364 aesenc KEY, STATE3
365 aesenc KEY, STATE4
366 movaps -0x50(TKEYP), KEY
367 aesenc KEY, STATE1
368 aesenc KEY, STATE2
369 aesenc KEY, STATE3
370 aesenc KEY, STATE4
371#.align 4
372.L4enc192:
373 movaps -0x40(TKEYP), KEY
374 aesenc KEY, STATE1
375 aesenc KEY, STATE2
376 aesenc KEY, STATE3
377 aesenc KEY, STATE4
378 movaps -0x30(TKEYP), KEY
379 aesenc KEY, STATE1
380 aesenc KEY, STATE2
381 aesenc KEY, STATE3
382 aesenc KEY, STATE4
383#.align 4
384.L4enc128:
385 movaps -0x20(TKEYP), KEY
386 aesenc KEY, STATE1
387 aesenc KEY, STATE2
388 aesenc KEY, STATE3
389 aesenc KEY, STATE4
390 movaps -0x10(TKEYP), KEY
391 aesenc KEY, STATE1
392 aesenc KEY, STATE2
393 aesenc KEY, STATE3
394 aesenc KEY, STATE4
395 movaps (TKEYP), KEY
396 aesenc KEY, STATE1
397 aesenc KEY, STATE2
398 aesenc KEY, STATE3
399 aesenc KEY, STATE4
400 movaps 0x10(TKEYP), KEY
401 aesenc KEY, STATE1
402 aesenc KEY, STATE2
403 aesenc KEY, STATE3
404 aesenc KEY, STATE4
405 movaps 0x20(TKEYP), KEY
406 aesenc KEY, STATE1
407 aesenc KEY, STATE2
408 aesenc KEY, STATE3
409 aesenc KEY, STATE4
410 movaps 0x30(TKEYP), KEY
411 aesenc KEY, STATE1
412 aesenc KEY, STATE2
413 aesenc KEY, STATE3
414 aesenc KEY, STATE4
415 movaps 0x40(TKEYP), KEY
416 aesenc KEY, STATE1
417 aesenc KEY, STATE2
418 aesenc KEY, STATE3
419 aesenc KEY, STATE4
420 movaps 0x50(TKEYP), KEY
421 aesenc KEY, STATE1
422 aesenc KEY, STATE2
423 aesenc KEY, STATE3
424 aesenc KEY, STATE4
425 movaps 0x60(TKEYP), KEY
426 aesenc KEY, STATE1
427 aesenc KEY, STATE2
428 aesenc KEY, STATE3
429 aesenc KEY, STATE4
430 movaps 0x70(TKEYP), KEY
431 aesenclast KEY, STATE1 # last round
432 aesenclast KEY, STATE2
433 aesenclast KEY, STATE3
434 aesenclast KEY, STATE4
435 RET
436SYM_FUNC_END(_aesni_enc4)
437
438/*
439 * void aesni_dec (const void *ctx, u8 *dst, const u8 *src)
440 */
441SYM_FUNC_START(aesni_dec)
442 FRAME_BEGIN
443#ifndef __x86_64__
444 pushl KEYP
445 pushl KLEN
446 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
447 movl (FRAME_OFFSET+16)(%esp), OUTP # dst
448 movl (FRAME_OFFSET+20)(%esp), INP # src
449#endif
450 mov 480(KEYP), KLEN # key length
451 add $240, KEYP
452 movups (INP), STATE # input
453 call _aesni_dec1
454 movups STATE, (OUTP) #output
455#ifndef __x86_64__
456 popl KLEN
457 popl KEYP
458#endif
459 FRAME_END
460 RET
461SYM_FUNC_END(aesni_dec)
462
463/*
464 * _aesni_dec1: internal ABI
465 * input:
466 * KEYP: key struct pointer
467 * KLEN: key length
468 * STATE: initial state (input)
469 * output:
470 * STATE: finial state (output)
471 * changed:
472 * KEY
473 * TKEYP (T1)
474 */
475SYM_FUNC_START_LOCAL(_aesni_dec1)
476 movaps (KEYP), KEY # key
477 mov KEYP, TKEYP
478 pxor KEY, STATE # round 0
479 add $0x30, TKEYP
480 cmp $24, KLEN
481 jb .Ldec128
482 lea 0x20(TKEYP), TKEYP
483 je .Ldec192
484 add $0x20, TKEYP
485 movaps -0x60(TKEYP), KEY
486 aesdec KEY, STATE
487 movaps -0x50(TKEYP), KEY
488 aesdec KEY, STATE
489.align 4
490.Ldec192:
491 movaps -0x40(TKEYP), KEY
492 aesdec KEY, STATE
493 movaps -0x30(TKEYP), KEY
494 aesdec KEY, STATE
495.align 4
496.Ldec128:
497 movaps -0x20(TKEYP), KEY
498 aesdec KEY, STATE
499 movaps -0x10(TKEYP), KEY
500 aesdec KEY, STATE
501 movaps (TKEYP), KEY
502 aesdec KEY, STATE
503 movaps 0x10(TKEYP), KEY
504 aesdec KEY, STATE
505 movaps 0x20(TKEYP), KEY
506 aesdec KEY, STATE
507 movaps 0x30(TKEYP), KEY
508 aesdec KEY, STATE
509 movaps 0x40(TKEYP), KEY
510 aesdec KEY, STATE
511 movaps 0x50(TKEYP), KEY
512 aesdec KEY, STATE
513 movaps 0x60(TKEYP), KEY
514 aesdec KEY, STATE
515 movaps 0x70(TKEYP), KEY
516 aesdeclast KEY, STATE
517 RET
518SYM_FUNC_END(_aesni_dec1)
519
520/*
521 * _aesni_dec4: internal ABI
522 * input:
523 * KEYP: key struct pointer
524 * KLEN: key length
525 * STATE1: initial state (input)
526 * STATE2
527 * STATE3
528 * STATE4
529 * output:
530 * STATE1: finial state (output)
531 * STATE2
532 * STATE3
533 * STATE4
534 * changed:
535 * KEY
536 * TKEYP (T1)
537 */
538SYM_FUNC_START_LOCAL(_aesni_dec4)
539 movaps (KEYP), KEY # key
540 mov KEYP, TKEYP
541 pxor KEY, STATE1 # round 0
542 pxor KEY, STATE2
543 pxor KEY, STATE3
544 pxor KEY, STATE4
545 add $0x30, TKEYP
546 cmp $24, KLEN
547 jb .L4dec128
548 lea 0x20(TKEYP), TKEYP
549 je .L4dec192
550 add $0x20, TKEYP
551 movaps -0x60(TKEYP), KEY
552 aesdec KEY, STATE1
553 aesdec KEY, STATE2
554 aesdec KEY, STATE3
555 aesdec KEY, STATE4
556 movaps -0x50(TKEYP), KEY
557 aesdec KEY, STATE1
558 aesdec KEY, STATE2
559 aesdec KEY, STATE3
560 aesdec KEY, STATE4
561.align 4
562.L4dec192:
563 movaps -0x40(TKEYP), KEY
564 aesdec KEY, STATE1
565 aesdec KEY, STATE2
566 aesdec KEY, STATE3
567 aesdec KEY, STATE4
568 movaps -0x30(TKEYP), KEY
569 aesdec KEY, STATE1
570 aesdec KEY, STATE2
571 aesdec KEY, STATE3
572 aesdec KEY, STATE4
573.align 4
574.L4dec128:
575 movaps -0x20(TKEYP), KEY
576 aesdec KEY, STATE1
577 aesdec KEY, STATE2
578 aesdec KEY, STATE3
579 aesdec KEY, STATE4
580 movaps -0x10(TKEYP), KEY
581 aesdec KEY, STATE1
582 aesdec KEY, STATE2
583 aesdec KEY, STATE3
584 aesdec KEY, STATE4
585 movaps (TKEYP), KEY
586 aesdec KEY, STATE1
587 aesdec KEY, STATE2
588 aesdec KEY, STATE3
589 aesdec KEY, STATE4
590 movaps 0x10(TKEYP), KEY
591 aesdec KEY, STATE1
592 aesdec KEY, STATE2
593 aesdec KEY, STATE3
594 aesdec KEY, STATE4
595 movaps 0x20(TKEYP), KEY
596 aesdec KEY, STATE1
597 aesdec KEY, STATE2
598 aesdec KEY, STATE3
599 aesdec KEY, STATE4
600 movaps 0x30(TKEYP), KEY
601 aesdec KEY, STATE1
602 aesdec KEY, STATE2
603 aesdec KEY, STATE3
604 aesdec KEY, STATE4
605 movaps 0x40(TKEYP), KEY
606 aesdec KEY, STATE1
607 aesdec KEY, STATE2
608 aesdec KEY, STATE3
609 aesdec KEY, STATE4
610 movaps 0x50(TKEYP), KEY
611 aesdec KEY, STATE1
612 aesdec KEY, STATE2
613 aesdec KEY, STATE3
614 aesdec KEY, STATE4
615 movaps 0x60(TKEYP), KEY
616 aesdec KEY, STATE1
617 aesdec KEY, STATE2
618 aesdec KEY, STATE3
619 aesdec KEY, STATE4
620 movaps 0x70(TKEYP), KEY
621 aesdeclast KEY, STATE1 # last round
622 aesdeclast KEY, STATE2
623 aesdeclast KEY, STATE3
624 aesdeclast KEY, STATE4
625 RET
626SYM_FUNC_END(_aesni_dec4)
627
628/*
629 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
630 * size_t len)
631 */
632SYM_FUNC_START(aesni_ecb_enc)
633 FRAME_BEGIN
634#ifndef __x86_64__
635 pushl LEN
636 pushl KEYP
637 pushl KLEN
638 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
639 movl (FRAME_OFFSET+20)(%esp), OUTP # dst
640 movl (FRAME_OFFSET+24)(%esp), INP # src
641 movl (FRAME_OFFSET+28)(%esp), LEN # len
642#endif
643 test LEN, LEN # check length
644 jz .Lecb_enc_ret
645 mov 480(KEYP), KLEN
646 cmp $16, LEN
647 jb .Lecb_enc_ret
648 cmp $64, LEN
649 jb .Lecb_enc_loop1
650.align 4
651.Lecb_enc_loop4:
652 movups (INP), STATE1
653 movups 0x10(INP), STATE2
654 movups 0x20(INP), STATE3
655 movups 0x30(INP), STATE4
656 call _aesni_enc4
657 movups STATE1, (OUTP)
658 movups STATE2, 0x10(OUTP)
659 movups STATE3, 0x20(OUTP)
660 movups STATE4, 0x30(OUTP)
661 sub $64, LEN
662 add $64, INP
663 add $64, OUTP
664 cmp $64, LEN
665 jge .Lecb_enc_loop4
666 cmp $16, LEN
667 jb .Lecb_enc_ret
668.align 4
669.Lecb_enc_loop1:
670 movups (INP), STATE1
671 call _aesni_enc1
672 movups STATE1, (OUTP)
673 sub $16, LEN
674 add $16, INP
675 add $16, OUTP
676 cmp $16, LEN
677 jge .Lecb_enc_loop1
678.Lecb_enc_ret:
679#ifndef __x86_64__
680 popl KLEN
681 popl KEYP
682 popl LEN
683#endif
684 FRAME_END
685 RET
686SYM_FUNC_END(aesni_ecb_enc)
687
688/*
689 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
690 * size_t len);
691 */
692SYM_FUNC_START(aesni_ecb_dec)
693 FRAME_BEGIN
694#ifndef __x86_64__
695 pushl LEN
696 pushl KEYP
697 pushl KLEN
698 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
699 movl (FRAME_OFFSET+20)(%esp), OUTP # dst
700 movl (FRAME_OFFSET+24)(%esp), INP # src
701 movl (FRAME_OFFSET+28)(%esp), LEN # len
702#endif
703 test LEN, LEN
704 jz .Lecb_dec_ret
705 mov 480(KEYP), KLEN
706 add $240, KEYP
707 cmp $16, LEN
708 jb .Lecb_dec_ret
709 cmp $64, LEN
710 jb .Lecb_dec_loop1
711.align 4
712.Lecb_dec_loop4:
713 movups (INP), STATE1
714 movups 0x10(INP), STATE2
715 movups 0x20(INP), STATE3
716 movups 0x30(INP), STATE4
717 call _aesni_dec4
718 movups STATE1, (OUTP)
719 movups STATE2, 0x10(OUTP)
720 movups STATE3, 0x20(OUTP)
721 movups STATE4, 0x30(OUTP)
722 sub $64, LEN
723 add $64, INP
724 add $64, OUTP
725 cmp $64, LEN
726 jge .Lecb_dec_loop4
727 cmp $16, LEN
728 jb .Lecb_dec_ret
729.align 4
730.Lecb_dec_loop1:
731 movups (INP), STATE1
732 call _aesni_dec1
733 movups STATE1, (OUTP)
734 sub $16, LEN
735 add $16, INP
736 add $16, OUTP
737 cmp $16, LEN
738 jge .Lecb_dec_loop1
739.Lecb_dec_ret:
740#ifndef __x86_64__
741 popl KLEN
742 popl KEYP
743 popl LEN
744#endif
745 FRAME_END
746 RET
747SYM_FUNC_END(aesni_ecb_dec)
748
749/*
750 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
751 * size_t len, u8 *iv)
752 */
753SYM_FUNC_START(aesni_cbc_enc)
754 FRAME_BEGIN
755#ifndef __x86_64__
756 pushl IVP
757 pushl LEN
758 pushl KEYP
759 pushl KLEN
760 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
761 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
762 movl (FRAME_OFFSET+28)(%esp), INP # src
763 movl (FRAME_OFFSET+32)(%esp), LEN # len
764 movl (FRAME_OFFSET+36)(%esp), IVP # iv
765#endif
766 cmp $16, LEN
767 jb .Lcbc_enc_ret
768 mov 480(KEYP), KLEN
769 movups (IVP), STATE # load iv as initial state
770.align 4
771.Lcbc_enc_loop:
772 movups (INP), IN # load input
773 pxor IN, STATE
774 call _aesni_enc1
775 movups STATE, (OUTP) # store output
776 sub $16, LEN
777 add $16, INP
778 add $16, OUTP
779 cmp $16, LEN
780 jge .Lcbc_enc_loop
781 movups STATE, (IVP)
782.Lcbc_enc_ret:
783#ifndef __x86_64__
784 popl KLEN
785 popl KEYP
786 popl LEN
787 popl IVP
788#endif
789 FRAME_END
790 RET
791SYM_FUNC_END(aesni_cbc_enc)
792
793/*
794 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
795 * size_t len, u8 *iv)
796 */
797SYM_FUNC_START(aesni_cbc_dec)
798 FRAME_BEGIN
799#ifndef __x86_64__
800 pushl IVP
801 pushl LEN
802 pushl KEYP
803 pushl KLEN
804 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
805 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
806 movl (FRAME_OFFSET+28)(%esp), INP # src
807 movl (FRAME_OFFSET+32)(%esp), LEN # len
808 movl (FRAME_OFFSET+36)(%esp), IVP # iv
809#endif
810 cmp $16, LEN
811 jb .Lcbc_dec_just_ret
812 mov 480(KEYP), KLEN
813 add $240, KEYP
814 movups (IVP), IV
815 cmp $64, LEN
816 jb .Lcbc_dec_loop1
817.align 4
818.Lcbc_dec_loop4:
819 movups (INP), IN1
820 movaps IN1, STATE1
821 movups 0x10(INP), IN2
822 movaps IN2, STATE2
823#ifdef __x86_64__
824 movups 0x20(INP), IN3
825 movaps IN3, STATE3
826 movups 0x30(INP), IN4
827 movaps IN4, STATE4
828#else
829 movups 0x20(INP), IN1
830 movaps IN1, STATE3
831 movups 0x30(INP), IN2
832 movaps IN2, STATE4
833#endif
834 call _aesni_dec4
835 pxor IV, STATE1
836#ifdef __x86_64__
837 pxor IN1, STATE2
838 pxor IN2, STATE3
839 pxor IN3, STATE4
840 movaps IN4, IV
841#else
842 pxor IN1, STATE4
843 movaps IN2, IV
844 movups (INP), IN1
845 pxor IN1, STATE2
846 movups 0x10(INP), IN2
847 pxor IN2, STATE3
848#endif
849 movups STATE1, (OUTP)
850 movups STATE2, 0x10(OUTP)
851 movups STATE3, 0x20(OUTP)
852 movups STATE4, 0x30(OUTP)
853 sub $64, LEN
854 add $64, INP
855 add $64, OUTP
856 cmp $64, LEN
857 jge .Lcbc_dec_loop4
858 cmp $16, LEN
859 jb .Lcbc_dec_ret
860.align 4
861.Lcbc_dec_loop1:
862 movups (INP), IN
863 movaps IN, STATE
864 call _aesni_dec1
865 pxor IV, STATE
866 movups STATE, (OUTP)
867 movaps IN, IV
868 sub $16, LEN
869 add $16, INP
870 add $16, OUTP
871 cmp $16, LEN
872 jge .Lcbc_dec_loop1
873.Lcbc_dec_ret:
874 movups IV, (IVP)
875.Lcbc_dec_just_ret:
876#ifndef __x86_64__
877 popl KLEN
878 popl KEYP
879 popl LEN
880 popl IVP
881#endif
882 FRAME_END
883 RET
884SYM_FUNC_END(aesni_cbc_dec)
885
886/*
887 * void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
888 * size_t len, u8 *iv)
889 */
890SYM_FUNC_START(aesni_cts_cbc_enc)
891 FRAME_BEGIN
892#ifndef __x86_64__
893 pushl IVP
894 pushl LEN
895 pushl KEYP
896 pushl KLEN
897 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
898 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
899 movl (FRAME_OFFSET+28)(%esp), INP # src
900 movl (FRAME_OFFSET+32)(%esp), LEN # len
901 movl (FRAME_OFFSET+36)(%esp), IVP # iv
902 lea .Lcts_permute_table, T1
903#else
904 lea .Lcts_permute_table(%rip), T1
905#endif
906 mov 480(KEYP), KLEN
907 movups (IVP), STATE
908 sub $16, LEN
909 mov T1, IVP
910 add $32, IVP
911 add LEN, T1
912 sub LEN, IVP
913 movups (T1), %xmm4
914 movups (IVP), %xmm5
915
916 movups (INP), IN1
917 add LEN, INP
918 movups (INP), IN2
919
920 pxor IN1, STATE
921 call _aesni_enc1
922
923 pshufb %xmm5, IN2
924 pxor STATE, IN2
925 pshufb %xmm4, STATE
926 add OUTP, LEN
927 movups STATE, (LEN)
928
929 movaps IN2, STATE
930 call _aesni_enc1
931 movups STATE, (OUTP)
932
933#ifndef __x86_64__
934 popl KLEN
935 popl KEYP
936 popl LEN
937 popl IVP
938#endif
939 FRAME_END
940 RET
941SYM_FUNC_END(aesni_cts_cbc_enc)
942
943/*
944 * void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
945 * size_t len, u8 *iv)
946 */
947SYM_FUNC_START(aesni_cts_cbc_dec)
948 FRAME_BEGIN
949#ifndef __x86_64__
950 pushl IVP
951 pushl LEN
952 pushl KEYP
953 pushl KLEN
954 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
955 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
956 movl (FRAME_OFFSET+28)(%esp), INP # src
957 movl (FRAME_OFFSET+32)(%esp), LEN # len
958 movl (FRAME_OFFSET+36)(%esp), IVP # iv
959 lea .Lcts_permute_table, T1
960#else
961 lea .Lcts_permute_table(%rip), T1
962#endif
963 mov 480(KEYP), KLEN
964 add $240, KEYP
965 movups (IVP), IV
966 sub $16, LEN
967 mov T1, IVP
968 add $32, IVP
969 add LEN, T1
970 sub LEN, IVP
971 movups (T1), %xmm4
972
973 movups (INP), STATE
974 add LEN, INP
975 movups (INP), IN1
976
977 call _aesni_dec1
978 movaps STATE, IN2
979 pshufb %xmm4, STATE
980 pxor IN1, STATE
981
982 add OUTP, LEN
983 movups STATE, (LEN)
984
985 movups (IVP), %xmm0
986 pshufb %xmm0, IN1
987 pblendvb IN2, IN1
988 movaps IN1, STATE
989 call _aesni_dec1
990
991 pxor IV, STATE
992 movups STATE, (OUTP)
993
994#ifndef __x86_64__
995 popl KLEN
996 popl KEYP
997 popl LEN
998 popl IVP
999#endif
1000 FRAME_END
1001 RET
1002SYM_FUNC_END(aesni_cts_cbc_dec)
1003
1004.pushsection .rodata
1005.align 16
1006.Lcts_permute_table:
1007 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
1008 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
1009 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
1010 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
1011 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
1012 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
1013#ifdef __x86_64__
1014.Lbswap_mask:
1015 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
1016#endif
1017.popsection
1018
1019#ifdef __x86_64__
1020/*
1021 * _aesni_inc_init: internal ABI
1022 * setup registers used by _aesni_inc
1023 * input:
1024 * IV
1025 * output:
1026 * CTR: == IV, in little endian
1027 * TCTR_LOW: == lower qword of CTR
1028 * INC: == 1, in little endian
1029 * BSWAP_MASK == endian swapping mask
1030 */
1031SYM_FUNC_START_LOCAL(_aesni_inc_init)
1032 movaps .Lbswap_mask(%rip), BSWAP_MASK
1033 movaps IV, CTR
1034 pshufb BSWAP_MASK, CTR
1035 mov $1, TCTR_LOW
1036 movq TCTR_LOW, INC
1037 movq CTR, TCTR_LOW
1038 RET
1039SYM_FUNC_END(_aesni_inc_init)
1040
1041/*
1042 * _aesni_inc: internal ABI
1043 * Increase IV by 1, IV is in big endian
1044 * input:
1045 * IV
1046 * CTR: == IV, in little endian
1047 * TCTR_LOW: == lower qword of CTR
1048 * INC: == 1, in little endian
1049 * BSWAP_MASK == endian swapping mask
1050 * output:
1051 * IV: Increase by 1
1052 * changed:
1053 * CTR: == output IV, in little endian
1054 * TCTR_LOW: == lower qword of CTR
1055 */
1056SYM_FUNC_START_LOCAL(_aesni_inc)
1057 paddq INC, CTR
1058 add $1, TCTR_LOW
1059 jnc .Linc_low
1060 pslldq $8, INC
1061 paddq INC, CTR
1062 psrldq $8, INC
1063.Linc_low:
1064 movaps CTR, IV
1065 pshufb BSWAP_MASK, IV
1066 RET
1067SYM_FUNC_END(_aesni_inc)
1068
1069/*
1070 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
1071 * size_t len, u8 *iv)
1072 */
1073SYM_FUNC_START(aesni_ctr_enc)
1074 FRAME_BEGIN
1075 cmp $16, LEN
1076 jb .Lctr_enc_just_ret
1077 mov 480(KEYP), KLEN
1078 movups (IVP), IV
1079 call _aesni_inc_init
1080 cmp $64, LEN
1081 jb .Lctr_enc_loop1
1082.align 4
1083.Lctr_enc_loop4:
1084 movaps IV, STATE1
1085 call _aesni_inc
1086 movups (INP), IN1
1087 movaps IV, STATE2
1088 call _aesni_inc
1089 movups 0x10(INP), IN2
1090 movaps IV, STATE3
1091 call _aesni_inc
1092 movups 0x20(INP), IN3
1093 movaps IV, STATE4
1094 call _aesni_inc
1095 movups 0x30(INP), IN4
1096 call _aesni_enc4
1097 pxor IN1, STATE1
1098 movups STATE1, (OUTP)
1099 pxor IN2, STATE2
1100 movups STATE2, 0x10(OUTP)
1101 pxor IN3, STATE3
1102 movups STATE3, 0x20(OUTP)
1103 pxor IN4, STATE4
1104 movups STATE4, 0x30(OUTP)
1105 sub $64, LEN
1106 add $64, INP
1107 add $64, OUTP
1108 cmp $64, LEN
1109 jge .Lctr_enc_loop4
1110 cmp $16, LEN
1111 jb .Lctr_enc_ret
1112.align 4
1113.Lctr_enc_loop1:
1114 movaps IV, STATE
1115 call _aesni_inc
1116 movups (INP), IN
1117 call _aesni_enc1
1118 pxor IN, STATE
1119 movups STATE, (OUTP)
1120 sub $16, LEN
1121 add $16, INP
1122 add $16, OUTP
1123 cmp $16, LEN
1124 jge .Lctr_enc_loop1
1125.Lctr_enc_ret:
1126 movups IV, (IVP)
1127.Lctr_enc_just_ret:
1128 FRAME_END
1129 RET
1130SYM_FUNC_END(aesni_ctr_enc)
1131
1132#endif
1133
1134.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16
1135.align 16
1136.Lgf128mul_x_ble_mask:
1137 .octa 0x00000000000000010000000000000087
1138.previous
1139
1140/*
1141 * _aesni_gf128mul_x_ble: Multiply in GF(2^128) for XTS IVs
1142 * input:
1143 * IV: current IV
1144 * GF128MUL_MASK == mask with 0x87 and 0x01
1145 * output:
1146 * IV: next IV
1147 * changed:
1148 * KEY: == temporary value
1149 */
1150.macro _aesni_gf128mul_x_ble
1151 pshufd $0x13, IV, KEY
1152 paddq IV, IV
1153 psrad $31, KEY
1154 pand GF128MUL_MASK, KEY
1155 pxor KEY, IV
1156.endm
1157
1158.macro _aesni_xts_crypt enc
1159 FRAME_BEGIN
1160#ifndef __x86_64__
1161 pushl IVP
1162 pushl LEN
1163 pushl KEYP
1164 pushl KLEN
1165 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
1166 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
1167 movl (FRAME_OFFSET+28)(%esp), INP # src
1168 movl (FRAME_OFFSET+32)(%esp), LEN # len
1169 movl (FRAME_OFFSET+36)(%esp), IVP # iv
1170 movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK
1171#else
1172 movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK
1173#endif
1174 movups (IVP), IV
1175
1176 mov 480(KEYP), KLEN
1177.if !\enc
1178 add $240, KEYP
1179
1180 test $15, LEN
1181 jz .Lxts_loop4\@
1182 sub $16, LEN
1183.endif
1184
1185.Lxts_loop4\@:
1186 sub $64, LEN
1187 jl .Lxts_1x\@
1188
1189 movdqa IV, STATE1
1190 movdqu 0x00(INP), IN
1191 pxor IN, STATE1
1192 movdqu IV, 0x00(OUTP)
1193
1194 _aesni_gf128mul_x_ble
1195 movdqa IV, STATE2
1196 movdqu 0x10(INP), IN
1197 pxor IN, STATE2
1198 movdqu IV, 0x10(OUTP)
1199
1200 _aesni_gf128mul_x_ble
1201 movdqa IV, STATE3
1202 movdqu 0x20(INP), IN
1203 pxor IN, STATE3
1204 movdqu IV, 0x20(OUTP)
1205
1206 _aesni_gf128mul_x_ble
1207 movdqa IV, STATE4
1208 movdqu 0x30(INP), IN
1209 pxor IN, STATE4
1210 movdqu IV, 0x30(OUTP)
1211
1212.if \enc
1213 call _aesni_enc4
1214.else
1215 call _aesni_dec4
1216.endif
1217
1218 movdqu 0x00(OUTP), IN
1219 pxor IN, STATE1
1220 movdqu STATE1, 0x00(OUTP)
1221
1222 movdqu 0x10(OUTP), IN
1223 pxor IN, STATE2
1224 movdqu STATE2, 0x10(OUTP)
1225
1226 movdqu 0x20(OUTP), IN
1227 pxor IN, STATE3
1228 movdqu STATE3, 0x20(OUTP)
1229
1230 movdqu 0x30(OUTP), IN
1231 pxor IN, STATE4
1232 movdqu STATE4, 0x30(OUTP)
1233
1234 _aesni_gf128mul_x_ble
1235
1236 add $64, INP
1237 add $64, OUTP
1238 test LEN, LEN
1239 jnz .Lxts_loop4\@
1240
1241.Lxts_ret_iv\@:
1242 movups IV, (IVP)
1243
1244.Lxts_ret\@:
1245#ifndef __x86_64__
1246 popl KLEN
1247 popl KEYP
1248 popl LEN
1249 popl IVP
1250#endif
1251 FRAME_END
1252 RET
1253
1254.Lxts_1x\@:
1255 add $64, LEN
1256 jz .Lxts_ret_iv\@
1257.if \enc
1258 sub $16, LEN
1259 jl .Lxts_cts4\@
1260.endif
1261
1262.Lxts_loop1\@:
1263 movdqu (INP), STATE
1264.if \enc
1265 pxor IV, STATE
1266 call _aesni_enc1
1267.else
1268 add $16, INP
1269 sub $16, LEN
1270 jl .Lxts_cts1\@
1271 pxor IV, STATE
1272 call _aesni_dec1
1273.endif
1274 pxor IV, STATE
1275 _aesni_gf128mul_x_ble
1276
1277 test LEN, LEN
1278 jz .Lxts_out\@
1279
1280.if \enc
1281 add $16, INP
1282 sub $16, LEN
1283 jl .Lxts_cts1\@
1284.endif
1285
1286 movdqu STATE, (OUTP)
1287 add $16, OUTP
1288 jmp .Lxts_loop1\@
1289
1290.Lxts_out\@:
1291 movdqu STATE, (OUTP)
1292 jmp .Lxts_ret_iv\@
1293
1294.if \enc
1295.Lxts_cts4\@:
1296 movdqa STATE4, STATE
1297 sub $16, OUTP
1298.Lxts_cts1\@:
1299.else
1300.Lxts_cts1\@:
1301 movdqa IV, STATE4
1302 _aesni_gf128mul_x_ble
1303
1304 pxor IV, STATE
1305 call _aesni_dec1
1306 pxor IV, STATE
1307.endif
1308#ifndef __x86_64__
1309 lea .Lcts_permute_table, T1
1310#else
1311 lea .Lcts_permute_table(%rip), T1
1312#endif
1313 add LEN, INP /* rewind input pointer */
1314 add $16, LEN /* # bytes in final block */
1315 movups (INP), IN1
1316
1317 mov T1, IVP
1318 add $32, IVP
1319 add LEN, T1
1320 sub LEN, IVP
1321 add OUTP, LEN
1322
1323 movups (T1), %xmm4
1324 movaps STATE, IN2
1325 pshufb %xmm4, STATE
1326 movups STATE, (LEN)
1327
1328 movups (IVP), %xmm0
1329 pshufb %xmm0, IN1
1330 pblendvb IN2, IN1
1331 movaps IN1, STATE
1332
1333.if \enc
1334 pxor IV, STATE
1335 call _aesni_enc1
1336 pxor IV, STATE
1337.else
1338 pxor STATE4, STATE
1339 call _aesni_dec1
1340 pxor STATE4, STATE
1341.endif
1342
1343 movups STATE, (OUTP)
1344 jmp .Lxts_ret\@
1345.endm
1346
1347/*
1348 * void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *dst,
1349 * const u8 *src, unsigned int len, le128 *iv)
1350 */
1351SYM_FUNC_START(aesni_xts_enc)
1352 _aesni_xts_crypt 1
1353SYM_FUNC_END(aesni_xts_enc)
1354
1355/*
1356 * void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *dst,
1357 * const u8 *src, unsigned int len, le128 *iv)
1358 */
1359SYM_FUNC_START(aesni_xts_dec)
1360 _aesni_xts_crypt 0
1361SYM_FUNC_END(aesni_xts_dec)
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Implement AES algorithm in Intel AES-NI instructions.
4 *
5 * The white paper of AES-NI instructions can be downloaded from:
6 * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
7 *
8 * Copyright (C) 2008, Intel Corp.
9 * Author: Huang Ying <ying.huang@intel.com>
10 * Vinodh Gopal <vinodh.gopal@intel.com>
11 * Kahraman Akdemir
12 *
13 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
14 * interface for 64-bit kernels.
15 * Authors: Erdinc Ozturk (erdinc.ozturk@intel.com)
16 * Aidan O'Mahony (aidan.o.mahony@intel.com)
17 * Adrian Hoban <adrian.hoban@intel.com>
18 * James Guilford (james.guilford@intel.com)
19 * Gabriele Paoloni <gabriele.paoloni@intel.com>
20 * Tadeusz Struk (tadeusz.struk@intel.com)
21 * Wajdi Feghali (wajdi.k.feghali@intel.com)
22 * Copyright (c) 2010, Intel Corporation.
23 *
24 * Ported x86_64 version to x86:
25 * Author: Mathias Krause <minipli@googlemail.com>
26 */
27
28#include <linux/linkage.h>
29#include <asm/frame.h>
30#include <asm/nospec-branch.h>
31
32/*
33 * The following macros are used to move an (un)aligned 16 byte value to/from
34 * an XMM register. This can done for either FP or integer values, for FP use
35 * movaps (move aligned packed single) or integer use movdqa (move double quad
36 * aligned). It doesn't make a performance difference which instruction is used
37 * since Nehalem (original Core i7) was released. However, the movaps is a byte
38 * shorter, so that is the one we'll use for now. (same for unaligned).
39 */
40#define MOVADQ movaps
41#define MOVUDQ movups
42
43#ifdef __x86_64__
44
45# constants in mergeable sections, linker can reorder and merge
46.section .rodata.cst16.POLY, "aM", @progbits, 16
47.align 16
48POLY: .octa 0xC2000000000000000000000000000001
49.section .rodata.cst16.TWOONE, "aM", @progbits, 16
50.align 16
51TWOONE: .octa 0x00000001000000000000000000000001
52
53.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16
54.align 16
55SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F
56.section .rodata.cst16.MASK1, "aM", @progbits, 16
57.align 16
58MASK1: .octa 0x0000000000000000ffffffffffffffff
59.section .rodata.cst16.MASK2, "aM", @progbits, 16
60.align 16
61MASK2: .octa 0xffffffffffffffff0000000000000000
62.section .rodata.cst16.ONE, "aM", @progbits, 16
63.align 16
64ONE: .octa 0x00000000000000000000000000000001
65.section .rodata.cst16.F_MIN_MASK, "aM", @progbits, 16
66.align 16
67F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0
68.section .rodata.cst16.dec, "aM", @progbits, 16
69.align 16
70dec: .octa 0x1
71.section .rodata.cst16.enc, "aM", @progbits, 16
72.align 16
73enc: .octa 0x2
74
75# order of these constants should not change.
76# more specifically, ALL_F should follow SHIFT_MASK,
77# and zero should follow ALL_F
78.section .rodata, "a", @progbits
79.align 16
80SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
81ALL_F: .octa 0xffffffffffffffffffffffffffffffff
82 .octa 0x00000000000000000000000000000000
83
84.text
85
86
87#define STACK_OFFSET 8*3
88
89#define AadHash 16*0
90#define AadLen 16*1
91#define InLen (16*1)+8
92#define PBlockEncKey 16*2
93#define OrigIV 16*3
94#define CurCount 16*4
95#define PBlockLen 16*5
96#define HashKey 16*6 // store HashKey <<1 mod poly here
97#define HashKey_2 16*7 // store HashKey^2 <<1 mod poly here
98#define HashKey_3 16*8 // store HashKey^3 <<1 mod poly here
99#define HashKey_4 16*9 // store HashKey^4 <<1 mod poly here
100#define HashKey_k 16*10 // store XOR of High 64 bits and Low 64
101 // bits of HashKey <<1 mod poly here
102 //(for Karatsuba purposes)
103#define HashKey_2_k 16*11 // store XOR of High 64 bits and Low 64
104 // bits of HashKey^2 <<1 mod poly here
105 // (for Karatsuba purposes)
106#define HashKey_3_k 16*12 // store XOR of High 64 bits and Low 64
107 // bits of HashKey^3 <<1 mod poly here
108 // (for Karatsuba purposes)
109#define HashKey_4_k 16*13 // store XOR of High 64 bits and Low 64
110 // bits of HashKey^4 <<1 mod poly here
111 // (for Karatsuba purposes)
112
113#define arg1 rdi
114#define arg2 rsi
115#define arg3 rdx
116#define arg4 rcx
117#define arg5 r8
118#define arg6 r9
119#define arg7 STACK_OFFSET+8(%rsp)
120#define arg8 STACK_OFFSET+16(%rsp)
121#define arg9 STACK_OFFSET+24(%rsp)
122#define arg10 STACK_OFFSET+32(%rsp)
123#define arg11 STACK_OFFSET+40(%rsp)
124#define keysize 2*15*16(%arg1)
125#endif
126
127
128#define STATE1 %xmm0
129#define STATE2 %xmm4
130#define STATE3 %xmm5
131#define STATE4 %xmm6
132#define STATE STATE1
133#define IN1 %xmm1
134#define IN2 %xmm7
135#define IN3 %xmm8
136#define IN4 %xmm9
137#define IN IN1
138#define KEY %xmm2
139#define IV %xmm3
140
141#define BSWAP_MASK %xmm10
142#define CTR %xmm11
143#define INC %xmm12
144
145#define GF128MUL_MASK %xmm7
146
147#ifdef __x86_64__
148#define AREG %rax
149#define KEYP %rdi
150#define OUTP %rsi
151#define UKEYP OUTP
152#define INP %rdx
153#define LEN %rcx
154#define IVP %r8
155#define KLEN %r9d
156#define T1 %r10
157#define TKEYP T1
158#define T2 %r11
159#define TCTR_LOW T2
160#else
161#define AREG %eax
162#define KEYP %edi
163#define OUTP AREG
164#define UKEYP OUTP
165#define INP %edx
166#define LEN %esi
167#define IVP %ebp
168#define KLEN %ebx
169#define T1 %ecx
170#define TKEYP T1
171#endif
172
173.macro FUNC_SAVE
174 push %r12
175 push %r13
176 push %r14
177#
178# states of %xmm registers %xmm6:%xmm15 not saved
179# all %xmm registers are clobbered
180#
181.endm
182
183
184.macro FUNC_RESTORE
185 pop %r14
186 pop %r13
187 pop %r12
188.endm
189
190# Precompute hashkeys.
191# Input: Hash subkey.
192# Output: HashKeys stored in gcm_context_data. Only needs to be called
193# once per key.
194# clobbers r12, and tmp xmm registers.
195.macro PRECOMPUTE SUBKEY TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 TMP7
196 mov \SUBKEY, %r12
197 movdqu (%r12), \TMP3
198 movdqa SHUF_MASK(%rip), \TMP2
199 pshufb \TMP2, \TMP3
200
201 # precompute HashKey<<1 mod poly from the HashKey (required for GHASH)
202
203 movdqa \TMP3, \TMP2
204 psllq $1, \TMP3
205 psrlq $63, \TMP2
206 movdqa \TMP2, \TMP1
207 pslldq $8, \TMP2
208 psrldq $8, \TMP1
209 por \TMP2, \TMP3
210
211 # reduce HashKey<<1
212
213 pshufd $0x24, \TMP1, \TMP2
214 pcmpeqd TWOONE(%rip), \TMP2
215 pand POLY(%rip), \TMP2
216 pxor \TMP2, \TMP3
217 movdqu \TMP3, HashKey(%arg2)
218
219 movdqa \TMP3, \TMP5
220 pshufd $78, \TMP3, \TMP1
221 pxor \TMP3, \TMP1
222 movdqu \TMP1, HashKey_k(%arg2)
223
224 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
225# TMP5 = HashKey^2<<1 (mod poly)
226 movdqu \TMP5, HashKey_2(%arg2)
227# HashKey_2 = HashKey^2<<1 (mod poly)
228 pshufd $78, \TMP5, \TMP1
229 pxor \TMP5, \TMP1
230 movdqu \TMP1, HashKey_2_k(%arg2)
231
232 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
233# TMP5 = HashKey^3<<1 (mod poly)
234 movdqu \TMP5, HashKey_3(%arg2)
235 pshufd $78, \TMP5, \TMP1
236 pxor \TMP5, \TMP1
237 movdqu \TMP1, HashKey_3_k(%arg2)
238
239 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
240# TMP5 = HashKey^3<<1 (mod poly)
241 movdqu \TMP5, HashKey_4(%arg2)
242 pshufd $78, \TMP5, \TMP1
243 pxor \TMP5, \TMP1
244 movdqu \TMP1, HashKey_4_k(%arg2)
245.endm
246
247# GCM_INIT initializes a gcm_context struct to prepare for encoding/decoding.
248# Clobbers rax, r10-r13 and xmm0-xmm6, %xmm13
249.macro GCM_INIT Iv SUBKEY AAD AADLEN
250 mov \AADLEN, %r11
251 mov %r11, AadLen(%arg2) # ctx_data.aad_length = aad_length
252 xor %r11d, %r11d
253 mov %r11, InLen(%arg2) # ctx_data.in_length = 0
254 mov %r11, PBlockLen(%arg2) # ctx_data.partial_block_length = 0
255 mov %r11, PBlockEncKey(%arg2) # ctx_data.partial_block_enc_key = 0
256 mov \Iv, %rax
257 movdqu (%rax), %xmm0
258 movdqu %xmm0, OrigIV(%arg2) # ctx_data.orig_IV = iv
259
260 movdqa SHUF_MASK(%rip), %xmm2
261 pshufb %xmm2, %xmm0
262 movdqu %xmm0, CurCount(%arg2) # ctx_data.current_counter = iv
263
264 PRECOMPUTE \SUBKEY, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7
265 movdqu HashKey(%arg2), %xmm13
266
267 CALC_AAD_HASH %xmm13, \AAD, \AADLEN, %xmm0, %xmm1, %xmm2, %xmm3, \
268 %xmm4, %xmm5, %xmm6
269.endm
270
271# GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context
272# struct has been initialized by GCM_INIT.
273# Requires the input data be at least 1 byte long because of READ_PARTIAL_BLOCK
274# Clobbers rax, r10-r13, and xmm0-xmm15
275.macro GCM_ENC_DEC operation
276 movdqu AadHash(%arg2), %xmm8
277 movdqu HashKey(%arg2), %xmm13
278 add %arg5, InLen(%arg2)
279
280 xor %r11d, %r11d # initialise the data pointer offset as zero
281 PARTIAL_BLOCK %arg3 %arg4 %arg5 %r11 %xmm8 \operation
282
283 sub %r11, %arg5 # sub partial block data used
284 mov %arg5, %r13 # save the number of bytes
285
286 and $-16, %r13 # %r13 = %r13 - (%r13 mod 16)
287 mov %r13, %r12
288 # Encrypt/Decrypt first few blocks
289
290 and $(3<<4), %r12
291 jz .L_initial_num_blocks_is_0_\@
292 cmp $(2<<4), %r12
293 jb .L_initial_num_blocks_is_1_\@
294 je .L_initial_num_blocks_is_2_\@
295.L_initial_num_blocks_is_3_\@:
296 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
297%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, \operation
298 sub $48, %r13
299 jmp .L_initial_blocks_\@
300.L_initial_num_blocks_is_2_\@:
301 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
302%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, \operation
303 sub $32, %r13
304 jmp .L_initial_blocks_\@
305.L_initial_num_blocks_is_1_\@:
306 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
307%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, \operation
308 sub $16, %r13
309 jmp .L_initial_blocks_\@
310.L_initial_num_blocks_is_0_\@:
311 INITIAL_BLOCKS_ENC_DEC %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
312%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, \operation
313.L_initial_blocks_\@:
314
315 # Main loop - Encrypt/Decrypt remaining blocks
316
317 test %r13, %r13
318 je .L_zero_cipher_left_\@
319 sub $64, %r13
320 je .L_four_cipher_left_\@
321.L_crypt_by_4_\@:
322 GHASH_4_ENCRYPT_4_PARALLEL_\operation %xmm9, %xmm10, %xmm11, %xmm12, \
323 %xmm13, %xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, \
324 %xmm7, %xmm8, enc
325 add $64, %r11
326 sub $64, %r13
327 jne .L_crypt_by_4_\@
328.L_four_cipher_left_\@:
329 GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
330%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
331.L_zero_cipher_left_\@:
332 movdqu %xmm8, AadHash(%arg2)
333 movdqu %xmm0, CurCount(%arg2)
334
335 mov %arg5, %r13
336 and $15, %r13 # %r13 = arg5 (mod 16)
337 je .L_multiple_of_16_bytes_\@
338
339 mov %r13, PBlockLen(%arg2)
340
341 # Handle the last <16 Byte block separately
342 paddd ONE(%rip), %xmm0 # INCR CNT to get Yn
343 movdqu %xmm0, CurCount(%arg2)
344 movdqa SHUF_MASK(%rip), %xmm10
345 pshufb %xmm10, %xmm0
346
347 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn)
348 movdqu %xmm0, PBlockEncKey(%arg2)
349
350 cmp $16, %arg5
351 jge .L_large_enough_update_\@
352
353 lea (%arg4,%r11,1), %r10
354 mov %r13, %r12
355 READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1
356 jmp .L_data_read_\@
357
358.L_large_enough_update_\@:
359 sub $16, %r11
360 add %r13, %r11
361
362 # receive the last <16 Byte block
363 movdqu (%arg4, %r11, 1), %xmm1
364
365 sub %r13, %r11
366 add $16, %r11
367
368 lea SHIFT_MASK+16(%rip), %r12
369 # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
370 # (r13 is the number of bytes in plaintext mod 16)
371 sub %r13, %r12
372 # get the appropriate shuffle mask
373 movdqu (%r12), %xmm2
374 # shift right 16-r13 bytes
375 pshufb %xmm2, %xmm1
376
377.L_data_read_\@:
378 lea ALL_F+16(%rip), %r12
379 sub %r13, %r12
380
381.ifc \operation, dec
382 movdqa %xmm1, %xmm2
383.endif
384 pxor %xmm1, %xmm0 # XOR Encrypt(K, Yn)
385 movdqu (%r12), %xmm1
386 # get the appropriate mask to mask out top 16-r13 bytes of xmm0
387 pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0
388.ifc \operation, dec
389 pand %xmm1, %xmm2
390 movdqa SHUF_MASK(%rip), %xmm10
391 pshufb %xmm10 ,%xmm2
392
393 pxor %xmm2, %xmm8
394.else
395 movdqa SHUF_MASK(%rip), %xmm10
396 pshufb %xmm10,%xmm0
397
398 pxor %xmm0, %xmm8
399.endif
400
401 movdqu %xmm8, AadHash(%arg2)
402.ifc \operation, enc
403 # GHASH computation for the last <16 byte block
404 movdqa SHUF_MASK(%rip), %xmm10
405 # shuffle xmm0 back to output as ciphertext
406 pshufb %xmm10, %xmm0
407.endif
408
409 # Output %r13 bytes
410 movq %xmm0, %rax
411 cmp $8, %r13
412 jle .L_less_than_8_bytes_left_\@
413 mov %rax, (%arg3 , %r11, 1)
414 add $8, %r11
415 psrldq $8, %xmm0
416 movq %xmm0, %rax
417 sub $8, %r13
418.L_less_than_8_bytes_left_\@:
419 mov %al, (%arg3, %r11, 1)
420 add $1, %r11
421 shr $8, %rax
422 sub $1, %r13
423 jne .L_less_than_8_bytes_left_\@
424.L_multiple_of_16_bytes_\@:
425.endm
426
427# GCM_COMPLETE Finishes update of tag of last partial block
428# Output: Authorization Tag (AUTH_TAG)
429# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
430.macro GCM_COMPLETE AUTHTAG AUTHTAGLEN
431 movdqu AadHash(%arg2), %xmm8
432 movdqu HashKey(%arg2), %xmm13
433
434 mov PBlockLen(%arg2), %r12
435
436 test %r12, %r12
437 je .L_partial_done\@
438
439 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
440
441.L_partial_done\@:
442 mov AadLen(%arg2), %r12 # %r13 = aadLen (number of bytes)
443 shl $3, %r12 # convert into number of bits
444 movd %r12d, %xmm15 # len(A) in %xmm15
445 mov InLen(%arg2), %r12
446 shl $3, %r12 # len(C) in bits (*128)
447 movq %r12, %xmm1
448
449 pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
450 pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
451 pxor %xmm15, %xmm8
452 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
453 # final GHASH computation
454 movdqa SHUF_MASK(%rip), %xmm10
455 pshufb %xmm10, %xmm8
456
457 movdqu OrigIV(%arg2), %xmm0 # %xmm0 = Y0
458 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
459 pxor %xmm8, %xmm0
460.L_return_T_\@:
461 mov \AUTHTAG, %r10 # %r10 = authTag
462 mov \AUTHTAGLEN, %r11 # %r11 = auth_tag_len
463 cmp $16, %r11
464 je .L_T_16_\@
465 cmp $8, %r11
466 jl .L_T_4_\@
467.L_T_8_\@:
468 movq %xmm0, %rax
469 mov %rax, (%r10)
470 add $8, %r10
471 sub $8, %r11
472 psrldq $8, %xmm0
473 test %r11, %r11
474 je .L_return_T_done_\@
475.L_T_4_\@:
476 movd %xmm0, %eax
477 mov %eax, (%r10)
478 add $4, %r10
479 sub $4, %r11
480 psrldq $4, %xmm0
481 test %r11, %r11
482 je .L_return_T_done_\@
483.L_T_123_\@:
484 movd %xmm0, %eax
485 cmp $2, %r11
486 jl .L_T_1_\@
487 mov %ax, (%r10)
488 cmp $2, %r11
489 je .L_return_T_done_\@
490 add $2, %r10
491 sar $16, %eax
492.L_T_1_\@:
493 mov %al, (%r10)
494 jmp .L_return_T_done_\@
495.L_T_16_\@:
496 movdqu %xmm0, (%r10)
497.L_return_T_done_\@:
498.endm
499
500#ifdef __x86_64__
501/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
502*
503*
504* Input: A and B (128-bits each, bit-reflected)
505* Output: C = A*B*x mod poly, (i.e. >>1 )
506* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
507* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
508*
509*/
510.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5
511 movdqa \GH, \TMP1
512 pshufd $78, \GH, \TMP2
513 pshufd $78, \HK, \TMP3
514 pxor \GH, \TMP2 # TMP2 = a1+a0
515 pxor \HK, \TMP3 # TMP3 = b1+b0
516 pclmulqdq $0x11, \HK, \TMP1 # TMP1 = a1*b1
517 pclmulqdq $0x00, \HK, \GH # GH = a0*b0
518 pclmulqdq $0x00, \TMP3, \TMP2 # TMP2 = (a0+a1)*(b1+b0)
519 pxor \GH, \TMP2
520 pxor \TMP1, \TMP2 # TMP2 = (a0*b0)+(a1*b0)
521 movdqa \TMP2, \TMP3
522 pslldq $8, \TMP3 # left shift TMP3 2 DWs
523 psrldq $8, \TMP2 # right shift TMP2 2 DWs
524 pxor \TMP3, \GH
525 pxor \TMP2, \TMP1 # TMP2:GH holds the result of GH*HK
526
527 # first phase of the reduction
528
529 movdqa \GH, \TMP2
530 movdqa \GH, \TMP3
531 movdqa \GH, \TMP4 # copy GH into TMP2,TMP3 and TMP4
532 # in in order to perform
533 # independent shifts
534 pslld $31, \TMP2 # packed right shift <<31
535 pslld $30, \TMP3 # packed right shift <<30
536 pslld $25, \TMP4 # packed right shift <<25
537 pxor \TMP3, \TMP2 # xor the shifted versions
538 pxor \TMP4, \TMP2
539 movdqa \TMP2, \TMP5
540 psrldq $4, \TMP5 # right shift TMP5 1 DW
541 pslldq $12, \TMP2 # left shift TMP2 3 DWs
542 pxor \TMP2, \GH
543
544 # second phase of the reduction
545
546 movdqa \GH,\TMP2 # copy GH into TMP2,TMP3 and TMP4
547 # in in order to perform
548 # independent shifts
549 movdqa \GH,\TMP3
550 movdqa \GH,\TMP4
551 psrld $1,\TMP2 # packed left shift >>1
552 psrld $2,\TMP3 # packed left shift >>2
553 psrld $7,\TMP4 # packed left shift >>7
554 pxor \TMP3,\TMP2 # xor the shifted versions
555 pxor \TMP4,\TMP2
556 pxor \TMP5, \TMP2
557 pxor \TMP2, \GH
558 pxor \TMP1, \GH # result is in TMP1
559.endm
560
561# Reads DLEN bytes starting at DPTR and stores in XMMDst
562# where 0 < DLEN < 16
563# Clobbers %rax, DLEN and XMM1
564.macro READ_PARTIAL_BLOCK DPTR DLEN XMM1 XMMDst
565 cmp $8, \DLEN
566 jl .L_read_lt8_\@
567 mov (\DPTR), %rax
568 movq %rax, \XMMDst
569 sub $8, \DLEN
570 jz .L_done_read_partial_block_\@
571 xor %eax, %eax
572.L_read_next_byte_\@:
573 shl $8, %rax
574 mov 7(\DPTR, \DLEN, 1), %al
575 dec \DLEN
576 jnz .L_read_next_byte_\@
577 movq %rax, \XMM1
578 pslldq $8, \XMM1
579 por \XMM1, \XMMDst
580 jmp .L_done_read_partial_block_\@
581.L_read_lt8_\@:
582 xor %eax, %eax
583.L_read_next_byte_lt8_\@:
584 shl $8, %rax
585 mov -1(\DPTR, \DLEN, 1), %al
586 dec \DLEN
587 jnz .L_read_next_byte_lt8_\@
588 movq %rax, \XMMDst
589.L_done_read_partial_block_\@:
590.endm
591
592# CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted.
593# clobbers r10-11, xmm14
594.macro CALC_AAD_HASH HASHKEY AAD AADLEN TMP1 TMP2 TMP3 TMP4 TMP5 \
595 TMP6 TMP7
596 MOVADQ SHUF_MASK(%rip), %xmm14
597 mov \AAD, %r10 # %r10 = AAD
598 mov \AADLEN, %r11 # %r11 = aadLen
599 pxor \TMP7, \TMP7
600 pxor \TMP6, \TMP6
601
602 cmp $16, %r11
603 jl .L_get_AAD_rest\@
604.L_get_AAD_blocks\@:
605 movdqu (%r10), \TMP7
606 pshufb %xmm14, \TMP7 # byte-reflect the AAD data
607 pxor \TMP7, \TMP6
608 GHASH_MUL \TMP6, \HASHKEY, \TMP1, \TMP2, \TMP3, \TMP4, \TMP5
609 add $16, %r10
610 sub $16, %r11
611 cmp $16, %r11
612 jge .L_get_AAD_blocks\@
613
614 movdqu \TMP6, \TMP7
615
616 /* read the last <16B of AAD */
617.L_get_AAD_rest\@:
618 test %r11, %r11
619 je .L_get_AAD_done\@
620
621 READ_PARTIAL_BLOCK %r10, %r11, \TMP1, \TMP7
622 pshufb %xmm14, \TMP7 # byte-reflect the AAD data
623 pxor \TMP6, \TMP7
624 GHASH_MUL \TMP7, \HASHKEY, \TMP1, \TMP2, \TMP3, \TMP4, \TMP5
625 movdqu \TMP7, \TMP6
626
627.L_get_AAD_done\@:
628 movdqu \TMP6, AadHash(%arg2)
629.endm
630
631# PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks
632# between update calls.
633# Requires the input data be at least 1 byte long due to READ_PARTIAL_BLOCK
634# Outputs encrypted bytes, and updates hash and partial info in gcm_data_context
635# Clobbers rax, r10, r12, r13, xmm0-6, xmm9-13
636.macro PARTIAL_BLOCK CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \
637 AAD_HASH operation
638 mov PBlockLen(%arg2), %r13
639 test %r13, %r13
640 je .L_partial_block_done_\@ # Leave Macro if no partial blocks
641 # Read in input data without over reading
642 cmp $16, \PLAIN_CYPH_LEN
643 jl .L_fewer_than_16_bytes_\@
644 movups (\PLAIN_CYPH_IN), %xmm1 # If more than 16 bytes, just fill xmm
645 jmp .L_data_read_\@
646
647.L_fewer_than_16_bytes_\@:
648 lea (\PLAIN_CYPH_IN, \DATA_OFFSET, 1), %r10
649 mov \PLAIN_CYPH_LEN, %r12
650 READ_PARTIAL_BLOCK %r10 %r12 %xmm0 %xmm1
651
652 mov PBlockLen(%arg2), %r13
653
654.L_data_read_\@: # Finished reading in data
655
656 movdqu PBlockEncKey(%arg2), %xmm9
657 movdqu HashKey(%arg2), %xmm13
658
659 lea SHIFT_MASK(%rip), %r12
660
661 # adjust the shuffle mask pointer to be able to shift r13 bytes
662 # r16-r13 is the number of bytes in plaintext mod 16)
663 add %r13, %r12
664 movdqu (%r12), %xmm2 # get the appropriate shuffle mask
665 pshufb %xmm2, %xmm9 # shift right r13 bytes
666
667.ifc \operation, dec
668 movdqa %xmm1, %xmm3
669 pxor %xmm1, %xmm9 # Ciphertext XOR E(K, Yn)
670
671 mov \PLAIN_CYPH_LEN, %r10
672 add %r13, %r10
673 # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling
674 sub $16, %r10
675 # Determine if partial block is not being filled and
676 # shift mask accordingly
677 jge .L_no_extra_mask_1_\@
678 sub %r10, %r12
679.L_no_extra_mask_1_\@:
680
681 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
682 # get the appropriate mask to mask out bottom r13 bytes of xmm9
683 pand %xmm1, %xmm9 # mask out bottom r13 bytes of xmm9
684
685 pand %xmm1, %xmm3
686 movdqa SHUF_MASK(%rip), %xmm10
687 pshufb %xmm10, %xmm3
688 pshufb %xmm2, %xmm3
689 pxor %xmm3, \AAD_HASH
690
691 test %r10, %r10
692 jl .L_partial_incomplete_1_\@
693
694 # GHASH computation for the last <16 Byte block
695 GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
696 xor %eax, %eax
697
698 mov %rax, PBlockLen(%arg2)
699 jmp .L_dec_done_\@
700.L_partial_incomplete_1_\@:
701 add \PLAIN_CYPH_LEN, PBlockLen(%arg2)
702.L_dec_done_\@:
703 movdqu \AAD_HASH, AadHash(%arg2)
704.else
705 pxor %xmm1, %xmm9 # Plaintext XOR E(K, Yn)
706
707 mov \PLAIN_CYPH_LEN, %r10
708 add %r13, %r10
709 # Set r10 to be the amount of data left in CYPH_PLAIN_IN after filling
710 sub $16, %r10
711 # Determine if partial block is not being filled and
712 # shift mask accordingly
713 jge .L_no_extra_mask_2_\@
714 sub %r10, %r12
715.L_no_extra_mask_2_\@:
716
717 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
718 # get the appropriate mask to mask out bottom r13 bytes of xmm9
719 pand %xmm1, %xmm9
720
721 movdqa SHUF_MASK(%rip), %xmm1
722 pshufb %xmm1, %xmm9
723 pshufb %xmm2, %xmm9
724 pxor %xmm9, \AAD_HASH
725
726 test %r10, %r10
727 jl .L_partial_incomplete_2_\@
728
729 # GHASH computation for the last <16 Byte block
730 GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
731 xor %eax, %eax
732
733 mov %rax, PBlockLen(%arg2)
734 jmp .L_encode_done_\@
735.L_partial_incomplete_2_\@:
736 add \PLAIN_CYPH_LEN, PBlockLen(%arg2)
737.L_encode_done_\@:
738 movdqu \AAD_HASH, AadHash(%arg2)
739
740 movdqa SHUF_MASK(%rip), %xmm10
741 # shuffle xmm9 back to output as ciphertext
742 pshufb %xmm10, %xmm9
743 pshufb %xmm2, %xmm9
744.endif
745 # output encrypted Bytes
746 test %r10, %r10
747 jl .L_partial_fill_\@
748 mov %r13, %r12
749 mov $16, %r13
750 # Set r13 to be the number of bytes to write out
751 sub %r12, %r13
752 jmp .L_count_set_\@
753.L_partial_fill_\@:
754 mov \PLAIN_CYPH_LEN, %r13
755.L_count_set_\@:
756 movdqa %xmm9, %xmm0
757 movq %xmm0, %rax
758 cmp $8, %r13
759 jle .L_less_than_8_bytes_left_\@
760
761 mov %rax, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1)
762 add $8, \DATA_OFFSET
763 psrldq $8, %xmm0
764 movq %xmm0, %rax
765 sub $8, %r13
766.L_less_than_8_bytes_left_\@:
767 movb %al, (\CYPH_PLAIN_OUT, \DATA_OFFSET, 1)
768 add $1, \DATA_OFFSET
769 shr $8, %rax
770 sub $1, %r13
771 jne .L_less_than_8_bytes_left_\@
772.L_partial_block_done_\@:
773.endm # PARTIAL_BLOCK
774
775/*
776* if a = number of total plaintext bytes
777* b = floor(a/16)
778* num_initial_blocks = b mod 4
779* encrypt the initial num_initial_blocks blocks and apply ghash on
780* the ciphertext
781* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
782* are clobbered
783* arg1, %arg2, %arg3 are used as a pointer only, not modified
784*/
785
786
787.macro INITIAL_BLOCKS_ENC_DEC TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
788 XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
789 MOVADQ SHUF_MASK(%rip), %xmm14
790
791 movdqu AadHash(%arg2), %xmm\i # XMM0 = Y0
792
793 # start AES for num_initial_blocks blocks
794
795 movdqu CurCount(%arg2), \XMM0 # XMM0 = Y0
796
797.if (\i == 5) || (\i == 6) || (\i == 7)
798
799 MOVADQ ONE(%RIP),\TMP1
800 MOVADQ 0(%arg1),\TMP2
801.irpc index, \i_seq
802 paddd \TMP1, \XMM0 # INCR Y0
803.ifc \operation, dec
804 movdqa \XMM0, %xmm\index
805.else
806 MOVADQ \XMM0, %xmm\index
807.endif
808 pshufb %xmm14, %xmm\index # perform a 16 byte swap
809 pxor \TMP2, %xmm\index
810.endr
811 lea 0x10(%arg1),%r10
812 mov keysize,%eax
813 shr $2,%eax # 128->4, 192->6, 256->8
814 add $5,%eax # 128->9, 192->11, 256->13
815
816.Laes_loop_initial_\@:
817 MOVADQ (%r10),\TMP1
818.irpc index, \i_seq
819 aesenc \TMP1, %xmm\index
820.endr
821 add $16,%r10
822 sub $1,%eax
823 jnz .Laes_loop_initial_\@
824
825 MOVADQ (%r10), \TMP1
826.irpc index, \i_seq
827 aesenclast \TMP1, %xmm\index # Last Round
828.endr
829.irpc index, \i_seq
830 movdqu (%arg4 , %r11, 1), \TMP1
831 pxor \TMP1, %xmm\index
832 movdqu %xmm\index, (%arg3 , %r11, 1)
833 # write back plaintext/ciphertext for num_initial_blocks
834 add $16, %r11
835
836.ifc \operation, dec
837 movdqa \TMP1, %xmm\index
838.endif
839 pshufb %xmm14, %xmm\index
840
841 # prepare plaintext/ciphertext for GHASH computation
842.endr
843.endif
844
845 # apply GHASH on num_initial_blocks blocks
846
847.if \i == 5
848 pxor %xmm5, %xmm6
849 GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
850 pxor %xmm6, %xmm7
851 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
852 pxor %xmm7, %xmm8
853 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
854.elseif \i == 6
855 pxor %xmm6, %xmm7
856 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
857 pxor %xmm7, %xmm8
858 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
859.elseif \i == 7
860 pxor %xmm7, %xmm8
861 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
862.endif
863 cmp $64, %r13
864 jl .L_initial_blocks_done\@
865 # no need for precomputed values
866/*
867*
868* Precomputations for HashKey parallel with encryption of first 4 blocks.
869* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
870*/
871 MOVADQ ONE(%RIP),\TMP1
872 paddd \TMP1, \XMM0 # INCR Y0
873 MOVADQ \XMM0, \XMM1
874 pshufb %xmm14, \XMM1 # perform a 16 byte swap
875
876 paddd \TMP1, \XMM0 # INCR Y0
877 MOVADQ \XMM0, \XMM2
878 pshufb %xmm14, \XMM2 # perform a 16 byte swap
879
880 paddd \TMP1, \XMM0 # INCR Y0
881 MOVADQ \XMM0, \XMM3
882 pshufb %xmm14, \XMM3 # perform a 16 byte swap
883
884 paddd \TMP1, \XMM0 # INCR Y0
885 MOVADQ \XMM0, \XMM4
886 pshufb %xmm14, \XMM4 # perform a 16 byte swap
887
888 MOVADQ 0(%arg1),\TMP1
889 pxor \TMP1, \XMM1
890 pxor \TMP1, \XMM2
891 pxor \TMP1, \XMM3
892 pxor \TMP1, \XMM4
893.irpc index, 1234 # do 4 rounds
894 movaps 0x10*\index(%arg1), \TMP1
895 aesenc \TMP1, \XMM1
896 aesenc \TMP1, \XMM2
897 aesenc \TMP1, \XMM3
898 aesenc \TMP1, \XMM4
899.endr
900.irpc index, 56789 # do next 5 rounds
901 movaps 0x10*\index(%arg1), \TMP1
902 aesenc \TMP1, \XMM1
903 aesenc \TMP1, \XMM2
904 aesenc \TMP1, \XMM3
905 aesenc \TMP1, \XMM4
906.endr
907 lea 0xa0(%arg1),%r10
908 mov keysize,%eax
909 shr $2,%eax # 128->4, 192->6, 256->8
910 sub $4,%eax # 128->0, 192->2, 256->4
911 jz .Laes_loop_pre_done\@
912
913.Laes_loop_pre_\@:
914 MOVADQ (%r10),\TMP2
915.irpc index, 1234
916 aesenc \TMP2, %xmm\index
917.endr
918 add $16,%r10
919 sub $1,%eax
920 jnz .Laes_loop_pre_\@
921
922.Laes_loop_pre_done\@:
923 MOVADQ (%r10), \TMP2
924 aesenclast \TMP2, \XMM1
925 aesenclast \TMP2, \XMM2
926 aesenclast \TMP2, \XMM3
927 aesenclast \TMP2, \XMM4
928 movdqu 16*0(%arg4 , %r11 , 1), \TMP1
929 pxor \TMP1, \XMM1
930.ifc \operation, dec
931 movdqu \XMM1, 16*0(%arg3 , %r11 , 1)
932 movdqa \TMP1, \XMM1
933.endif
934 movdqu 16*1(%arg4 , %r11 , 1), \TMP1
935 pxor \TMP1, \XMM2
936.ifc \operation, dec
937 movdqu \XMM2, 16*1(%arg3 , %r11 , 1)
938 movdqa \TMP1, \XMM2
939.endif
940 movdqu 16*2(%arg4 , %r11 , 1), \TMP1
941 pxor \TMP1, \XMM3
942.ifc \operation, dec
943 movdqu \XMM3, 16*2(%arg3 , %r11 , 1)
944 movdqa \TMP1, \XMM3
945.endif
946 movdqu 16*3(%arg4 , %r11 , 1), \TMP1
947 pxor \TMP1, \XMM4
948.ifc \operation, dec
949 movdqu \XMM4, 16*3(%arg3 , %r11 , 1)
950 movdqa \TMP1, \XMM4
951.else
952 movdqu \XMM1, 16*0(%arg3 , %r11 , 1)
953 movdqu \XMM2, 16*1(%arg3 , %r11 , 1)
954 movdqu \XMM3, 16*2(%arg3 , %r11 , 1)
955 movdqu \XMM4, 16*3(%arg3 , %r11 , 1)
956.endif
957
958 add $64, %r11
959 pshufb %xmm14, \XMM1 # perform a 16 byte swap
960 pxor \XMMDst, \XMM1
961# combine GHASHed value with the corresponding ciphertext
962 pshufb %xmm14, \XMM2 # perform a 16 byte swap
963 pshufb %xmm14, \XMM3 # perform a 16 byte swap
964 pshufb %xmm14, \XMM4 # perform a 16 byte swap
965
966.L_initial_blocks_done\@:
967
968.endm
969
970/*
971* encrypt 4 blocks at a time
972* ghash the 4 previously encrypted ciphertext blocks
973* arg1, %arg3, %arg4 are used as pointers only, not modified
974* %r11 is the data offset value
975*/
976.macro GHASH_4_ENCRYPT_4_PARALLEL_enc TMP1 TMP2 TMP3 TMP4 TMP5 \
977TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
978
979 movdqa \XMM1, \XMM5
980 movdqa \XMM2, \XMM6
981 movdqa \XMM3, \XMM7
982 movdqa \XMM4, \XMM8
983
984 movdqa SHUF_MASK(%rip), %xmm15
985 # multiply TMP5 * HashKey using karatsuba
986
987 movdqa \XMM5, \TMP4
988 pshufd $78, \XMM5, \TMP6
989 pxor \XMM5, \TMP6
990 paddd ONE(%rip), \XMM0 # INCR CNT
991 movdqu HashKey_4(%arg2), \TMP5
992 pclmulqdq $0x11, \TMP5, \TMP4 # TMP4 = a1*b1
993 movdqa \XMM0, \XMM1
994 paddd ONE(%rip), \XMM0 # INCR CNT
995 movdqa \XMM0, \XMM2
996 paddd ONE(%rip), \XMM0 # INCR CNT
997 movdqa \XMM0, \XMM3
998 paddd ONE(%rip), \XMM0 # INCR CNT
999 movdqa \XMM0, \XMM4
1000 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1001 pclmulqdq $0x00, \TMP5, \XMM5 # XMM5 = a0*b0
1002 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1003 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1004 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1005
1006 pxor (%arg1), \XMM1
1007 pxor (%arg1), \XMM2
1008 pxor (%arg1), \XMM3
1009 pxor (%arg1), \XMM4
1010 movdqu HashKey_4_k(%arg2), \TMP5
1011 pclmulqdq $0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0)
1012 movaps 0x10(%arg1), \TMP1
1013 aesenc \TMP1, \XMM1 # Round 1
1014 aesenc \TMP1, \XMM2
1015 aesenc \TMP1, \XMM3
1016 aesenc \TMP1, \XMM4
1017 movaps 0x20(%arg1), \TMP1
1018 aesenc \TMP1, \XMM1 # Round 2
1019 aesenc \TMP1, \XMM2
1020 aesenc \TMP1, \XMM3
1021 aesenc \TMP1, \XMM4
1022 movdqa \XMM6, \TMP1
1023 pshufd $78, \XMM6, \TMP2
1024 pxor \XMM6, \TMP2
1025 movdqu HashKey_3(%arg2), \TMP5
1026 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1 * b1
1027 movaps 0x30(%arg1), \TMP3
1028 aesenc \TMP3, \XMM1 # Round 3
1029 aesenc \TMP3, \XMM2
1030 aesenc \TMP3, \XMM3
1031 aesenc \TMP3, \XMM4
1032 pclmulqdq $0x00, \TMP5, \XMM6 # XMM6 = a0*b0
1033 movaps 0x40(%arg1), \TMP3
1034 aesenc \TMP3, \XMM1 # Round 4
1035 aesenc \TMP3, \XMM2
1036 aesenc \TMP3, \XMM3
1037 aesenc \TMP3, \XMM4
1038 movdqu HashKey_3_k(%arg2), \TMP5
1039 pclmulqdq $0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1040 movaps 0x50(%arg1), \TMP3
1041 aesenc \TMP3, \XMM1 # Round 5
1042 aesenc \TMP3, \XMM2
1043 aesenc \TMP3, \XMM3
1044 aesenc \TMP3, \XMM4
1045 pxor \TMP1, \TMP4
1046# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
1047 pxor \XMM6, \XMM5
1048 pxor \TMP2, \TMP6
1049 movdqa \XMM7, \TMP1
1050 pshufd $78, \XMM7, \TMP2
1051 pxor \XMM7, \TMP2
1052 movdqu HashKey_2(%arg2), \TMP5
1053
1054 # Multiply TMP5 * HashKey using karatsuba
1055
1056 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1057 movaps 0x60(%arg1), \TMP3
1058 aesenc \TMP3, \XMM1 # Round 6
1059 aesenc \TMP3, \XMM2
1060 aesenc \TMP3, \XMM3
1061 aesenc \TMP3, \XMM4
1062 pclmulqdq $0x00, \TMP5, \XMM7 # XMM7 = a0*b0
1063 movaps 0x70(%arg1), \TMP3
1064 aesenc \TMP3, \XMM1 # Round 7
1065 aesenc \TMP3, \XMM2
1066 aesenc \TMP3, \XMM3
1067 aesenc \TMP3, \XMM4
1068 movdqu HashKey_2_k(%arg2), \TMP5
1069 pclmulqdq $0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1070 movaps 0x80(%arg1), \TMP3
1071 aesenc \TMP3, \XMM1 # Round 8
1072 aesenc \TMP3, \XMM2
1073 aesenc \TMP3, \XMM3
1074 aesenc \TMP3, \XMM4
1075 pxor \TMP1, \TMP4
1076# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
1077 pxor \XMM7, \XMM5
1078 pxor \TMP2, \TMP6
1079
1080 # Multiply XMM8 * HashKey
1081 # XMM8 and TMP5 hold the values for the two operands
1082
1083 movdqa \XMM8, \TMP1
1084 pshufd $78, \XMM8, \TMP2
1085 pxor \XMM8, \TMP2
1086 movdqu HashKey(%arg2), \TMP5
1087 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1088 movaps 0x90(%arg1), \TMP3
1089 aesenc \TMP3, \XMM1 # Round 9
1090 aesenc \TMP3, \XMM2
1091 aesenc \TMP3, \XMM3
1092 aesenc \TMP3, \XMM4
1093 pclmulqdq $0x00, \TMP5, \XMM8 # XMM8 = a0*b0
1094 lea 0xa0(%arg1),%r10
1095 mov keysize,%eax
1096 shr $2,%eax # 128->4, 192->6, 256->8
1097 sub $4,%eax # 128->0, 192->2, 256->4
1098 jz .Laes_loop_par_enc_done\@
1099
1100.Laes_loop_par_enc\@:
1101 MOVADQ (%r10),\TMP3
1102.irpc index, 1234
1103 aesenc \TMP3, %xmm\index
1104.endr
1105 add $16,%r10
1106 sub $1,%eax
1107 jnz .Laes_loop_par_enc\@
1108
1109.Laes_loop_par_enc_done\@:
1110 MOVADQ (%r10), \TMP3
1111 aesenclast \TMP3, \XMM1 # Round 10
1112 aesenclast \TMP3, \XMM2
1113 aesenclast \TMP3, \XMM3
1114 aesenclast \TMP3, \XMM4
1115 movdqu HashKey_k(%arg2), \TMP5
1116 pclmulqdq $0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1117 movdqu (%arg4,%r11,1), \TMP3
1118 pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK
1119 movdqu 16(%arg4,%r11,1), \TMP3
1120 pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK
1121 movdqu 32(%arg4,%r11,1), \TMP3
1122 pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK
1123 movdqu 48(%arg4,%r11,1), \TMP3
1124 pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK
1125 movdqu \XMM1, (%arg3,%r11,1) # Write to the ciphertext buffer
1126 movdqu \XMM2, 16(%arg3,%r11,1) # Write to the ciphertext buffer
1127 movdqu \XMM3, 32(%arg3,%r11,1) # Write to the ciphertext buffer
1128 movdqu \XMM4, 48(%arg3,%r11,1) # Write to the ciphertext buffer
1129 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1130 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1131 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1132 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1133
1134 pxor \TMP4, \TMP1
1135 pxor \XMM8, \XMM5
1136 pxor \TMP6, \TMP2
1137 pxor \TMP1, \TMP2
1138 pxor \XMM5, \TMP2
1139 movdqa \TMP2, \TMP3
1140 pslldq $8, \TMP3 # left shift TMP3 2 DWs
1141 psrldq $8, \TMP2 # right shift TMP2 2 DWs
1142 pxor \TMP3, \XMM5
1143 pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5
1144
1145 # first phase of reduction
1146
1147 movdqa \XMM5, \TMP2
1148 movdqa \XMM5, \TMP3
1149 movdqa \XMM5, \TMP4
1150# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
1151 pslld $31, \TMP2 # packed right shift << 31
1152 pslld $30, \TMP3 # packed right shift << 30
1153 pslld $25, \TMP4 # packed right shift << 25
1154 pxor \TMP3, \TMP2 # xor the shifted versions
1155 pxor \TMP4, \TMP2
1156 movdqa \TMP2, \TMP5
1157 psrldq $4, \TMP5 # right shift T5 1 DW
1158 pslldq $12, \TMP2 # left shift T2 3 DWs
1159 pxor \TMP2, \XMM5
1160
1161 # second phase of reduction
1162
1163 movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
1164 movdqa \XMM5,\TMP3
1165 movdqa \XMM5,\TMP4
1166 psrld $1, \TMP2 # packed left shift >>1
1167 psrld $2, \TMP3 # packed left shift >>2
1168 psrld $7, \TMP4 # packed left shift >>7
1169 pxor \TMP3,\TMP2 # xor the shifted versions
1170 pxor \TMP4,\TMP2
1171 pxor \TMP5, \TMP2
1172 pxor \TMP2, \XMM5
1173 pxor \TMP1, \XMM5 # result is in TMP1
1174
1175 pxor \XMM5, \XMM1
1176.endm
1177
1178/*
1179* decrypt 4 blocks at a time
1180* ghash the 4 previously decrypted ciphertext blocks
1181* arg1, %arg3, %arg4 are used as pointers only, not modified
1182* %r11 is the data offset value
1183*/
1184.macro GHASH_4_ENCRYPT_4_PARALLEL_dec TMP1 TMP2 TMP3 TMP4 TMP5 \
1185TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
1186
1187 movdqa \XMM1, \XMM5
1188 movdqa \XMM2, \XMM6
1189 movdqa \XMM3, \XMM7
1190 movdqa \XMM4, \XMM8
1191
1192 movdqa SHUF_MASK(%rip), %xmm15
1193 # multiply TMP5 * HashKey using karatsuba
1194
1195 movdqa \XMM5, \TMP4
1196 pshufd $78, \XMM5, \TMP6
1197 pxor \XMM5, \TMP6
1198 paddd ONE(%rip), \XMM0 # INCR CNT
1199 movdqu HashKey_4(%arg2), \TMP5
1200 pclmulqdq $0x11, \TMP5, \TMP4 # TMP4 = a1*b1
1201 movdqa \XMM0, \XMM1
1202 paddd ONE(%rip), \XMM0 # INCR CNT
1203 movdqa \XMM0, \XMM2
1204 paddd ONE(%rip), \XMM0 # INCR CNT
1205 movdqa \XMM0, \XMM3
1206 paddd ONE(%rip), \XMM0 # INCR CNT
1207 movdqa \XMM0, \XMM4
1208 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1209 pclmulqdq $0x00, \TMP5, \XMM5 # XMM5 = a0*b0
1210 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1211 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1212 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1213
1214 pxor (%arg1), \XMM1
1215 pxor (%arg1), \XMM2
1216 pxor (%arg1), \XMM3
1217 pxor (%arg1), \XMM4
1218 movdqu HashKey_4_k(%arg2), \TMP5
1219 pclmulqdq $0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0)
1220 movaps 0x10(%arg1), \TMP1
1221 aesenc \TMP1, \XMM1 # Round 1
1222 aesenc \TMP1, \XMM2
1223 aesenc \TMP1, \XMM3
1224 aesenc \TMP1, \XMM4
1225 movaps 0x20(%arg1), \TMP1
1226 aesenc \TMP1, \XMM1 # Round 2
1227 aesenc \TMP1, \XMM2
1228 aesenc \TMP1, \XMM3
1229 aesenc \TMP1, \XMM4
1230 movdqa \XMM6, \TMP1
1231 pshufd $78, \XMM6, \TMP2
1232 pxor \XMM6, \TMP2
1233 movdqu HashKey_3(%arg2), \TMP5
1234 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1 * b1
1235 movaps 0x30(%arg1), \TMP3
1236 aesenc \TMP3, \XMM1 # Round 3
1237 aesenc \TMP3, \XMM2
1238 aesenc \TMP3, \XMM3
1239 aesenc \TMP3, \XMM4
1240 pclmulqdq $0x00, \TMP5, \XMM6 # XMM6 = a0*b0
1241 movaps 0x40(%arg1), \TMP3
1242 aesenc \TMP3, \XMM1 # Round 4
1243 aesenc \TMP3, \XMM2
1244 aesenc \TMP3, \XMM3
1245 aesenc \TMP3, \XMM4
1246 movdqu HashKey_3_k(%arg2), \TMP5
1247 pclmulqdq $0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1248 movaps 0x50(%arg1), \TMP3
1249 aesenc \TMP3, \XMM1 # Round 5
1250 aesenc \TMP3, \XMM2
1251 aesenc \TMP3, \XMM3
1252 aesenc \TMP3, \XMM4
1253 pxor \TMP1, \TMP4
1254# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
1255 pxor \XMM6, \XMM5
1256 pxor \TMP2, \TMP6
1257 movdqa \XMM7, \TMP1
1258 pshufd $78, \XMM7, \TMP2
1259 pxor \XMM7, \TMP2
1260 movdqu HashKey_2(%arg2), \TMP5
1261
1262 # Multiply TMP5 * HashKey using karatsuba
1263
1264 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1265 movaps 0x60(%arg1), \TMP3
1266 aesenc \TMP3, \XMM1 # Round 6
1267 aesenc \TMP3, \XMM2
1268 aesenc \TMP3, \XMM3
1269 aesenc \TMP3, \XMM4
1270 pclmulqdq $0x00, \TMP5, \XMM7 # XMM7 = a0*b0
1271 movaps 0x70(%arg1), \TMP3
1272 aesenc \TMP3, \XMM1 # Round 7
1273 aesenc \TMP3, \XMM2
1274 aesenc \TMP3, \XMM3
1275 aesenc \TMP3, \XMM4
1276 movdqu HashKey_2_k(%arg2), \TMP5
1277 pclmulqdq $0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1278 movaps 0x80(%arg1), \TMP3
1279 aesenc \TMP3, \XMM1 # Round 8
1280 aesenc \TMP3, \XMM2
1281 aesenc \TMP3, \XMM3
1282 aesenc \TMP3, \XMM4
1283 pxor \TMP1, \TMP4
1284# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
1285 pxor \XMM7, \XMM5
1286 pxor \TMP2, \TMP6
1287
1288 # Multiply XMM8 * HashKey
1289 # XMM8 and TMP5 hold the values for the two operands
1290
1291 movdqa \XMM8, \TMP1
1292 pshufd $78, \XMM8, \TMP2
1293 pxor \XMM8, \TMP2
1294 movdqu HashKey(%arg2), \TMP5
1295 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1296 movaps 0x90(%arg1), \TMP3
1297 aesenc \TMP3, \XMM1 # Round 9
1298 aesenc \TMP3, \XMM2
1299 aesenc \TMP3, \XMM3
1300 aesenc \TMP3, \XMM4
1301 pclmulqdq $0x00, \TMP5, \XMM8 # XMM8 = a0*b0
1302 lea 0xa0(%arg1),%r10
1303 mov keysize,%eax
1304 shr $2,%eax # 128->4, 192->6, 256->8
1305 sub $4,%eax # 128->0, 192->2, 256->4
1306 jz .Laes_loop_par_dec_done\@
1307
1308.Laes_loop_par_dec\@:
1309 MOVADQ (%r10),\TMP3
1310.irpc index, 1234
1311 aesenc \TMP3, %xmm\index
1312.endr
1313 add $16,%r10
1314 sub $1,%eax
1315 jnz .Laes_loop_par_dec\@
1316
1317.Laes_loop_par_dec_done\@:
1318 MOVADQ (%r10), \TMP3
1319 aesenclast \TMP3, \XMM1 # last round
1320 aesenclast \TMP3, \XMM2
1321 aesenclast \TMP3, \XMM3
1322 aesenclast \TMP3, \XMM4
1323 movdqu HashKey_k(%arg2), \TMP5
1324 pclmulqdq $0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1325 movdqu (%arg4,%r11,1), \TMP3
1326 pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK
1327 movdqu \XMM1, (%arg3,%r11,1) # Write to plaintext buffer
1328 movdqa \TMP3, \XMM1
1329 movdqu 16(%arg4,%r11,1), \TMP3
1330 pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK
1331 movdqu \XMM2, 16(%arg3,%r11,1) # Write to plaintext buffer
1332 movdqa \TMP3, \XMM2
1333 movdqu 32(%arg4,%r11,1), \TMP3
1334 pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK
1335 movdqu \XMM3, 32(%arg3,%r11,1) # Write to plaintext buffer
1336 movdqa \TMP3, \XMM3
1337 movdqu 48(%arg4,%r11,1), \TMP3
1338 pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK
1339 movdqu \XMM4, 48(%arg3,%r11,1) # Write to plaintext buffer
1340 movdqa \TMP3, \XMM4
1341 pshufb %xmm15, \XMM1 # perform a 16 byte swap
1342 pshufb %xmm15, \XMM2 # perform a 16 byte swap
1343 pshufb %xmm15, \XMM3 # perform a 16 byte swap
1344 pshufb %xmm15, \XMM4 # perform a 16 byte swap
1345
1346 pxor \TMP4, \TMP1
1347 pxor \XMM8, \XMM5
1348 pxor \TMP6, \TMP2
1349 pxor \TMP1, \TMP2
1350 pxor \XMM5, \TMP2
1351 movdqa \TMP2, \TMP3
1352 pslldq $8, \TMP3 # left shift TMP3 2 DWs
1353 psrldq $8, \TMP2 # right shift TMP2 2 DWs
1354 pxor \TMP3, \XMM5
1355 pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5
1356
1357 # first phase of reduction
1358
1359 movdqa \XMM5, \TMP2
1360 movdqa \XMM5, \TMP3
1361 movdqa \XMM5, \TMP4
1362# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
1363 pslld $31, \TMP2 # packed right shift << 31
1364 pslld $30, \TMP3 # packed right shift << 30
1365 pslld $25, \TMP4 # packed right shift << 25
1366 pxor \TMP3, \TMP2 # xor the shifted versions
1367 pxor \TMP4, \TMP2
1368 movdqa \TMP2, \TMP5
1369 psrldq $4, \TMP5 # right shift T5 1 DW
1370 pslldq $12, \TMP2 # left shift T2 3 DWs
1371 pxor \TMP2, \XMM5
1372
1373 # second phase of reduction
1374
1375 movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
1376 movdqa \XMM5,\TMP3
1377 movdqa \XMM5,\TMP4
1378 psrld $1, \TMP2 # packed left shift >>1
1379 psrld $2, \TMP3 # packed left shift >>2
1380 psrld $7, \TMP4 # packed left shift >>7
1381 pxor \TMP3,\TMP2 # xor the shifted versions
1382 pxor \TMP4,\TMP2
1383 pxor \TMP5, \TMP2
1384 pxor \TMP2, \XMM5
1385 pxor \TMP1, \XMM5 # result is in TMP1
1386
1387 pxor \XMM5, \XMM1
1388.endm
1389
1390/* GHASH the last 4 ciphertext blocks. */
1391.macro GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \
1392TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
1393
1394 # Multiply TMP6 * HashKey (using Karatsuba)
1395
1396 movdqa \XMM1, \TMP6
1397 pshufd $78, \XMM1, \TMP2
1398 pxor \XMM1, \TMP2
1399 movdqu HashKey_4(%arg2), \TMP5
1400 pclmulqdq $0x11, \TMP5, \TMP6 # TMP6 = a1*b1
1401 pclmulqdq $0x00, \TMP5, \XMM1 # XMM1 = a0*b0
1402 movdqu HashKey_4_k(%arg2), \TMP4
1403 pclmulqdq $0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1404 movdqa \XMM1, \XMMDst
1405 movdqa \TMP2, \XMM1 # result in TMP6, XMMDst, XMM1
1406
1407 # Multiply TMP1 * HashKey (using Karatsuba)
1408
1409 movdqa \XMM2, \TMP1
1410 pshufd $78, \XMM2, \TMP2
1411 pxor \XMM2, \TMP2
1412 movdqu HashKey_3(%arg2), \TMP5
1413 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1414 pclmulqdq $0x00, \TMP5, \XMM2 # XMM2 = a0*b0
1415 movdqu HashKey_3_k(%arg2), \TMP4
1416 pclmulqdq $0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1417 pxor \TMP1, \TMP6
1418 pxor \XMM2, \XMMDst
1419 pxor \TMP2, \XMM1
1420# results accumulated in TMP6, XMMDst, XMM1
1421
1422 # Multiply TMP1 * HashKey (using Karatsuba)
1423
1424 movdqa \XMM3, \TMP1
1425 pshufd $78, \XMM3, \TMP2
1426 pxor \XMM3, \TMP2
1427 movdqu HashKey_2(%arg2), \TMP5
1428 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1429 pclmulqdq $0x00, \TMP5, \XMM3 # XMM3 = a0*b0
1430 movdqu HashKey_2_k(%arg2), \TMP4
1431 pclmulqdq $0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1432 pxor \TMP1, \TMP6
1433 pxor \XMM3, \XMMDst
1434 pxor \TMP2, \XMM1 # results accumulated in TMP6, XMMDst, XMM1
1435
1436 # Multiply TMP1 * HashKey (using Karatsuba)
1437 movdqa \XMM4, \TMP1
1438 pshufd $78, \XMM4, \TMP2
1439 pxor \XMM4, \TMP2
1440 movdqu HashKey(%arg2), \TMP5
1441 pclmulqdq $0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1442 pclmulqdq $0x00, \TMP5, \XMM4 # XMM4 = a0*b0
1443 movdqu HashKey_k(%arg2), \TMP4
1444 pclmulqdq $0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1445 pxor \TMP1, \TMP6
1446 pxor \XMM4, \XMMDst
1447 pxor \XMM1, \TMP2
1448 pxor \TMP6, \TMP2
1449 pxor \XMMDst, \TMP2
1450 # middle section of the temp results combined as in karatsuba algorithm
1451 movdqa \TMP2, \TMP4
1452 pslldq $8, \TMP4 # left shift TMP4 2 DWs
1453 psrldq $8, \TMP2 # right shift TMP2 2 DWs
1454 pxor \TMP4, \XMMDst
1455 pxor \TMP2, \TMP6
1456# TMP6:XMMDst holds the result of the accumulated carry-less multiplications
1457 # first phase of the reduction
1458 movdqa \XMMDst, \TMP2
1459 movdqa \XMMDst, \TMP3
1460 movdqa \XMMDst, \TMP4
1461# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently
1462 pslld $31, \TMP2 # packed right shifting << 31
1463 pslld $30, \TMP3 # packed right shifting << 30
1464 pslld $25, \TMP4 # packed right shifting << 25
1465 pxor \TMP3, \TMP2 # xor the shifted versions
1466 pxor \TMP4, \TMP2
1467 movdqa \TMP2, \TMP7
1468 psrldq $4, \TMP7 # right shift TMP7 1 DW
1469 pslldq $12, \TMP2 # left shift TMP2 3 DWs
1470 pxor \TMP2, \XMMDst
1471
1472 # second phase of the reduction
1473 movdqa \XMMDst, \TMP2
1474 # make 3 copies of XMMDst for doing 3 shift operations
1475 movdqa \XMMDst, \TMP3
1476 movdqa \XMMDst, \TMP4
1477 psrld $1, \TMP2 # packed left shift >> 1
1478 psrld $2, \TMP3 # packed left shift >> 2
1479 psrld $7, \TMP4 # packed left shift >> 7
1480 pxor \TMP3, \TMP2 # xor the shifted versions
1481 pxor \TMP4, \TMP2
1482 pxor \TMP7, \TMP2
1483 pxor \TMP2, \XMMDst
1484 pxor \TMP6, \XMMDst # reduced result is in XMMDst
1485.endm
1486
1487
1488/* Encryption of a single block
1489* uses eax & r10
1490*/
1491
1492.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
1493
1494 pxor (%arg1), \XMM0
1495 mov keysize,%eax
1496 shr $2,%eax # 128->4, 192->6, 256->8
1497 add $5,%eax # 128->9, 192->11, 256->13
1498 lea 16(%arg1), %r10 # get first expanded key address
1499
1500_esb_loop_\@:
1501 MOVADQ (%r10),\TMP1
1502 aesenc \TMP1,\XMM0
1503 add $16,%r10
1504 sub $1,%eax
1505 jnz _esb_loop_\@
1506
1507 MOVADQ (%r10),\TMP1
1508 aesenclast \TMP1,\XMM0
1509.endm
1510/*****************************************************************************
1511* void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1512* struct gcm_context_data *data
1513* // Context data
1514* u8 *out, // Plaintext output. Encrypt in-place is allowed.
1515* const u8 *in, // Ciphertext input
1516* u64 plaintext_len, // Length of data in bytes for decryption.
1517* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association)
1518* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1519* // concatenated with 0x00000001. 16-byte aligned pointer.
1520* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary.
1521* const u8 *aad, // Additional Authentication Data (AAD)
1522* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
1523* u8 *auth_tag, // Authenticated Tag output. The driver will compare this to the
1524* // given authentication tag and only return the plaintext if they match.
1525* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16
1526* // (most likely), 12 or 8.
1527*
1528* Assumptions:
1529*
1530* keys:
1531* keys are pre-expanded and aligned to 16 bytes. we are using the first
1532* set of 11 keys in the data structure void *aes_ctx
1533*
1534* iv:
1535* 0 1 2 3
1536* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1537* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1538* | Salt (From the SA) |
1539* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1540* | Initialization Vector |
1541* | (This is the sequence number from IPSec header) |
1542* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1543* | 0x1 |
1544* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1545*
1546*
1547*
1548* AAD:
1549* AAD padded to 128 bits with 0
1550* for example, assume AAD is a u32 vector
1551*
1552* if AAD is 8 bytes:
1553* AAD[3] = {A0, A1};
1554* padded AAD in xmm register = {A1 A0 0 0}
1555*
1556* 0 1 2 3
1557* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1558* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1559* | SPI (A1) |
1560* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1561* | 32-bit Sequence Number (A0) |
1562* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1563* | 0x0 |
1564* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1565*
1566* AAD Format with 32-bit Sequence Number
1567*
1568* if AAD is 12 bytes:
1569* AAD[3] = {A0, A1, A2};
1570* padded AAD in xmm register = {A2 A1 A0 0}
1571*
1572* 0 1 2 3
1573* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1574* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1575* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1576* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1577* | SPI (A2) |
1578* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1579* | 64-bit Extended Sequence Number {A1,A0} |
1580* | |
1581* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1582* | 0x0 |
1583* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1584*
1585* AAD Format with 64-bit Extended Sequence Number
1586*
1587* poly = x^128 + x^127 + x^126 + x^121 + 1
1588*
1589*****************************************************************************/
1590SYM_FUNC_START(aesni_gcm_dec)
1591 FUNC_SAVE
1592
1593 GCM_INIT %arg6, arg7, arg8, arg9
1594 GCM_ENC_DEC dec
1595 GCM_COMPLETE arg10, arg11
1596 FUNC_RESTORE
1597 RET
1598SYM_FUNC_END(aesni_gcm_dec)
1599
1600
1601/*****************************************************************************
1602* void aesni_gcm_enc(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1603* struct gcm_context_data *data
1604* // Context data
1605* u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1606* const u8 *in, // Plaintext input
1607* u64 plaintext_len, // Length of data in bytes for encryption.
1608* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association)
1609* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1610* // concatenated with 0x00000001. 16-byte aligned pointer.
1611* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary.
1612* const u8 *aad, // Additional Authentication Data (AAD)
1613* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
1614* u8 *auth_tag, // Authenticated Tag output.
1615* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
1616* // 12 or 8.
1617*
1618* Assumptions:
1619*
1620* keys:
1621* keys are pre-expanded and aligned to 16 bytes. we are using the
1622* first set of 11 keys in the data structure void *aes_ctx
1623*
1624*
1625* iv:
1626* 0 1 2 3
1627* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1628* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1629* | Salt (From the SA) |
1630* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1631* | Initialization Vector |
1632* | (This is the sequence number from IPSec header) |
1633* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1634* | 0x1 |
1635* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1636*
1637*
1638*
1639* AAD:
1640* AAD padded to 128 bits with 0
1641* for example, assume AAD is a u32 vector
1642*
1643* if AAD is 8 bytes:
1644* AAD[3] = {A0, A1};
1645* padded AAD in xmm register = {A1 A0 0 0}
1646*
1647* 0 1 2 3
1648* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1649* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1650* | SPI (A1) |
1651* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1652* | 32-bit Sequence Number (A0) |
1653* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1654* | 0x0 |
1655* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1656*
1657* AAD Format with 32-bit Sequence Number
1658*
1659* if AAD is 12 bytes:
1660* AAD[3] = {A0, A1, A2};
1661* padded AAD in xmm register = {A2 A1 A0 0}
1662*
1663* 0 1 2 3
1664* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1665* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1666* | SPI (A2) |
1667* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1668* | 64-bit Extended Sequence Number {A1,A0} |
1669* | |
1670* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1671* | 0x0 |
1672* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1673*
1674* AAD Format with 64-bit Extended Sequence Number
1675*
1676* poly = x^128 + x^127 + x^126 + x^121 + 1
1677***************************************************************************/
1678SYM_FUNC_START(aesni_gcm_enc)
1679 FUNC_SAVE
1680
1681 GCM_INIT %arg6, arg7, arg8, arg9
1682 GCM_ENC_DEC enc
1683
1684 GCM_COMPLETE arg10, arg11
1685 FUNC_RESTORE
1686 RET
1687SYM_FUNC_END(aesni_gcm_enc)
1688
1689/*****************************************************************************
1690* void aesni_gcm_init(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1691* struct gcm_context_data *data,
1692* // context data
1693* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association)
1694* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1695* // concatenated with 0x00000001. 16-byte aligned pointer.
1696* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary.
1697* const u8 *aad, // Additional Authentication Data (AAD)
1698* u64 aad_len) // Length of AAD in bytes.
1699*/
1700SYM_FUNC_START(aesni_gcm_init)
1701 FUNC_SAVE
1702 GCM_INIT %arg3, %arg4,%arg5, %arg6
1703 FUNC_RESTORE
1704 RET
1705SYM_FUNC_END(aesni_gcm_init)
1706
1707/*****************************************************************************
1708* void aesni_gcm_enc_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1709* struct gcm_context_data *data,
1710* // context data
1711* u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1712* const u8 *in, // Plaintext input
1713* u64 plaintext_len, // Length of data in bytes for encryption.
1714*/
1715SYM_FUNC_START(aesni_gcm_enc_update)
1716 FUNC_SAVE
1717 GCM_ENC_DEC enc
1718 FUNC_RESTORE
1719 RET
1720SYM_FUNC_END(aesni_gcm_enc_update)
1721
1722/*****************************************************************************
1723* void aesni_gcm_dec_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1724* struct gcm_context_data *data,
1725* // context data
1726* u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1727* const u8 *in, // Plaintext input
1728* u64 plaintext_len, // Length of data in bytes for encryption.
1729*/
1730SYM_FUNC_START(aesni_gcm_dec_update)
1731 FUNC_SAVE
1732 GCM_ENC_DEC dec
1733 FUNC_RESTORE
1734 RET
1735SYM_FUNC_END(aesni_gcm_dec_update)
1736
1737/*****************************************************************************
1738* void aesni_gcm_finalize(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1739* struct gcm_context_data *data,
1740* // context data
1741* u8 *auth_tag, // Authenticated Tag output.
1742* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
1743* // 12 or 8.
1744*/
1745SYM_FUNC_START(aesni_gcm_finalize)
1746 FUNC_SAVE
1747 GCM_COMPLETE %arg3 %arg4
1748 FUNC_RESTORE
1749 RET
1750SYM_FUNC_END(aesni_gcm_finalize)
1751
1752#endif
1753
1754SYM_FUNC_START_LOCAL(_key_expansion_256a)
1755 pshufd $0b11111111, %xmm1, %xmm1
1756 shufps $0b00010000, %xmm0, %xmm4
1757 pxor %xmm4, %xmm0
1758 shufps $0b10001100, %xmm0, %xmm4
1759 pxor %xmm4, %xmm0
1760 pxor %xmm1, %xmm0
1761 movaps %xmm0, (TKEYP)
1762 add $0x10, TKEYP
1763 RET
1764SYM_FUNC_END(_key_expansion_256a)
1765SYM_FUNC_ALIAS_LOCAL(_key_expansion_128, _key_expansion_256a)
1766
1767SYM_FUNC_START_LOCAL(_key_expansion_192a)
1768 pshufd $0b01010101, %xmm1, %xmm1
1769 shufps $0b00010000, %xmm0, %xmm4
1770 pxor %xmm4, %xmm0
1771 shufps $0b10001100, %xmm0, %xmm4
1772 pxor %xmm4, %xmm0
1773 pxor %xmm1, %xmm0
1774
1775 movaps %xmm2, %xmm5
1776 movaps %xmm2, %xmm6
1777 pslldq $4, %xmm5
1778 pshufd $0b11111111, %xmm0, %xmm3
1779 pxor %xmm3, %xmm2
1780 pxor %xmm5, %xmm2
1781
1782 movaps %xmm0, %xmm1
1783 shufps $0b01000100, %xmm0, %xmm6
1784 movaps %xmm6, (TKEYP)
1785 shufps $0b01001110, %xmm2, %xmm1
1786 movaps %xmm1, 0x10(TKEYP)
1787 add $0x20, TKEYP
1788 RET
1789SYM_FUNC_END(_key_expansion_192a)
1790
1791SYM_FUNC_START_LOCAL(_key_expansion_192b)
1792 pshufd $0b01010101, %xmm1, %xmm1
1793 shufps $0b00010000, %xmm0, %xmm4
1794 pxor %xmm4, %xmm0
1795 shufps $0b10001100, %xmm0, %xmm4
1796 pxor %xmm4, %xmm0
1797 pxor %xmm1, %xmm0
1798
1799 movaps %xmm2, %xmm5
1800 pslldq $4, %xmm5
1801 pshufd $0b11111111, %xmm0, %xmm3
1802 pxor %xmm3, %xmm2
1803 pxor %xmm5, %xmm2
1804
1805 movaps %xmm0, (TKEYP)
1806 add $0x10, TKEYP
1807 RET
1808SYM_FUNC_END(_key_expansion_192b)
1809
1810SYM_FUNC_START_LOCAL(_key_expansion_256b)
1811 pshufd $0b10101010, %xmm1, %xmm1
1812 shufps $0b00010000, %xmm2, %xmm4
1813 pxor %xmm4, %xmm2
1814 shufps $0b10001100, %xmm2, %xmm4
1815 pxor %xmm4, %xmm2
1816 pxor %xmm1, %xmm2
1817 movaps %xmm2, (TKEYP)
1818 add $0x10, TKEYP
1819 RET
1820SYM_FUNC_END(_key_expansion_256b)
1821
1822/*
1823 * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
1824 * unsigned int key_len)
1825 */
1826SYM_FUNC_START(aesni_set_key)
1827 FRAME_BEGIN
1828#ifndef __x86_64__
1829 pushl KEYP
1830 movl (FRAME_OFFSET+8)(%esp), KEYP # ctx
1831 movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key
1832 movl (FRAME_OFFSET+16)(%esp), %edx # key_len
1833#endif
1834 movups (UKEYP), %xmm0 # user key (first 16 bytes)
1835 movaps %xmm0, (KEYP)
1836 lea 0x10(KEYP), TKEYP # key addr
1837 movl %edx, 480(KEYP)
1838 pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x
1839 cmp $24, %dl
1840 jb .Lenc_key128
1841 je .Lenc_key192
1842 movups 0x10(UKEYP), %xmm2 # other user key
1843 movaps %xmm2, (TKEYP)
1844 add $0x10, TKEYP
1845 aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
1846 call _key_expansion_256a
1847 aeskeygenassist $0x1, %xmm0, %xmm1
1848 call _key_expansion_256b
1849 aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
1850 call _key_expansion_256a
1851 aeskeygenassist $0x2, %xmm0, %xmm1
1852 call _key_expansion_256b
1853 aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
1854 call _key_expansion_256a
1855 aeskeygenassist $0x4, %xmm0, %xmm1
1856 call _key_expansion_256b
1857 aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
1858 call _key_expansion_256a
1859 aeskeygenassist $0x8, %xmm0, %xmm1
1860 call _key_expansion_256b
1861 aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
1862 call _key_expansion_256a
1863 aeskeygenassist $0x10, %xmm0, %xmm1
1864 call _key_expansion_256b
1865 aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
1866 call _key_expansion_256a
1867 aeskeygenassist $0x20, %xmm0, %xmm1
1868 call _key_expansion_256b
1869 aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
1870 call _key_expansion_256a
1871 jmp .Ldec_key
1872.Lenc_key192:
1873 movq 0x10(UKEYP), %xmm2 # other user key
1874 aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
1875 call _key_expansion_192a
1876 aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
1877 call _key_expansion_192b
1878 aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
1879 call _key_expansion_192a
1880 aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
1881 call _key_expansion_192b
1882 aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
1883 call _key_expansion_192a
1884 aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
1885 call _key_expansion_192b
1886 aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
1887 call _key_expansion_192a
1888 aeskeygenassist $0x80, %xmm2, %xmm1 # round 8
1889 call _key_expansion_192b
1890 jmp .Ldec_key
1891.Lenc_key128:
1892 aeskeygenassist $0x1, %xmm0, %xmm1 # round 1
1893 call _key_expansion_128
1894 aeskeygenassist $0x2, %xmm0, %xmm1 # round 2
1895 call _key_expansion_128
1896 aeskeygenassist $0x4, %xmm0, %xmm1 # round 3
1897 call _key_expansion_128
1898 aeskeygenassist $0x8, %xmm0, %xmm1 # round 4
1899 call _key_expansion_128
1900 aeskeygenassist $0x10, %xmm0, %xmm1 # round 5
1901 call _key_expansion_128
1902 aeskeygenassist $0x20, %xmm0, %xmm1 # round 6
1903 call _key_expansion_128
1904 aeskeygenassist $0x40, %xmm0, %xmm1 # round 7
1905 call _key_expansion_128
1906 aeskeygenassist $0x80, %xmm0, %xmm1 # round 8
1907 call _key_expansion_128
1908 aeskeygenassist $0x1b, %xmm0, %xmm1 # round 9
1909 call _key_expansion_128
1910 aeskeygenassist $0x36, %xmm0, %xmm1 # round 10
1911 call _key_expansion_128
1912.Ldec_key:
1913 sub $0x10, TKEYP
1914 movaps (KEYP), %xmm0
1915 movaps (TKEYP), %xmm1
1916 movaps %xmm0, 240(TKEYP)
1917 movaps %xmm1, 240(KEYP)
1918 add $0x10, KEYP
1919 lea 240-16(TKEYP), UKEYP
1920.align 4
1921.Ldec_key_loop:
1922 movaps (KEYP), %xmm0
1923 aesimc %xmm0, %xmm1
1924 movaps %xmm1, (UKEYP)
1925 add $0x10, KEYP
1926 sub $0x10, UKEYP
1927 cmp TKEYP, KEYP
1928 jb .Ldec_key_loop
1929 xor AREG, AREG
1930#ifndef __x86_64__
1931 popl KEYP
1932#endif
1933 FRAME_END
1934 RET
1935SYM_FUNC_END(aesni_set_key)
1936
1937/*
1938 * void aesni_enc(const void *ctx, u8 *dst, const u8 *src)
1939 */
1940SYM_FUNC_START(aesni_enc)
1941 FRAME_BEGIN
1942#ifndef __x86_64__
1943 pushl KEYP
1944 pushl KLEN
1945 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
1946 movl (FRAME_OFFSET+16)(%esp), OUTP # dst
1947 movl (FRAME_OFFSET+20)(%esp), INP # src
1948#endif
1949 movl 480(KEYP), KLEN # key length
1950 movups (INP), STATE # input
1951 call _aesni_enc1
1952 movups STATE, (OUTP) # output
1953#ifndef __x86_64__
1954 popl KLEN
1955 popl KEYP
1956#endif
1957 FRAME_END
1958 RET
1959SYM_FUNC_END(aesni_enc)
1960
1961/*
1962 * _aesni_enc1: internal ABI
1963 * input:
1964 * KEYP: key struct pointer
1965 * KLEN: round count
1966 * STATE: initial state (input)
1967 * output:
1968 * STATE: finial state (output)
1969 * changed:
1970 * KEY
1971 * TKEYP (T1)
1972 */
1973SYM_FUNC_START_LOCAL(_aesni_enc1)
1974 movaps (KEYP), KEY # key
1975 mov KEYP, TKEYP
1976 pxor KEY, STATE # round 0
1977 add $0x30, TKEYP
1978 cmp $24, KLEN
1979 jb .Lenc128
1980 lea 0x20(TKEYP), TKEYP
1981 je .Lenc192
1982 add $0x20, TKEYP
1983 movaps -0x60(TKEYP), KEY
1984 aesenc KEY, STATE
1985 movaps -0x50(TKEYP), KEY
1986 aesenc KEY, STATE
1987.align 4
1988.Lenc192:
1989 movaps -0x40(TKEYP), KEY
1990 aesenc KEY, STATE
1991 movaps -0x30(TKEYP), KEY
1992 aesenc KEY, STATE
1993.align 4
1994.Lenc128:
1995 movaps -0x20(TKEYP), KEY
1996 aesenc KEY, STATE
1997 movaps -0x10(TKEYP), KEY
1998 aesenc KEY, STATE
1999 movaps (TKEYP), KEY
2000 aesenc KEY, STATE
2001 movaps 0x10(TKEYP), KEY
2002 aesenc KEY, STATE
2003 movaps 0x20(TKEYP), KEY
2004 aesenc KEY, STATE
2005 movaps 0x30(TKEYP), KEY
2006 aesenc KEY, STATE
2007 movaps 0x40(TKEYP), KEY
2008 aesenc KEY, STATE
2009 movaps 0x50(TKEYP), KEY
2010 aesenc KEY, STATE
2011 movaps 0x60(TKEYP), KEY
2012 aesenc KEY, STATE
2013 movaps 0x70(TKEYP), KEY
2014 aesenclast KEY, STATE
2015 RET
2016SYM_FUNC_END(_aesni_enc1)
2017
2018/*
2019 * _aesni_enc4: internal ABI
2020 * input:
2021 * KEYP: key struct pointer
2022 * KLEN: round count
2023 * STATE1: initial state (input)
2024 * STATE2
2025 * STATE3
2026 * STATE4
2027 * output:
2028 * STATE1: finial state (output)
2029 * STATE2
2030 * STATE3
2031 * STATE4
2032 * changed:
2033 * KEY
2034 * TKEYP (T1)
2035 */
2036SYM_FUNC_START_LOCAL(_aesni_enc4)
2037 movaps (KEYP), KEY # key
2038 mov KEYP, TKEYP
2039 pxor KEY, STATE1 # round 0
2040 pxor KEY, STATE2
2041 pxor KEY, STATE3
2042 pxor KEY, STATE4
2043 add $0x30, TKEYP
2044 cmp $24, KLEN
2045 jb .L4enc128
2046 lea 0x20(TKEYP), TKEYP
2047 je .L4enc192
2048 add $0x20, TKEYP
2049 movaps -0x60(TKEYP), KEY
2050 aesenc KEY, STATE1
2051 aesenc KEY, STATE2
2052 aesenc KEY, STATE3
2053 aesenc KEY, STATE4
2054 movaps -0x50(TKEYP), KEY
2055 aesenc KEY, STATE1
2056 aesenc KEY, STATE2
2057 aesenc KEY, STATE3
2058 aesenc KEY, STATE4
2059#.align 4
2060.L4enc192:
2061 movaps -0x40(TKEYP), KEY
2062 aesenc KEY, STATE1
2063 aesenc KEY, STATE2
2064 aesenc KEY, STATE3
2065 aesenc KEY, STATE4
2066 movaps -0x30(TKEYP), KEY
2067 aesenc KEY, STATE1
2068 aesenc KEY, STATE2
2069 aesenc KEY, STATE3
2070 aesenc KEY, STATE4
2071#.align 4
2072.L4enc128:
2073 movaps -0x20(TKEYP), KEY
2074 aesenc KEY, STATE1
2075 aesenc KEY, STATE2
2076 aesenc KEY, STATE3
2077 aesenc KEY, STATE4
2078 movaps -0x10(TKEYP), KEY
2079 aesenc KEY, STATE1
2080 aesenc KEY, STATE2
2081 aesenc KEY, STATE3
2082 aesenc KEY, STATE4
2083 movaps (TKEYP), KEY
2084 aesenc KEY, STATE1
2085 aesenc KEY, STATE2
2086 aesenc KEY, STATE3
2087 aesenc KEY, STATE4
2088 movaps 0x10(TKEYP), KEY
2089 aesenc KEY, STATE1
2090 aesenc KEY, STATE2
2091 aesenc KEY, STATE3
2092 aesenc KEY, STATE4
2093 movaps 0x20(TKEYP), KEY
2094 aesenc KEY, STATE1
2095 aesenc KEY, STATE2
2096 aesenc KEY, STATE3
2097 aesenc KEY, STATE4
2098 movaps 0x30(TKEYP), KEY
2099 aesenc KEY, STATE1
2100 aesenc KEY, STATE2
2101 aesenc KEY, STATE3
2102 aesenc KEY, STATE4
2103 movaps 0x40(TKEYP), KEY
2104 aesenc KEY, STATE1
2105 aesenc KEY, STATE2
2106 aesenc KEY, STATE3
2107 aesenc KEY, STATE4
2108 movaps 0x50(TKEYP), KEY
2109 aesenc KEY, STATE1
2110 aesenc KEY, STATE2
2111 aesenc KEY, STATE3
2112 aesenc KEY, STATE4
2113 movaps 0x60(TKEYP), KEY
2114 aesenc KEY, STATE1
2115 aesenc KEY, STATE2
2116 aesenc KEY, STATE3
2117 aesenc KEY, STATE4
2118 movaps 0x70(TKEYP), KEY
2119 aesenclast KEY, STATE1 # last round
2120 aesenclast KEY, STATE2
2121 aesenclast KEY, STATE3
2122 aesenclast KEY, STATE4
2123 RET
2124SYM_FUNC_END(_aesni_enc4)
2125
2126/*
2127 * void aesni_dec (const void *ctx, u8 *dst, const u8 *src)
2128 */
2129SYM_FUNC_START(aesni_dec)
2130 FRAME_BEGIN
2131#ifndef __x86_64__
2132 pushl KEYP
2133 pushl KLEN
2134 movl (FRAME_OFFSET+12)(%esp), KEYP # ctx
2135 movl (FRAME_OFFSET+16)(%esp), OUTP # dst
2136 movl (FRAME_OFFSET+20)(%esp), INP # src
2137#endif
2138 mov 480(KEYP), KLEN # key length
2139 add $240, KEYP
2140 movups (INP), STATE # input
2141 call _aesni_dec1
2142 movups STATE, (OUTP) #output
2143#ifndef __x86_64__
2144 popl KLEN
2145 popl KEYP
2146#endif
2147 FRAME_END
2148 RET
2149SYM_FUNC_END(aesni_dec)
2150
2151/*
2152 * _aesni_dec1: internal ABI
2153 * input:
2154 * KEYP: key struct pointer
2155 * KLEN: key length
2156 * STATE: initial state (input)
2157 * output:
2158 * STATE: finial state (output)
2159 * changed:
2160 * KEY
2161 * TKEYP (T1)
2162 */
2163SYM_FUNC_START_LOCAL(_aesni_dec1)
2164 movaps (KEYP), KEY # key
2165 mov KEYP, TKEYP
2166 pxor KEY, STATE # round 0
2167 add $0x30, TKEYP
2168 cmp $24, KLEN
2169 jb .Ldec128
2170 lea 0x20(TKEYP), TKEYP
2171 je .Ldec192
2172 add $0x20, TKEYP
2173 movaps -0x60(TKEYP), KEY
2174 aesdec KEY, STATE
2175 movaps -0x50(TKEYP), KEY
2176 aesdec KEY, STATE
2177.align 4
2178.Ldec192:
2179 movaps -0x40(TKEYP), KEY
2180 aesdec KEY, STATE
2181 movaps -0x30(TKEYP), KEY
2182 aesdec KEY, STATE
2183.align 4
2184.Ldec128:
2185 movaps -0x20(TKEYP), KEY
2186 aesdec KEY, STATE
2187 movaps -0x10(TKEYP), KEY
2188 aesdec KEY, STATE
2189 movaps (TKEYP), KEY
2190 aesdec KEY, STATE
2191 movaps 0x10(TKEYP), KEY
2192 aesdec KEY, STATE
2193 movaps 0x20(TKEYP), KEY
2194 aesdec KEY, STATE
2195 movaps 0x30(TKEYP), KEY
2196 aesdec KEY, STATE
2197 movaps 0x40(TKEYP), KEY
2198 aesdec KEY, STATE
2199 movaps 0x50(TKEYP), KEY
2200 aesdec KEY, STATE
2201 movaps 0x60(TKEYP), KEY
2202 aesdec KEY, STATE
2203 movaps 0x70(TKEYP), KEY
2204 aesdeclast KEY, STATE
2205 RET
2206SYM_FUNC_END(_aesni_dec1)
2207
2208/*
2209 * _aesni_dec4: internal ABI
2210 * input:
2211 * KEYP: key struct pointer
2212 * KLEN: key length
2213 * STATE1: initial state (input)
2214 * STATE2
2215 * STATE3
2216 * STATE4
2217 * output:
2218 * STATE1: finial state (output)
2219 * STATE2
2220 * STATE3
2221 * STATE4
2222 * changed:
2223 * KEY
2224 * TKEYP (T1)
2225 */
2226SYM_FUNC_START_LOCAL(_aesni_dec4)
2227 movaps (KEYP), KEY # key
2228 mov KEYP, TKEYP
2229 pxor KEY, STATE1 # round 0
2230 pxor KEY, STATE2
2231 pxor KEY, STATE3
2232 pxor KEY, STATE4
2233 add $0x30, TKEYP
2234 cmp $24, KLEN
2235 jb .L4dec128
2236 lea 0x20(TKEYP), TKEYP
2237 je .L4dec192
2238 add $0x20, TKEYP
2239 movaps -0x60(TKEYP), KEY
2240 aesdec KEY, STATE1
2241 aesdec KEY, STATE2
2242 aesdec KEY, STATE3
2243 aesdec KEY, STATE4
2244 movaps -0x50(TKEYP), KEY
2245 aesdec KEY, STATE1
2246 aesdec KEY, STATE2
2247 aesdec KEY, STATE3
2248 aesdec KEY, STATE4
2249.align 4
2250.L4dec192:
2251 movaps -0x40(TKEYP), KEY
2252 aesdec KEY, STATE1
2253 aesdec KEY, STATE2
2254 aesdec KEY, STATE3
2255 aesdec KEY, STATE4
2256 movaps -0x30(TKEYP), KEY
2257 aesdec KEY, STATE1
2258 aesdec KEY, STATE2
2259 aesdec KEY, STATE3
2260 aesdec KEY, STATE4
2261.align 4
2262.L4dec128:
2263 movaps -0x20(TKEYP), KEY
2264 aesdec KEY, STATE1
2265 aesdec KEY, STATE2
2266 aesdec KEY, STATE3
2267 aesdec KEY, STATE4
2268 movaps -0x10(TKEYP), KEY
2269 aesdec KEY, STATE1
2270 aesdec KEY, STATE2
2271 aesdec KEY, STATE3
2272 aesdec KEY, STATE4
2273 movaps (TKEYP), KEY
2274 aesdec KEY, STATE1
2275 aesdec KEY, STATE2
2276 aesdec KEY, STATE3
2277 aesdec KEY, STATE4
2278 movaps 0x10(TKEYP), KEY
2279 aesdec KEY, STATE1
2280 aesdec KEY, STATE2
2281 aesdec KEY, STATE3
2282 aesdec KEY, STATE4
2283 movaps 0x20(TKEYP), KEY
2284 aesdec KEY, STATE1
2285 aesdec KEY, STATE2
2286 aesdec KEY, STATE3
2287 aesdec KEY, STATE4
2288 movaps 0x30(TKEYP), KEY
2289 aesdec KEY, STATE1
2290 aesdec KEY, STATE2
2291 aesdec KEY, STATE3
2292 aesdec KEY, STATE4
2293 movaps 0x40(TKEYP), KEY
2294 aesdec KEY, STATE1
2295 aesdec KEY, STATE2
2296 aesdec KEY, STATE3
2297 aesdec KEY, STATE4
2298 movaps 0x50(TKEYP), KEY
2299 aesdec KEY, STATE1
2300 aesdec KEY, STATE2
2301 aesdec KEY, STATE3
2302 aesdec KEY, STATE4
2303 movaps 0x60(TKEYP), KEY
2304 aesdec KEY, STATE1
2305 aesdec KEY, STATE2
2306 aesdec KEY, STATE3
2307 aesdec KEY, STATE4
2308 movaps 0x70(TKEYP), KEY
2309 aesdeclast KEY, STATE1 # last round
2310 aesdeclast KEY, STATE2
2311 aesdeclast KEY, STATE3
2312 aesdeclast KEY, STATE4
2313 RET
2314SYM_FUNC_END(_aesni_dec4)
2315
2316/*
2317 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2318 * size_t len)
2319 */
2320SYM_FUNC_START(aesni_ecb_enc)
2321 FRAME_BEGIN
2322#ifndef __x86_64__
2323 pushl LEN
2324 pushl KEYP
2325 pushl KLEN
2326 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
2327 movl (FRAME_OFFSET+20)(%esp), OUTP # dst
2328 movl (FRAME_OFFSET+24)(%esp), INP # src
2329 movl (FRAME_OFFSET+28)(%esp), LEN # len
2330#endif
2331 test LEN, LEN # check length
2332 jz .Lecb_enc_ret
2333 mov 480(KEYP), KLEN
2334 cmp $16, LEN
2335 jb .Lecb_enc_ret
2336 cmp $64, LEN
2337 jb .Lecb_enc_loop1
2338.align 4
2339.Lecb_enc_loop4:
2340 movups (INP), STATE1
2341 movups 0x10(INP), STATE2
2342 movups 0x20(INP), STATE3
2343 movups 0x30(INP), STATE4
2344 call _aesni_enc4
2345 movups STATE1, (OUTP)
2346 movups STATE2, 0x10(OUTP)
2347 movups STATE3, 0x20(OUTP)
2348 movups STATE4, 0x30(OUTP)
2349 sub $64, LEN
2350 add $64, INP
2351 add $64, OUTP
2352 cmp $64, LEN
2353 jge .Lecb_enc_loop4
2354 cmp $16, LEN
2355 jb .Lecb_enc_ret
2356.align 4
2357.Lecb_enc_loop1:
2358 movups (INP), STATE1
2359 call _aesni_enc1
2360 movups STATE1, (OUTP)
2361 sub $16, LEN
2362 add $16, INP
2363 add $16, OUTP
2364 cmp $16, LEN
2365 jge .Lecb_enc_loop1
2366.Lecb_enc_ret:
2367#ifndef __x86_64__
2368 popl KLEN
2369 popl KEYP
2370 popl LEN
2371#endif
2372 FRAME_END
2373 RET
2374SYM_FUNC_END(aesni_ecb_enc)
2375
2376/*
2377 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2378 * size_t len);
2379 */
2380SYM_FUNC_START(aesni_ecb_dec)
2381 FRAME_BEGIN
2382#ifndef __x86_64__
2383 pushl LEN
2384 pushl KEYP
2385 pushl KLEN
2386 movl (FRAME_OFFSET+16)(%esp), KEYP # ctx
2387 movl (FRAME_OFFSET+20)(%esp), OUTP # dst
2388 movl (FRAME_OFFSET+24)(%esp), INP # src
2389 movl (FRAME_OFFSET+28)(%esp), LEN # len
2390#endif
2391 test LEN, LEN
2392 jz .Lecb_dec_ret
2393 mov 480(KEYP), KLEN
2394 add $240, KEYP
2395 cmp $16, LEN
2396 jb .Lecb_dec_ret
2397 cmp $64, LEN
2398 jb .Lecb_dec_loop1
2399.align 4
2400.Lecb_dec_loop4:
2401 movups (INP), STATE1
2402 movups 0x10(INP), STATE2
2403 movups 0x20(INP), STATE3
2404 movups 0x30(INP), STATE4
2405 call _aesni_dec4
2406 movups STATE1, (OUTP)
2407 movups STATE2, 0x10(OUTP)
2408 movups STATE3, 0x20(OUTP)
2409 movups STATE4, 0x30(OUTP)
2410 sub $64, LEN
2411 add $64, INP
2412 add $64, OUTP
2413 cmp $64, LEN
2414 jge .Lecb_dec_loop4
2415 cmp $16, LEN
2416 jb .Lecb_dec_ret
2417.align 4
2418.Lecb_dec_loop1:
2419 movups (INP), STATE1
2420 call _aesni_dec1
2421 movups STATE1, (OUTP)
2422 sub $16, LEN
2423 add $16, INP
2424 add $16, OUTP
2425 cmp $16, LEN
2426 jge .Lecb_dec_loop1
2427.Lecb_dec_ret:
2428#ifndef __x86_64__
2429 popl KLEN
2430 popl KEYP
2431 popl LEN
2432#endif
2433 FRAME_END
2434 RET
2435SYM_FUNC_END(aesni_ecb_dec)
2436
2437/*
2438 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2439 * size_t len, u8 *iv)
2440 */
2441SYM_FUNC_START(aesni_cbc_enc)
2442 FRAME_BEGIN
2443#ifndef __x86_64__
2444 pushl IVP
2445 pushl LEN
2446 pushl KEYP
2447 pushl KLEN
2448 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2449 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2450 movl (FRAME_OFFSET+28)(%esp), INP # src
2451 movl (FRAME_OFFSET+32)(%esp), LEN # len
2452 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2453#endif
2454 cmp $16, LEN
2455 jb .Lcbc_enc_ret
2456 mov 480(KEYP), KLEN
2457 movups (IVP), STATE # load iv as initial state
2458.align 4
2459.Lcbc_enc_loop:
2460 movups (INP), IN # load input
2461 pxor IN, STATE
2462 call _aesni_enc1
2463 movups STATE, (OUTP) # store output
2464 sub $16, LEN
2465 add $16, INP
2466 add $16, OUTP
2467 cmp $16, LEN
2468 jge .Lcbc_enc_loop
2469 movups STATE, (IVP)
2470.Lcbc_enc_ret:
2471#ifndef __x86_64__
2472 popl KLEN
2473 popl KEYP
2474 popl LEN
2475 popl IVP
2476#endif
2477 FRAME_END
2478 RET
2479SYM_FUNC_END(aesni_cbc_enc)
2480
2481/*
2482 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2483 * size_t len, u8 *iv)
2484 */
2485SYM_FUNC_START(aesni_cbc_dec)
2486 FRAME_BEGIN
2487#ifndef __x86_64__
2488 pushl IVP
2489 pushl LEN
2490 pushl KEYP
2491 pushl KLEN
2492 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2493 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2494 movl (FRAME_OFFSET+28)(%esp), INP # src
2495 movl (FRAME_OFFSET+32)(%esp), LEN # len
2496 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2497#endif
2498 cmp $16, LEN
2499 jb .Lcbc_dec_just_ret
2500 mov 480(KEYP), KLEN
2501 add $240, KEYP
2502 movups (IVP), IV
2503 cmp $64, LEN
2504 jb .Lcbc_dec_loop1
2505.align 4
2506.Lcbc_dec_loop4:
2507 movups (INP), IN1
2508 movaps IN1, STATE1
2509 movups 0x10(INP), IN2
2510 movaps IN2, STATE2
2511#ifdef __x86_64__
2512 movups 0x20(INP), IN3
2513 movaps IN3, STATE3
2514 movups 0x30(INP), IN4
2515 movaps IN4, STATE4
2516#else
2517 movups 0x20(INP), IN1
2518 movaps IN1, STATE3
2519 movups 0x30(INP), IN2
2520 movaps IN2, STATE4
2521#endif
2522 call _aesni_dec4
2523 pxor IV, STATE1
2524#ifdef __x86_64__
2525 pxor IN1, STATE2
2526 pxor IN2, STATE3
2527 pxor IN3, STATE4
2528 movaps IN4, IV
2529#else
2530 pxor IN1, STATE4
2531 movaps IN2, IV
2532 movups (INP), IN1
2533 pxor IN1, STATE2
2534 movups 0x10(INP), IN2
2535 pxor IN2, STATE3
2536#endif
2537 movups STATE1, (OUTP)
2538 movups STATE2, 0x10(OUTP)
2539 movups STATE3, 0x20(OUTP)
2540 movups STATE4, 0x30(OUTP)
2541 sub $64, LEN
2542 add $64, INP
2543 add $64, OUTP
2544 cmp $64, LEN
2545 jge .Lcbc_dec_loop4
2546 cmp $16, LEN
2547 jb .Lcbc_dec_ret
2548.align 4
2549.Lcbc_dec_loop1:
2550 movups (INP), IN
2551 movaps IN, STATE
2552 call _aesni_dec1
2553 pxor IV, STATE
2554 movups STATE, (OUTP)
2555 movaps IN, IV
2556 sub $16, LEN
2557 add $16, INP
2558 add $16, OUTP
2559 cmp $16, LEN
2560 jge .Lcbc_dec_loop1
2561.Lcbc_dec_ret:
2562 movups IV, (IVP)
2563.Lcbc_dec_just_ret:
2564#ifndef __x86_64__
2565 popl KLEN
2566 popl KEYP
2567 popl LEN
2568 popl IVP
2569#endif
2570 FRAME_END
2571 RET
2572SYM_FUNC_END(aesni_cbc_dec)
2573
2574/*
2575 * void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2576 * size_t len, u8 *iv)
2577 */
2578SYM_FUNC_START(aesni_cts_cbc_enc)
2579 FRAME_BEGIN
2580#ifndef __x86_64__
2581 pushl IVP
2582 pushl LEN
2583 pushl KEYP
2584 pushl KLEN
2585 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2586 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2587 movl (FRAME_OFFSET+28)(%esp), INP # src
2588 movl (FRAME_OFFSET+32)(%esp), LEN # len
2589 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2590 lea .Lcts_permute_table, T1
2591#else
2592 lea .Lcts_permute_table(%rip), T1
2593#endif
2594 mov 480(KEYP), KLEN
2595 movups (IVP), STATE
2596 sub $16, LEN
2597 mov T1, IVP
2598 add $32, IVP
2599 add LEN, T1
2600 sub LEN, IVP
2601 movups (T1), %xmm4
2602 movups (IVP), %xmm5
2603
2604 movups (INP), IN1
2605 add LEN, INP
2606 movups (INP), IN2
2607
2608 pxor IN1, STATE
2609 call _aesni_enc1
2610
2611 pshufb %xmm5, IN2
2612 pxor STATE, IN2
2613 pshufb %xmm4, STATE
2614 add OUTP, LEN
2615 movups STATE, (LEN)
2616
2617 movaps IN2, STATE
2618 call _aesni_enc1
2619 movups STATE, (OUTP)
2620
2621#ifndef __x86_64__
2622 popl KLEN
2623 popl KEYP
2624 popl LEN
2625 popl IVP
2626#endif
2627 FRAME_END
2628 RET
2629SYM_FUNC_END(aesni_cts_cbc_enc)
2630
2631/*
2632 * void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2633 * size_t len, u8 *iv)
2634 */
2635SYM_FUNC_START(aesni_cts_cbc_dec)
2636 FRAME_BEGIN
2637#ifndef __x86_64__
2638 pushl IVP
2639 pushl LEN
2640 pushl KEYP
2641 pushl KLEN
2642 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2643 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2644 movl (FRAME_OFFSET+28)(%esp), INP # src
2645 movl (FRAME_OFFSET+32)(%esp), LEN # len
2646 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2647 lea .Lcts_permute_table, T1
2648#else
2649 lea .Lcts_permute_table(%rip), T1
2650#endif
2651 mov 480(KEYP), KLEN
2652 add $240, KEYP
2653 movups (IVP), IV
2654 sub $16, LEN
2655 mov T1, IVP
2656 add $32, IVP
2657 add LEN, T1
2658 sub LEN, IVP
2659 movups (T1), %xmm4
2660
2661 movups (INP), STATE
2662 add LEN, INP
2663 movups (INP), IN1
2664
2665 call _aesni_dec1
2666 movaps STATE, IN2
2667 pshufb %xmm4, STATE
2668 pxor IN1, STATE
2669
2670 add OUTP, LEN
2671 movups STATE, (LEN)
2672
2673 movups (IVP), %xmm0
2674 pshufb %xmm0, IN1
2675 pblendvb IN2, IN1
2676 movaps IN1, STATE
2677 call _aesni_dec1
2678
2679 pxor IV, STATE
2680 movups STATE, (OUTP)
2681
2682#ifndef __x86_64__
2683 popl KLEN
2684 popl KEYP
2685 popl LEN
2686 popl IVP
2687#endif
2688 FRAME_END
2689 RET
2690SYM_FUNC_END(aesni_cts_cbc_dec)
2691
2692.pushsection .rodata
2693.align 16
2694.Lcts_permute_table:
2695 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2696 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2697 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
2698 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
2699 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2700 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
2701#ifdef __x86_64__
2702.Lbswap_mask:
2703 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
2704#endif
2705.popsection
2706
2707#ifdef __x86_64__
2708/*
2709 * _aesni_inc_init: internal ABI
2710 * setup registers used by _aesni_inc
2711 * input:
2712 * IV
2713 * output:
2714 * CTR: == IV, in little endian
2715 * TCTR_LOW: == lower qword of CTR
2716 * INC: == 1, in little endian
2717 * BSWAP_MASK == endian swapping mask
2718 */
2719SYM_FUNC_START_LOCAL(_aesni_inc_init)
2720 movaps .Lbswap_mask(%rip), BSWAP_MASK
2721 movaps IV, CTR
2722 pshufb BSWAP_MASK, CTR
2723 mov $1, TCTR_LOW
2724 movq TCTR_LOW, INC
2725 movq CTR, TCTR_LOW
2726 RET
2727SYM_FUNC_END(_aesni_inc_init)
2728
2729/*
2730 * _aesni_inc: internal ABI
2731 * Increase IV by 1, IV is in big endian
2732 * input:
2733 * IV
2734 * CTR: == IV, in little endian
2735 * TCTR_LOW: == lower qword of CTR
2736 * INC: == 1, in little endian
2737 * BSWAP_MASK == endian swapping mask
2738 * output:
2739 * IV: Increase by 1
2740 * changed:
2741 * CTR: == output IV, in little endian
2742 * TCTR_LOW: == lower qword of CTR
2743 */
2744SYM_FUNC_START_LOCAL(_aesni_inc)
2745 paddq INC, CTR
2746 add $1, TCTR_LOW
2747 jnc .Linc_low
2748 pslldq $8, INC
2749 paddq INC, CTR
2750 psrldq $8, INC
2751.Linc_low:
2752 movaps CTR, IV
2753 pshufb BSWAP_MASK, IV
2754 RET
2755SYM_FUNC_END(_aesni_inc)
2756
2757/*
2758 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2759 * size_t len, u8 *iv)
2760 */
2761SYM_FUNC_START(aesni_ctr_enc)
2762 FRAME_BEGIN
2763 cmp $16, LEN
2764 jb .Lctr_enc_just_ret
2765 mov 480(KEYP), KLEN
2766 movups (IVP), IV
2767 call _aesni_inc_init
2768 cmp $64, LEN
2769 jb .Lctr_enc_loop1
2770.align 4
2771.Lctr_enc_loop4:
2772 movaps IV, STATE1
2773 call _aesni_inc
2774 movups (INP), IN1
2775 movaps IV, STATE2
2776 call _aesni_inc
2777 movups 0x10(INP), IN2
2778 movaps IV, STATE3
2779 call _aesni_inc
2780 movups 0x20(INP), IN3
2781 movaps IV, STATE4
2782 call _aesni_inc
2783 movups 0x30(INP), IN4
2784 call _aesni_enc4
2785 pxor IN1, STATE1
2786 movups STATE1, (OUTP)
2787 pxor IN2, STATE2
2788 movups STATE2, 0x10(OUTP)
2789 pxor IN3, STATE3
2790 movups STATE3, 0x20(OUTP)
2791 pxor IN4, STATE4
2792 movups STATE4, 0x30(OUTP)
2793 sub $64, LEN
2794 add $64, INP
2795 add $64, OUTP
2796 cmp $64, LEN
2797 jge .Lctr_enc_loop4
2798 cmp $16, LEN
2799 jb .Lctr_enc_ret
2800.align 4
2801.Lctr_enc_loop1:
2802 movaps IV, STATE
2803 call _aesni_inc
2804 movups (INP), IN
2805 call _aesni_enc1
2806 pxor IN, STATE
2807 movups STATE, (OUTP)
2808 sub $16, LEN
2809 add $16, INP
2810 add $16, OUTP
2811 cmp $16, LEN
2812 jge .Lctr_enc_loop1
2813.Lctr_enc_ret:
2814 movups IV, (IVP)
2815.Lctr_enc_just_ret:
2816 FRAME_END
2817 RET
2818SYM_FUNC_END(aesni_ctr_enc)
2819
2820#endif
2821
2822.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16
2823.align 16
2824.Lgf128mul_x_ble_mask:
2825 .octa 0x00000000000000010000000000000087
2826.previous
2827
2828/*
2829 * _aesni_gf128mul_x_ble: internal ABI
2830 * Multiply in GF(2^128) for XTS IVs
2831 * input:
2832 * IV: current IV
2833 * GF128MUL_MASK == mask with 0x87 and 0x01
2834 * output:
2835 * IV: next IV
2836 * changed:
2837 * CTR: == temporary value
2838 */
2839#define _aesni_gf128mul_x_ble() \
2840 pshufd $0x13, IV, KEY; \
2841 paddq IV, IV; \
2842 psrad $31, KEY; \
2843 pand GF128MUL_MASK, KEY; \
2844 pxor KEY, IV;
2845
2846/*
2847 * void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst,
2848 * const u8 *src, unsigned int len, le128 *iv)
2849 */
2850SYM_FUNC_START(aesni_xts_encrypt)
2851 FRAME_BEGIN
2852#ifndef __x86_64__
2853 pushl IVP
2854 pushl LEN
2855 pushl KEYP
2856 pushl KLEN
2857 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
2858 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
2859 movl (FRAME_OFFSET+28)(%esp), INP # src
2860 movl (FRAME_OFFSET+32)(%esp), LEN # len
2861 movl (FRAME_OFFSET+36)(%esp), IVP # iv
2862 movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK
2863#else
2864 movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK
2865#endif
2866 movups (IVP), IV
2867
2868 mov 480(KEYP), KLEN
2869
2870.Lxts_enc_loop4:
2871 sub $64, LEN
2872 jl .Lxts_enc_1x
2873
2874 movdqa IV, STATE1
2875 movdqu 0x00(INP), IN
2876 pxor IN, STATE1
2877 movdqu IV, 0x00(OUTP)
2878
2879 _aesni_gf128mul_x_ble()
2880 movdqa IV, STATE2
2881 movdqu 0x10(INP), IN
2882 pxor IN, STATE2
2883 movdqu IV, 0x10(OUTP)
2884
2885 _aesni_gf128mul_x_ble()
2886 movdqa IV, STATE3
2887 movdqu 0x20(INP), IN
2888 pxor IN, STATE3
2889 movdqu IV, 0x20(OUTP)
2890
2891 _aesni_gf128mul_x_ble()
2892 movdqa IV, STATE4
2893 movdqu 0x30(INP), IN
2894 pxor IN, STATE4
2895 movdqu IV, 0x30(OUTP)
2896
2897 call _aesni_enc4
2898
2899 movdqu 0x00(OUTP), IN
2900 pxor IN, STATE1
2901 movdqu STATE1, 0x00(OUTP)
2902
2903 movdqu 0x10(OUTP), IN
2904 pxor IN, STATE2
2905 movdqu STATE2, 0x10(OUTP)
2906
2907 movdqu 0x20(OUTP), IN
2908 pxor IN, STATE3
2909 movdqu STATE3, 0x20(OUTP)
2910
2911 movdqu 0x30(OUTP), IN
2912 pxor IN, STATE4
2913 movdqu STATE4, 0x30(OUTP)
2914
2915 _aesni_gf128mul_x_ble()
2916
2917 add $64, INP
2918 add $64, OUTP
2919 test LEN, LEN
2920 jnz .Lxts_enc_loop4
2921
2922.Lxts_enc_ret_iv:
2923 movups IV, (IVP)
2924
2925.Lxts_enc_ret:
2926#ifndef __x86_64__
2927 popl KLEN
2928 popl KEYP
2929 popl LEN
2930 popl IVP
2931#endif
2932 FRAME_END
2933 RET
2934
2935.Lxts_enc_1x:
2936 add $64, LEN
2937 jz .Lxts_enc_ret_iv
2938 sub $16, LEN
2939 jl .Lxts_enc_cts4
2940
2941.Lxts_enc_loop1:
2942 movdqu (INP), STATE
2943 pxor IV, STATE
2944 call _aesni_enc1
2945 pxor IV, STATE
2946 _aesni_gf128mul_x_ble()
2947
2948 test LEN, LEN
2949 jz .Lxts_enc_out
2950
2951 add $16, INP
2952 sub $16, LEN
2953 jl .Lxts_enc_cts1
2954
2955 movdqu STATE, (OUTP)
2956 add $16, OUTP
2957 jmp .Lxts_enc_loop1
2958
2959.Lxts_enc_out:
2960 movdqu STATE, (OUTP)
2961 jmp .Lxts_enc_ret_iv
2962
2963.Lxts_enc_cts4:
2964 movdqa STATE4, STATE
2965 sub $16, OUTP
2966
2967.Lxts_enc_cts1:
2968#ifndef __x86_64__
2969 lea .Lcts_permute_table, T1
2970#else
2971 lea .Lcts_permute_table(%rip), T1
2972#endif
2973 add LEN, INP /* rewind input pointer */
2974 add $16, LEN /* # bytes in final block */
2975 movups (INP), IN1
2976
2977 mov T1, IVP
2978 add $32, IVP
2979 add LEN, T1
2980 sub LEN, IVP
2981 add OUTP, LEN
2982
2983 movups (T1), %xmm4
2984 movaps STATE, IN2
2985 pshufb %xmm4, STATE
2986 movups STATE, (LEN)
2987
2988 movups (IVP), %xmm0
2989 pshufb %xmm0, IN1
2990 pblendvb IN2, IN1
2991 movaps IN1, STATE
2992
2993 pxor IV, STATE
2994 call _aesni_enc1
2995 pxor IV, STATE
2996
2997 movups STATE, (OUTP)
2998 jmp .Lxts_enc_ret
2999SYM_FUNC_END(aesni_xts_encrypt)
3000
3001/*
3002 * void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst,
3003 * const u8 *src, unsigned int len, le128 *iv)
3004 */
3005SYM_FUNC_START(aesni_xts_decrypt)
3006 FRAME_BEGIN
3007#ifndef __x86_64__
3008 pushl IVP
3009 pushl LEN
3010 pushl KEYP
3011 pushl KLEN
3012 movl (FRAME_OFFSET+20)(%esp), KEYP # ctx
3013 movl (FRAME_OFFSET+24)(%esp), OUTP # dst
3014 movl (FRAME_OFFSET+28)(%esp), INP # src
3015 movl (FRAME_OFFSET+32)(%esp), LEN # len
3016 movl (FRAME_OFFSET+36)(%esp), IVP # iv
3017 movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK
3018#else
3019 movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK
3020#endif
3021 movups (IVP), IV
3022
3023 mov 480(KEYP), KLEN
3024 add $240, KEYP
3025
3026 test $15, LEN
3027 jz .Lxts_dec_loop4
3028 sub $16, LEN
3029
3030.Lxts_dec_loop4:
3031 sub $64, LEN
3032 jl .Lxts_dec_1x
3033
3034 movdqa IV, STATE1
3035 movdqu 0x00(INP), IN
3036 pxor IN, STATE1
3037 movdqu IV, 0x00(OUTP)
3038
3039 _aesni_gf128mul_x_ble()
3040 movdqa IV, STATE2
3041 movdqu 0x10(INP), IN
3042 pxor IN, STATE2
3043 movdqu IV, 0x10(OUTP)
3044
3045 _aesni_gf128mul_x_ble()
3046 movdqa IV, STATE3
3047 movdqu 0x20(INP), IN
3048 pxor IN, STATE3
3049 movdqu IV, 0x20(OUTP)
3050
3051 _aesni_gf128mul_x_ble()
3052 movdqa IV, STATE4
3053 movdqu 0x30(INP), IN
3054 pxor IN, STATE4
3055 movdqu IV, 0x30(OUTP)
3056
3057 call _aesni_dec4
3058
3059 movdqu 0x00(OUTP), IN
3060 pxor IN, STATE1
3061 movdqu STATE1, 0x00(OUTP)
3062
3063 movdqu 0x10(OUTP), IN
3064 pxor IN, STATE2
3065 movdqu STATE2, 0x10(OUTP)
3066
3067 movdqu 0x20(OUTP), IN
3068 pxor IN, STATE3
3069 movdqu STATE3, 0x20(OUTP)
3070
3071 movdqu 0x30(OUTP), IN
3072 pxor IN, STATE4
3073 movdqu STATE4, 0x30(OUTP)
3074
3075 _aesni_gf128mul_x_ble()
3076
3077 add $64, INP
3078 add $64, OUTP
3079 test LEN, LEN
3080 jnz .Lxts_dec_loop4
3081
3082.Lxts_dec_ret_iv:
3083 movups IV, (IVP)
3084
3085.Lxts_dec_ret:
3086#ifndef __x86_64__
3087 popl KLEN
3088 popl KEYP
3089 popl LEN
3090 popl IVP
3091#endif
3092 FRAME_END
3093 RET
3094
3095.Lxts_dec_1x:
3096 add $64, LEN
3097 jz .Lxts_dec_ret_iv
3098
3099.Lxts_dec_loop1:
3100 movdqu (INP), STATE
3101
3102 add $16, INP
3103 sub $16, LEN
3104 jl .Lxts_dec_cts1
3105
3106 pxor IV, STATE
3107 call _aesni_dec1
3108 pxor IV, STATE
3109 _aesni_gf128mul_x_ble()
3110
3111 test LEN, LEN
3112 jz .Lxts_dec_out
3113
3114 movdqu STATE, (OUTP)
3115 add $16, OUTP
3116 jmp .Lxts_dec_loop1
3117
3118.Lxts_dec_out:
3119 movdqu STATE, (OUTP)
3120 jmp .Lxts_dec_ret_iv
3121
3122.Lxts_dec_cts1:
3123 movdqa IV, STATE4
3124 _aesni_gf128mul_x_ble()
3125
3126 pxor IV, STATE
3127 call _aesni_dec1
3128 pxor IV, STATE
3129
3130#ifndef __x86_64__
3131 lea .Lcts_permute_table, T1
3132#else
3133 lea .Lcts_permute_table(%rip), T1
3134#endif
3135 add LEN, INP /* rewind input pointer */
3136 add $16, LEN /* # bytes in final block */
3137 movups (INP), IN1
3138
3139 mov T1, IVP
3140 add $32, IVP
3141 add LEN, T1
3142 sub LEN, IVP
3143 add OUTP, LEN
3144
3145 movups (T1), %xmm4
3146 movaps STATE, IN2
3147 pshufb %xmm4, STATE
3148 movups STATE, (LEN)
3149
3150 movups (IVP), %xmm0
3151 pshufb %xmm0, IN1
3152 pblendvb IN2, IN1
3153 movaps IN1, STATE
3154
3155 pxor STATE4, STATE
3156 call _aesni_dec1
3157 pxor STATE4, STATE
3158
3159 movups STATE, (OUTP)
3160 jmp .Lxts_dec_ret
3161SYM_FUNC_END(aesni_xts_decrypt)