Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Twofish Cipher 3-way parallel algorithm (x86_64)
4 *
5 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
6 */
7
8#include <linux/linkage.h>
9
10.file "twofish-x86_64-asm-3way.S"
11.text
12
13/* structure of crypto context */
14#define s0 0
15#define s1 1024
16#define s2 2048
17#define s3 3072
18#define w 4096
19#define k 4128
20
21/**********************************************************************
22 3-way twofish
23 **********************************************************************/
24#define CTX %rdi
25#define RIO %rdx
26
27#define RAB0 %rax
28#define RAB1 %rbx
29#define RAB2 %rcx
30
31#define RAB0d %eax
32#define RAB1d %ebx
33#define RAB2d %ecx
34
35#define RAB0bh %ah
36#define RAB1bh %bh
37#define RAB2bh %ch
38
39#define RAB0bl %al
40#define RAB1bl %bl
41#define RAB2bl %cl
42
43#define CD0 0x0(%rsp)
44#define CD1 0x8(%rsp)
45#define CD2 0x10(%rsp)
46
47# used only before/after all rounds
48#define RCD0 %r8
49#define RCD1 %r9
50#define RCD2 %r10
51
52# used only during rounds
53#define RX0 %r8
54#define RX1 %r9
55#define RX2 %r10
56
57#define RX0d %r8d
58#define RX1d %r9d
59#define RX2d %r10d
60
61#define RY0 %r11
62#define RY1 %r12
63#define RY2 %r13
64
65#define RY0d %r11d
66#define RY1d %r12d
67#define RY2d %r13d
68
69#define RT0 %rdx
70#define RT1 %rsi
71
72#define RT0d %edx
73#define RT1d %esi
74
75#define RT1bl %sil
76
77#define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
78 movzbl ab ## bl, tmp2 ## d; \
79 movzbl ab ## bh, tmp1 ## d; \
80 rorq $(rot), ab; \
81 op1##l T0(CTX, tmp2, 4), dst ## d; \
82 op2##l T1(CTX, tmp1, 4), dst ## d;
83
84#define swap_ab_with_cd(ab, cd, tmp) \
85 movq cd, tmp; \
86 movq ab, cd; \
87 movq tmp, ab;
88
89/*
90 * Combined G1 & G2 function. Reordered with help of rotates to have moves
91 * at beginning.
92 */
93#define g1g2_3(ab, cd, Tx0, Tx1, Tx2, Tx3, Ty0, Ty1, Ty2, Ty3, x, y) \
94 /* G1,1 && G2,1 */ \
95 do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 0, ab ## 0, x ## 0); \
96 do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 0, ab ## 0, y ## 0); \
97 \
98 do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 1, ab ## 1, x ## 1); \
99 do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 1, ab ## 1, y ## 1); \
100 \
101 do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 2, ab ## 2, x ## 2); \
102 do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 2, ab ## 2, y ## 2); \
103 \
104 /* G1,2 && G2,2 */ \
105 do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 0, x ## 0); \
106 do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 0, y ## 0); \
107 swap_ab_with_cd(ab ## 0, cd ## 0, RT0); \
108 \
109 do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 1, x ## 1); \
110 do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 1, y ## 1); \
111 swap_ab_with_cd(ab ## 1, cd ## 1, RT0); \
112 \
113 do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 2, x ## 2); \
114 do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 2, y ## 2); \
115 swap_ab_with_cd(ab ## 2, cd ## 2, RT0);
116
117#define enc_round_end(ab, x, y, n) \
118 addl y ## d, x ## d; \
119 addl x ## d, y ## d; \
120 addl k+4*(2*(n))(CTX), x ## d; \
121 xorl ab ## d, x ## d; \
122 addl k+4*(2*(n)+1)(CTX), y ## d; \
123 shrq $32, ab; \
124 roll $1, ab ## d; \
125 xorl y ## d, ab ## d; \
126 shlq $32, ab; \
127 rorl $1, x ## d; \
128 orq x, ab;
129
130#define dec_round_end(ba, x, y, n) \
131 addl y ## d, x ## d; \
132 addl x ## d, y ## d; \
133 addl k+4*(2*(n))(CTX), x ## d; \
134 addl k+4*(2*(n)+1)(CTX), y ## d; \
135 xorl ba ## d, y ## d; \
136 shrq $32, ba; \
137 roll $1, ba ## d; \
138 xorl x ## d, ba ## d; \
139 shlq $32, ba; \
140 rorl $1, y ## d; \
141 orq y, ba;
142
143#define encrypt_round3(ab, cd, n) \
144 g1g2_3(ab, cd, s0, s1, s2, s3, s0, s1, s2, s3, RX, RY); \
145 \
146 enc_round_end(ab ## 0, RX0, RY0, n); \
147 enc_round_end(ab ## 1, RX1, RY1, n); \
148 enc_round_end(ab ## 2, RX2, RY2, n);
149
150#define decrypt_round3(ba, dc, n) \
151 g1g2_3(ba, dc, s1, s2, s3, s0, s3, s0, s1, s2, RY, RX); \
152 \
153 dec_round_end(ba ## 0, RX0, RY0, n); \
154 dec_round_end(ba ## 1, RX1, RY1, n); \
155 dec_round_end(ba ## 2, RX2, RY2, n);
156
157#define encrypt_cycle3(ab, cd, n) \
158 encrypt_round3(ab, cd, n*2); \
159 encrypt_round3(ab, cd, (n*2)+1);
160
161#define decrypt_cycle3(ba, dc, n) \
162 decrypt_round3(ba, dc, (n*2)+1); \
163 decrypt_round3(ba, dc, (n*2));
164
165#define push_cd() \
166 pushq RCD2; \
167 pushq RCD1; \
168 pushq RCD0;
169
170#define pop_cd() \
171 popq RCD0; \
172 popq RCD1; \
173 popq RCD2;
174
175#define inpack3(in, n, xy, m) \
176 movq 4*(n)(in), xy ## 0; \
177 xorq w+4*m(CTX), xy ## 0; \
178 \
179 movq 4*(4+(n))(in), xy ## 1; \
180 xorq w+4*m(CTX), xy ## 1; \
181 \
182 movq 4*(8+(n))(in), xy ## 2; \
183 xorq w+4*m(CTX), xy ## 2;
184
185#define outunpack3(op, out, n, xy, m) \
186 xorq w+4*m(CTX), xy ## 0; \
187 op ## q xy ## 0, 4*(n)(out); \
188 \
189 xorq w+4*m(CTX), xy ## 1; \
190 op ## q xy ## 1, 4*(4+(n))(out); \
191 \
192 xorq w+4*m(CTX), xy ## 2; \
193 op ## q xy ## 2, 4*(8+(n))(out);
194
195#define inpack_enc3() \
196 inpack3(RIO, 0, RAB, 0); \
197 inpack3(RIO, 2, RCD, 2);
198
199#define outunpack_enc3(op) \
200 outunpack3(op, RIO, 2, RAB, 6); \
201 outunpack3(op, RIO, 0, RCD, 4);
202
203#define inpack_dec3() \
204 inpack3(RIO, 0, RAB, 4); \
205 rorq $32, RAB0; \
206 rorq $32, RAB1; \
207 rorq $32, RAB2; \
208 inpack3(RIO, 2, RCD, 6); \
209 rorq $32, RCD0; \
210 rorq $32, RCD1; \
211 rorq $32, RCD2;
212
213#define outunpack_dec3() \
214 rorq $32, RCD0; \
215 rorq $32, RCD1; \
216 rorq $32, RCD2; \
217 outunpack3(mov, RIO, 0, RCD, 0); \
218 rorq $32, RAB0; \
219 rorq $32, RAB1; \
220 rorq $32, RAB2; \
221 outunpack3(mov, RIO, 2, RAB, 2);
222
223SYM_FUNC_START(__twofish_enc_blk_3way)
224 /* input:
225 * %rdi: ctx, CTX
226 * %rsi: dst
227 * %rdx: src, RIO
228 * %rcx: bool, if true: xor output
229 */
230 pushq %r13;
231 pushq %r12;
232 pushq %rbx;
233
234 pushq %rcx; /* bool xor */
235 pushq %rsi; /* dst */
236
237 inpack_enc3();
238
239 push_cd();
240 encrypt_cycle3(RAB, CD, 0);
241 encrypt_cycle3(RAB, CD, 1);
242 encrypt_cycle3(RAB, CD, 2);
243 encrypt_cycle3(RAB, CD, 3);
244 encrypt_cycle3(RAB, CD, 4);
245 encrypt_cycle3(RAB, CD, 5);
246 encrypt_cycle3(RAB, CD, 6);
247 encrypt_cycle3(RAB, CD, 7);
248 pop_cd();
249
250 popq RIO; /* dst */
251 popq RT1; /* bool xor */
252
253 testb RT1bl, RT1bl;
254 jnz .L__enc_xor3;
255
256 outunpack_enc3(mov);
257
258 popq %rbx;
259 popq %r12;
260 popq %r13;
261 RET;
262
263.L__enc_xor3:
264 outunpack_enc3(xor);
265
266 popq %rbx;
267 popq %r12;
268 popq %r13;
269 RET;
270SYM_FUNC_END(__twofish_enc_blk_3way)
271
272SYM_FUNC_START(twofish_dec_blk_3way)
273 /* input:
274 * %rdi: ctx, CTX
275 * %rsi: dst
276 * %rdx: src, RIO
277 */
278 pushq %r13;
279 pushq %r12;
280 pushq %rbx;
281
282 pushq %rsi; /* dst */
283
284 inpack_dec3();
285
286 push_cd();
287 decrypt_cycle3(RAB, CD, 7);
288 decrypt_cycle3(RAB, CD, 6);
289 decrypt_cycle3(RAB, CD, 5);
290 decrypt_cycle3(RAB, CD, 4);
291 decrypt_cycle3(RAB, CD, 3);
292 decrypt_cycle3(RAB, CD, 2);
293 decrypt_cycle3(RAB, CD, 1);
294 decrypt_cycle3(RAB, CD, 0);
295 pop_cd();
296
297 popq RIO; /* dst */
298
299 outunpack_dec3();
300
301 popq %rbx;
302 popq %r12;
303 popq %r13;
304 RET;
305SYM_FUNC_END(twofish_dec_blk_3way)
1/*
2 * Twofish Cipher 3-way parallel algorithm (x86_64)
3 *
4 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19 * USA
20 *
21 */
22
23#include <linux/linkage.h>
24
25.file "twofish-x86_64-asm-3way.S"
26.text
27
28/* structure of crypto context */
29#define s0 0
30#define s1 1024
31#define s2 2048
32#define s3 3072
33#define w 4096
34#define k 4128
35
36/**********************************************************************
37 3-way twofish
38 **********************************************************************/
39#define CTX %rdi
40#define RIO %rdx
41
42#define RAB0 %rax
43#define RAB1 %rbx
44#define RAB2 %rcx
45
46#define RAB0d %eax
47#define RAB1d %ebx
48#define RAB2d %ecx
49
50#define RAB0bh %ah
51#define RAB1bh %bh
52#define RAB2bh %ch
53
54#define RAB0bl %al
55#define RAB1bl %bl
56#define RAB2bl %cl
57
58#define CD0 0x0(%rsp)
59#define CD1 0x8(%rsp)
60#define CD2 0x10(%rsp)
61
62# used only before/after all rounds
63#define RCD0 %r8
64#define RCD1 %r9
65#define RCD2 %r10
66
67# used only during rounds
68#define RX0 %r8
69#define RX1 %r9
70#define RX2 %r10
71
72#define RX0d %r8d
73#define RX1d %r9d
74#define RX2d %r10d
75
76#define RY0 %r11
77#define RY1 %r12
78#define RY2 %r13
79
80#define RY0d %r11d
81#define RY1d %r12d
82#define RY2d %r13d
83
84#define RT0 %rdx
85#define RT1 %rsi
86
87#define RT0d %edx
88#define RT1d %esi
89
90#define RT1bl %sil
91
92#define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
93 movzbl ab ## bl, tmp2 ## d; \
94 movzbl ab ## bh, tmp1 ## d; \
95 rorq $(rot), ab; \
96 op1##l T0(CTX, tmp2, 4), dst ## d; \
97 op2##l T1(CTX, tmp1, 4), dst ## d;
98
99#define swap_ab_with_cd(ab, cd, tmp) \
100 movq cd, tmp; \
101 movq ab, cd; \
102 movq tmp, ab;
103
104/*
105 * Combined G1 & G2 function. Reordered with help of rotates to have moves
106 * at begining.
107 */
108#define g1g2_3(ab, cd, Tx0, Tx1, Tx2, Tx3, Ty0, Ty1, Ty2, Ty3, x, y) \
109 /* G1,1 && G2,1 */ \
110 do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 0, ab ## 0, x ## 0); \
111 do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 0, ab ## 0, y ## 0); \
112 \
113 do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 1, ab ## 1, x ## 1); \
114 do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 1, ab ## 1, y ## 1); \
115 \
116 do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 2, ab ## 2, x ## 2); \
117 do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 2, ab ## 2, y ## 2); \
118 \
119 /* G1,2 && G2,2 */ \
120 do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 0, x ## 0); \
121 do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 0, y ## 0); \
122 swap_ab_with_cd(ab ## 0, cd ## 0, RT0); \
123 \
124 do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 1, x ## 1); \
125 do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 1, y ## 1); \
126 swap_ab_with_cd(ab ## 1, cd ## 1, RT0); \
127 \
128 do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 2, x ## 2); \
129 do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 2, y ## 2); \
130 swap_ab_with_cd(ab ## 2, cd ## 2, RT0);
131
132#define enc_round_end(ab, x, y, n) \
133 addl y ## d, x ## d; \
134 addl x ## d, y ## d; \
135 addl k+4*(2*(n))(CTX), x ## d; \
136 xorl ab ## d, x ## d; \
137 addl k+4*(2*(n)+1)(CTX), y ## d; \
138 shrq $32, ab; \
139 roll $1, ab ## d; \
140 xorl y ## d, ab ## d; \
141 shlq $32, ab; \
142 rorl $1, x ## d; \
143 orq x, ab;
144
145#define dec_round_end(ba, x, y, n) \
146 addl y ## d, x ## d; \
147 addl x ## d, y ## d; \
148 addl k+4*(2*(n))(CTX), x ## d; \
149 addl k+4*(2*(n)+1)(CTX), y ## d; \
150 xorl ba ## d, y ## d; \
151 shrq $32, ba; \
152 roll $1, ba ## d; \
153 xorl x ## d, ba ## d; \
154 shlq $32, ba; \
155 rorl $1, y ## d; \
156 orq y, ba;
157
158#define encrypt_round3(ab, cd, n) \
159 g1g2_3(ab, cd, s0, s1, s2, s3, s0, s1, s2, s3, RX, RY); \
160 \
161 enc_round_end(ab ## 0, RX0, RY0, n); \
162 enc_round_end(ab ## 1, RX1, RY1, n); \
163 enc_round_end(ab ## 2, RX2, RY2, n);
164
165#define decrypt_round3(ba, dc, n) \
166 g1g2_3(ba, dc, s1, s2, s3, s0, s3, s0, s1, s2, RY, RX); \
167 \
168 dec_round_end(ba ## 0, RX0, RY0, n); \
169 dec_round_end(ba ## 1, RX1, RY1, n); \
170 dec_round_end(ba ## 2, RX2, RY2, n);
171
172#define encrypt_cycle3(ab, cd, n) \
173 encrypt_round3(ab, cd, n*2); \
174 encrypt_round3(ab, cd, (n*2)+1);
175
176#define decrypt_cycle3(ba, dc, n) \
177 decrypt_round3(ba, dc, (n*2)+1); \
178 decrypt_round3(ba, dc, (n*2));
179
180#define push_cd() \
181 pushq RCD2; \
182 pushq RCD1; \
183 pushq RCD0;
184
185#define pop_cd() \
186 popq RCD0; \
187 popq RCD1; \
188 popq RCD2;
189
190#define inpack3(in, n, xy, m) \
191 movq 4*(n)(in), xy ## 0; \
192 xorq w+4*m(CTX), xy ## 0; \
193 \
194 movq 4*(4+(n))(in), xy ## 1; \
195 xorq w+4*m(CTX), xy ## 1; \
196 \
197 movq 4*(8+(n))(in), xy ## 2; \
198 xorq w+4*m(CTX), xy ## 2;
199
200#define outunpack3(op, out, n, xy, m) \
201 xorq w+4*m(CTX), xy ## 0; \
202 op ## q xy ## 0, 4*(n)(out); \
203 \
204 xorq w+4*m(CTX), xy ## 1; \
205 op ## q xy ## 1, 4*(4+(n))(out); \
206 \
207 xorq w+4*m(CTX), xy ## 2; \
208 op ## q xy ## 2, 4*(8+(n))(out);
209
210#define inpack_enc3() \
211 inpack3(RIO, 0, RAB, 0); \
212 inpack3(RIO, 2, RCD, 2);
213
214#define outunpack_enc3(op) \
215 outunpack3(op, RIO, 2, RAB, 6); \
216 outunpack3(op, RIO, 0, RCD, 4);
217
218#define inpack_dec3() \
219 inpack3(RIO, 0, RAB, 4); \
220 rorq $32, RAB0; \
221 rorq $32, RAB1; \
222 rorq $32, RAB2; \
223 inpack3(RIO, 2, RCD, 6); \
224 rorq $32, RCD0; \
225 rorq $32, RCD1; \
226 rorq $32, RCD2;
227
228#define outunpack_dec3() \
229 rorq $32, RCD0; \
230 rorq $32, RCD1; \
231 rorq $32, RCD2; \
232 outunpack3(mov, RIO, 0, RCD, 0); \
233 rorq $32, RAB0; \
234 rorq $32, RAB1; \
235 rorq $32, RAB2; \
236 outunpack3(mov, RIO, 2, RAB, 2);
237
238ENTRY(__twofish_enc_blk_3way)
239 /* input:
240 * %rdi: ctx, CTX
241 * %rsi: dst
242 * %rdx: src, RIO
243 * %rcx: bool, if true: xor output
244 */
245 pushq %r13;
246 pushq %r12;
247 pushq %rbx;
248
249 pushq %rcx; /* bool xor */
250 pushq %rsi; /* dst */
251
252 inpack_enc3();
253
254 push_cd();
255 encrypt_cycle3(RAB, CD, 0);
256 encrypt_cycle3(RAB, CD, 1);
257 encrypt_cycle3(RAB, CD, 2);
258 encrypt_cycle3(RAB, CD, 3);
259 encrypt_cycle3(RAB, CD, 4);
260 encrypt_cycle3(RAB, CD, 5);
261 encrypt_cycle3(RAB, CD, 6);
262 encrypt_cycle3(RAB, CD, 7);
263 pop_cd();
264
265 popq RIO; /* dst */
266 popq RT1; /* bool xor */
267
268 testb RT1bl, RT1bl;
269 jnz .L__enc_xor3;
270
271 outunpack_enc3(mov);
272
273 popq %rbx;
274 popq %r12;
275 popq %r13;
276 ret;
277
278.L__enc_xor3:
279 outunpack_enc3(xor);
280
281 popq %rbx;
282 popq %r12;
283 popq %r13;
284 ret;
285ENDPROC(__twofish_enc_blk_3way)
286
287ENTRY(twofish_dec_blk_3way)
288 /* input:
289 * %rdi: ctx, CTX
290 * %rsi: dst
291 * %rdx: src, RIO
292 */
293 pushq %r13;
294 pushq %r12;
295 pushq %rbx;
296
297 pushq %rsi; /* dst */
298
299 inpack_dec3();
300
301 push_cd();
302 decrypt_cycle3(RAB, CD, 7);
303 decrypt_cycle3(RAB, CD, 6);
304 decrypt_cycle3(RAB, CD, 5);
305 decrypt_cycle3(RAB, CD, 4);
306 decrypt_cycle3(RAB, CD, 3);
307 decrypt_cycle3(RAB, CD, 2);
308 decrypt_cycle3(RAB, CD, 1);
309 decrypt_cycle3(RAB, CD, 0);
310 pop_cd();
311
312 popq RIO; /* dst */
313
314 outunpack_dec3();
315
316 popq %rbx;
317 popq %r12;
318 popq %r13;
319 ret;
320ENDPROC(twofish_dec_blk_3way)