Loading...
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2/* Copyright (C) 2016-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
3 *
4 * SipHash: a fast short-input PRF
5 * https://131002.net/siphash/
6 *
7 * This implementation is specifically for SipHash2-4 for a secure PRF
8 * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for
9 * hashtables.
10 */
11
12#include <linux/siphash.h>
13#include <asm/unaligned.h>
14
15#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
16#include <linux/dcache.h>
17#include <asm/word-at-a-time.h>
18#endif
19
20#define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3)
21
22#define PREAMBLE(len) \
23 u64 v0 = SIPHASH_CONST_0; \
24 u64 v1 = SIPHASH_CONST_1; \
25 u64 v2 = SIPHASH_CONST_2; \
26 u64 v3 = SIPHASH_CONST_3; \
27 u64 b = ((u64)(len)) << 56; \
28 v3 ^= key->key[1]; \
29 v2 ^= key->key[0]; \
30 v1 ^= key->key[1]; \
31 v0 ^= key->key[0];
32
33#define POSTAMBLE \
34 v3 ^= b; \
35 SIPROUND; \
36 SIPROUND; \
37 v0 ^= b; \
38 v2 ^= 0xff; \
39 SIPROUND; \
40 SIPROUND; \
41 SIPROUND; \
42 SIPROUND; \
43 return (v0 ^ v1) ^ (v2 ^ v3);
44
45#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
46u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
47{
48 const u8 *end = data + len - (len % sizeof(u64));
49 const u8 left = len & (sizeof(u64) - 1);
50 u64 m;
51 PREAMBLE(len)
52 for (; data != end; data += sizeof(u64)) {
53 m = le64_to_cpup(data);
54 v3 ^= m;
55 SIPROUND;
56 SIPROUND;
57 v0 ^= m;
58 }
59#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
60 if (left)
61 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
62 bytemask_from_count(left)));
63#else
64 switch (left) {
65 case 7: b |= ((u64)end[6]) << 48; fallthrough;
66 case 6: b |= ((u64)end[5]) << 40; fallthrough;
67 case 5: b |= ((u64)end[4]) << 32; fallthrough;
68 case 4: b |= le32_to_cpup(data); break;
69 case 3: b |= ((u64)end[2]) << 16; fallthrough;
70 case 2: b |= le16_to_cpup(data); break;
71 case 1: b |= end[0];
72 }
73#endif
74 POSTAMBLE
75}
76EXPORT_SYMBOL(__siphash_aligned);
77#endif
78
79u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
80{
81 const u8 *end = data + len - (len % sizeof(u64));
82 const u8 left = len & (sizeof(u64) - 1);
83 u64 m;
84 PREAMBLE(len)
85 for (; data != end; data += sizeof(u64)) {
86 m = get_unaligned_le64(data);
87 v3 ^= m;
88 SIPROUND;
89 SIPROUND;
90 v0 ^= m;
91 }
92#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
93 if (left)
94 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
95 bytemask_from_count(left)));
96#else
97 switch (left) {
98 case 7: b |= ((u64)end[6]) << 48; fallthrough;
99 case 6: b |= ((u64)end[5]) << 40; fallthrough;
100 case 5: b |= ((u64)end[4]) << 32; fallthrough;
101 case 4: b |= get_unaligned_le32(end); break;
102 case 3: b |= ((u64)end[2]) << 16; fallthrough;
103 case 2: b |= get_unaligned_le16(end); break;
104 case 1: b |= end[0];
105 }
106#endif
107 POSTAMBLE
108}
109EXPORT_SYMBOL(__siphash_unaligned);
110
111/**
112 * siphash_1u64 - compute 64-bit siphash PRF value of a u64
113 * @first: first u64
114 * @key: the siphash key
115 */
116u64 siphash_1u64(const u64 first, const siphash_key_t *key)
117{
118 PREAMBLE(8)
119 v3 ^= first;
120 SIPROUND;
121 SIPROUND;
122 v0 ^= first;
123 POSTAMBLE
124}
125EXPORT_SYMBOL(siphash_1u64);
126
127/**
128 * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64
129 * @first: first u64
130 * @second: second u64
131 * @key: the siphash key
132 */
133u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key)
134{
135 PREAMBLE(16)
136 v3 ^= first;
137 SIPROUND;
138 SIPROUND;
139 v0 ^= first;
140 v3 ^= second;
141 SIPROUND;
142 SIPROUND;
143 v0 ^= second;
144 POSTAMBLE
145}
146EXPORT_SYMBOL(siphash_2u64);
147
148/**
149 * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64
150 * @first: first u64
151 * @second: second u64
152 * @third: third u64
153 * @key: the siphash key
154 */
155u64 siphash_3u64(const u64 first, const u64 second, const u64 third,
156 const siphash_key_t *key)
157{
158 PREAMBLE(24)
159 v3 ^= first;
160 SIPROUND;
161 SIPROUND;
162 v0 ^= first;
163 v3 ^= second;
164 SIPROUND;
165 SIPROUND;
166 v0 ^= second;
167 v3 ^= third;
168 SIPROUND;
169 SIPROUND;
170 v0 ^= third;
171 POSTAMBLE
172}
173EXPORT_SYMBOL(siphash_3u64);
174
175/**
176 * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64
177 * @first: first u64
178 * @second: second u64
179 * @third: third u64
180 * @forth: forth u64
181 * @key: the siphash key
182 */
183u64 siphash_4u64(const u64 first, const u64 second, const u64 third,
184 const u64 forth, const siphash_key_t *key)
185{
186 PREAMBLE(32)
187 v3 ^= first;
188 SIPROUND;
189 SIPROUND;
190 v0 ^= first;
191 v3 ^= second;
192 SIPROUND;
193 SIPROUND;
194 v0 ^= second;
195 v3 ^= third;
196 SIPROUND;
197 SIPROUND;
198 v0 ^= third;
199 v3 ^= forth;
200 SIPROUND;
201 SIPROUND;
202 v0 ^= forth;
203 POSTAMBLE
204}
205EXPORT_SYMBOL(siphash_4u64);
206
207u64 siphash_1u32(const u32 first, const siphash_key_t *key)
208{
209 PREAMBLE(4)
210 b |= first;
211 POSTAMBLE
212}
213EXPORT_SYMBOL(siphash_1u32);
214
215u64 siphash_3u32(const u32 first, const u32 second, const u32 third,
216 const siphash_key_t *key)
217{
218 u64 combined = (u64)second << 32 | first;
219 PREAMBLE(12)
220 v3 ^= combined;
221 SIPROUND;
222 SIPROUND;
223 v0 ^= combined;
224 b |= third;
225 POSTAMBLE
226}
227EXPORT_SYMBOL(siphash_3u32);
228
229#if BITS_PER_LONG == 64
230/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for
231 * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3.
232 */
233
234#define HSIPROUND SIPROUND
235#define HPREAMBLE(len) PREAMBLE(len)
236#define HPOSTAMBLE \
237 v3 ^= b; \
238 HSIPROUND; \
239 v0 ^= b; \
240 v2 ^= 0xff; \
241 HSIPROUND; \
242 HSIPROUND; \
243 HSIPROUND; \
244 return (v0 ^ v1) ^ (v2 ^ v3);
245
246#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
247u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
248{
249 const u8 *end = data + len - (len % sizeof(u64));
250 const u8 left = len & (sizeof(u64) - 1);
251 u64 m;
252 HPREAMBLE(len)
253 for (; data != end; data += sizeof(u64)) {
254 m = le64_to_cpup(data);
255 v3 ^= m;
256 HSIPROUND;
257 v0 ^= m;
258 }
259#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
260 if (left)
261 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
262 bytemask_from_count(left)));
263#else
264 switch (left) {
265 case 7: b |= ((u64)end[6]) << 48; fallthrough;
266 case 6: b |= ((u64)end[5]) << 40; fallthrough;
267 case 5: b |= ((u64)end[4]) << 32; fallthrough;
268 case 4: b |= le32_to_cpup(data); break;
269 case 3: b |= ((u64)end[2]) << 16; fallthrough;
270 case 2: b |= le16_to_cpup(data); break;
271 case 1: b |= end[0];
272 }
273#endif
274 HPOSTAMBLE
275}
276EXPORT_SYMBOL(__hsiphash_aligned);
277#endif
278
279u32 __hsiphash_unaligned(const void *data, size_t len,
280 const hsiphash_key_t *key)
281{
282 const u8 *end = data + len - (len % sizeof(u64));
283 const u8 left = len & (sizeof(u64) - 1);
284 u64 m;
285 HPREAMBLE(len)
286 for (; data != end; data += sizeof(u64)) {
287 m = get_unaligned_le64(data);
288 v3 ^= m;
289 HSIPROUND;
290 v0 ^= m;
291 }
292#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
293 if (left)
294 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
295 bytemask_from_count(left)));
296#else
297 switch (left) {
298 case 7: b |= ((u64)end[6]) << 48; fallthrough;
299 case 6: b |= ((u64)end[5]) << 40; fallthrough;
300 case 5: b |= ((u64)end[4]) << 32; fallthrough;
301 case 4: b |= get_unaligned_le32(end); break;
302 case 3: b |= ((u64)end[2]) << 16; fallthrough;
303 case 2: b |= get_unaligned_le16(end); break;
304 case 1: b |= end[0];
305 }
306#endif
307 HPOSTAMBLE
308}
309EXPORT_SYMBOL(__hsiphash_unaligned);
310
311/**
312 * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32
313 * @first: first u32
314 * @key: the hsiphash key
315 */
316u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
317{
318 HPREAMBLE(4)
319 b |= first;
320 HPOSTAMBLE
321}
322EXPORT_SYMBOL(hsiphash_1u32);
323
324/**
325 * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
326 * @first: first u32
327 * @second: second u32
328 * @key: the hsiphash key
329 */
330u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
331{
332 u64 combined = (u64)second << 32 | first;
333 HPREAMBLE(8)
334 v3 ^= combined;
335 HSIPROUND;
336 v0 ^= combined;
337 HPOSTAMBLE
338}
339EXPORT_SYMBOL(hsiphash_2u32);
340
341/**
342 * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
343 * @first: first u32
344 * @second: second u32
345 * @third: third u32
346 * @key: the hsiphash key
347 */
348u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
349 const hsiphash_key_t *key)
350{
351 u64 combined = (u64)second << 32 | first;
352 HPREAMBLE(12)
353 v3 ^= combined;
354 HSIPROUND;
355 v0 ^= combined;
356 b |= third;
357 HPOSTAMBLE
358}
359EXPORT_SYMBOL(hsiphash_3u32);
360
361/**
362 * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
363 * @first: first u32
364 * @second: second u32
365 * @third: third u32
366 * @forth: forth u32
367 * @key: the hsiphash key
368 */
369u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
370 const u32 forth, const hsiphash_key_t *key)
371{
372 u64 combined = (u64)second << 32 | first;
373 HPREAMBLE(16)
374 v3 ^= combined;
375 HSIPROUND;
376 v0 ^= combined;
377 combined = (u64)forth << 32 | third;
378 v3 ^= combined;
379 HSIPROUND;
380 v0 ^= combined;
381 HPOSTAMBLE
382}
383EXPORT_SYMBOL(hsiphash_4u32);
384#else
385#define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3)
386
387#define HPREAMBLE(len) \
388 u32 v0 = HSIPHASH_CONST_0; \
389 u32 v1 = HSIPHASH_CONST_1; \
390 u32 v2 = HSIPHASH_CONST_2; \
391 u32 v3 = HSIPHASH_CONST_3; \
392 u32 b = ((u32)(len)) << 24; \
393 v3 ^= key->key[1]; \
394 v2 ^= key->key[0]; \
395 v1 ^= key->key[1]; \
396 v0 ^= key->key[0];
397
398#define HPOSTAMBLE \
399 v3 ^= b; \
400 HSIPROUND; \
401 v0 ^= b; \
402 v2 ^= 0xff; \
403 HSIPROUND; \
404 HSIPROUND; \
405 HSIPROUND; \
406 return v1 ^ v3;
407
408#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
409u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
410{
411 const u8 *end = data + len - (len % sizeof(u32));
412 const u8 left = len & (sizeof(u32) - 1);
413 u32 m;
414 HPREAMBLE(len)
415 for (; data != end; data += sizeof(u32)) {
416 m = le32_to_cpup(data);
417 v3 ^= m;
418 HSIPROUND;
419 v0 ^= m;
420 }
421 switch (left) {
422 case 3: b |= ((u32)end[2]) << 16; fallthrough;
423 case 2: b |= le16_to_cpup(data); break;
424 case 1: b |= end[0];
425 }
426 HPOSTAMBLE
427}
428EXPORT_SYMBOL(__hsiphash_aligned);
429#endif
430
431u32 __hsiphash_unaligned(const void *data, size_t len,
432 const hsiphash_key_t *key)
433{
434 const u8 *end = data + len - (len % sizeof(u32));
435 const u8 left = len & (sizeof(u32) - 1);
436 u32 m;
437 HPREAMBLE(len)
438 for (; data != end; data += sizeof(u32)) {
439 m = get_unaligned_le32(data);
440 v3 ^= m;
441 HSIPROUND;
442 v0 ^= m;
443 }
444 switch (left) {
445 case 3: b |= ((u32)end[2]) << 16; fallthrough;
446 case 2: b |= get_unaligned_le16(end); break;
447 case 1: b |= end[0];
448 }
449 HPOSTAMBLE
450}
451EXPORT_SYMBOL(__hsiphash_unaligned);
452
453/**
454 * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32
455 * @first: first u32
456 * @key: the hsiphash key
457 */
458u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
459{
460 HPREAMBLE(4)
461 v3 ^= first;
462 HSIPROUND;
463 v0 ^= first;
464 HPOSTAMBLE
465}
466EXPORT_SYMBOL(hsiphash_1u32);
467
468/**
469 * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
470 * @first: first u32
471 * @second: second u32
472 * @key: the hsiphash key
473 */
474u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
475{
476 HPREAMBLE(8)
477 v3 ^= first;
478 HSIPROUND;
479 v0 ^= first;
480 v3 ^= second;
481 HSIPROUND;
482 v0 ^= second;
483 HPOSTAMBLE
484}
485EXPORT_SYMBOL(hsiphash_2u32);
486
487/**
488 * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
489 * @first: first u32
490 * @second: second u32
491 * @third: third u32
492 * @key: the hsiphash key
493 */
494u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
495 const hsiphash_key_t *key)
496{
497 HPREAMBLE(12)
498 v3 ^= first;
499 HSIPROUND;
500 v0 ^= first;
501 v3 ^= second;
502 HSIPROUND;
503 v0 ^= second;
504 v3 ^= third;
505 HSIPROUND;
506 v0 ^= third;
507 HPOSTAMBLE
508}
509EXPORT_SYMBOL(hsiphash_3u32);
510
511/**
512 * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
513 * @first: first u32
514 * @second: second u32
515 * @third: third u32
516 * @forth: forth u32
517 * @key: the hsiphash key
518 */
519u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
520 const u32 forth, const hsiphash_key_t *key)
521{
522 HPREAMBLE(16)
523 v3 ^= first;
524 HSIPROUND;
525 v0 ^= first;
526 v3 ^= second;
527 HSIPROUND;
528 v0 ^= second;
529 v3 ^= third;
530 HSIPROUND;
531 v0 ^= third;
532 v3 ^= forth;
533 HSIPROUND;
534 v0 ^= forth;
535 HPOSTAMBLE
536}
537EXPORT_SYMBOL(hsiphash_4u32);
538#endif
1/* Copyright (C) 2016 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
2 *
3 * This file is provided under a dual BSD/GPLv2 license.
4 *
5 * SipHash: a fast short-input PRF
6 * https://131002.net/siphash/
7 *
8 * This implementation is specifically for SipHash2-4 for a secure PRF
9 * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for
10 * hashtables.
11 */
12
13#include <linux/siphash.h>
14#include <asm/unaligned.h>
15
16#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
17#include <linux/dcache.h>
18#include <asm/word-at-a-time.h>
19#endif
20
21#define SIPROUND \
22 do { \
23 v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \
24 v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \
25 v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \
26 v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \
27 } while (0)
28
29#define PREAMBLE(len) \
30 u64 v0 = 0x736f6d6570736575ULL; \
31 u64 v1 = 0x646f72616e646f6dULL; \
32 u64 v2 = 0x6c7967656e657261ULL; \
33 u64 v3 = 0x7465646279746573ULL; \
34 u64 b = ((u64)(len)) << 56; \
35 v3 ^= key->key[1]; \
36 v2 ^= key->key[0]; \
37 v1 ^= key->key[1]; \
38 v0 ^= key->key[0];
39
40#define POSTAMBLE \
41 v3 ^= b; \
42 SIPROUND; \
43 SIPROUND; \
44 v0 ^= b; \
45 v2 ^= 0xff; \
46 SIPROUND; \
47 SIPROUND; \
48 SIPROUND; \
49 SIPROUND; \
50 return (v0 ^ v1) ^ (v2 ^ v3);
51
52u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)
53{
54 const u8 *end = data + len - (len % sizeof(u64));
55 const u8 left = len & (sizeof(u64) - 1);
56 u64 m;
57 PREAMBLE(len)
58 for (; data != end; data += sizeof(u64)) {
59 m = le64_to_cpup(data);
60 v3 ^= m;
61 SIPROUND;
62 SIPROUND;
63 v0 ^= m;
64 }
65#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
66 if (left)
67 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
68 bytemask_from_count(left)));
69#else
70 switch (left) {
71 case 7: b |= ((u64)end[6]) << 48; /* fall through */
72 case 6: b |= ((u64)end[5]) << 40; /* fall through */
73 case 5: b |= ((u64)end[4]) << 32; /* fall through */
74 case 4: b |= le32_to_cpup(data); break;
75 case 3: b |= ((u64)end[2]) << 16; /* fall through */
76 case 2: b |= le16_to_cpup(data); break;
77 case 1: b |= end[0];
78 }
79#endif
80 POSTAMBLE
81}
82EXPORT_SYMBOL(__siphash_aligned);
83
84#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
85u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)
86{
87 const u8 *end = data + len - (len % sizeof(u64));
88 const u8 left = len & (sizeof(u64) - 1);
89 u64 m;
90 PREAMBLE(len)
91 for (; data != end; data += sizeof(u64)) {
92 m = get_unaligned_le64(data);
93 v3 ^= m;
94 SIPROUND;
95 SIPROUND;
96 v0 ^= m;
97 }
98#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
99 if (left)
100 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
101 bytemask_from_count(left)));
102#else
103 switch (left) {
104 case 7: b |= ((u64)end[6]) << 48; /* fall through */
105 case 6: b |= ((u64)end[5]) << 40; /* fall through */
106 case 5: b |= ((u64)end[4]) << 32; /* fall through */
107 case 4: b |= get_unaligned_le32(end); break;
108 case 3: b |= ((u64)end[2]) << 16; /* fall through */
109 case 2: b |= get_unaligned_le16(end); break;
110 case 1: b |= end[0];
111 }
112#endif
113 POSTAMBLE
114}
115EXPORT_SYMBOL(__siphash_unaligned);
116#endif
117
118/**
119 * siphash_1u64 - compute 64-bit siphash PRF value of a u64
120 * @first: first u64
121 * @key: the siphash key
122 */
123u64 siphash_1u64(const u64 first, const siphash_key_t *key)
124{
125 PREAMBLE(8)
126 v3 ^= first;
127 SIPROUND;
128 SIPROUND;
129 v0 ^= first;
130 POSTAMBLE
131}
132EXPORT_SYMBOL(siphash_1u64);
133
134/**
135 * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64
136 * @first: first u64
137 * @second: second u64
138 * @key: the siphash key
139 */
140u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key)
141{
142 PREAMBLE(16)
143 v3 ^= first;
144 SIPROUND;
145 SIPROUND;
146 v0 ^= first;
147 v3 ^= second;
148 SIPROUND;
149 SIPROUND;
150 v0 ^= second;
151 POSTAMBLE
152}
153EXPORT_SYMBOL(siphash_2u64);
154
155/**
156 * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64
157 * @first: first u64
158 * @second: second u64
159 * @third: third u64
160 * @key: the siphash key
161 */
162u64 siphash_3u64(const u64 first, const u64 second, const u64 third,
163 const siphash_key_t *key)
164{
165 PREAMBLE(24)
166 v3 ^= first;
167 SIPROUND;
168 SIPROUND;
169 v0 ^= first;
170 v3 ^= second;
171 SIPROUND;
172 SIPROUND;
173 v0 ^= second;
174 v3 ^= third;
175 SIPROUND;
176 SIPROUND;
177 v0 ^= third;
178 POSTAMBLE
179}
180EXPORT_SYMBOL(siphash_3u64);
181
182/**
183 * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64
184 * @first: first u64
185 * @second: second u64
186 * @third: third u64
187 * @forth: forth u64
188 * @key: the siphash key
189 */
190u64 siphash_4u64(const u64 first, const u64 second, const u64 third,
191 const u64 forth, const siphash_key_t *key)
192{
193 PREAMBLE(32)
194 v3 ^= first;
195 SIPROUND;
196 SIPROUND;
197 v0 ^= first;
198 v3 ^= second;
199 SIPROUND;
200 SIPROUND;
201 v0 ^= second;
202 v3 ^= third;
203 SIPROUND;
204 SIPROUND;
205 v0 ^= third;
206 v3 ^= forth;
207 SIPROUND;
208 SIPROUND;
209 v0 ^= forth;
210 POSTAMBLE
211}
212EXPORT_SYMBOL(siphash_4u64);
213
214u64 siphash_1u32(const u32 first, const siphash_key_t *key)
215{
216 PREAMBLE(4)
217 b |= first;
218 POSTAMBLE
219}
220EXPORT_SYMBOL(siphash_1u32);
221
222u64 siphash_3u32(const u32 first, const u32 second, const u32 third,
223 const siphash_key_t *key)
224{
225 u64 combined = (u64)second << 32 | first;
226 PREAMBLE(12)
227 v3 ^= combined;
228 SIPROUND;
229 SIPROUND;
230 v0 ^= combined;
231 b |= third;
232 POSTAMBLE
233}
234EXPORT_SYMBOL(siphash_3u32);
235
236#if BITS_PER_LONG == 64
237/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for
238 * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3.
239 */
240
241#define HSIPROUND SIPROUND
242#define HPREAMBLE(len) PREAMBLE(len)
243#define HPOSTAMBLE \
244 v3 ^= b; \
245 HSIPROUND; \
246 v0 ^= b; \
247 v2 ^= 0xff; \
248 HSIPROUND; \
249 HSIPROUND; \
250 HSIPROUND; \
251 return (v0 ^ v1) ^ (v2 ^ v3);
252
253u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
254{
255 const u8 *end = data + len - (len % sizeof(u64));
256 const u8 left = len & (sizeof(u64) - 1);
257 u64 m;
258 HPREAMBLE(len)
259 for (; data != end; data += sizeof(u64)) {
260 m = le64_to_cpup(data);
261 v3 ^= m;
262 HSIPROUND;
263 v0 ^= m;
264 }
265#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
266 if (left)
267 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
268 bytemask_from_count(left)));
269#else
270 switch (left) {
271 case 7: b |= ((u64)end[6]) << 48; /* fall through */
272 case 6: b |= ((u64)end[5]) << 40; /* fall through */
273 case 5: b |= ((u64)end[4]) << 32; /* fall through */
274 case 4: b |= le32_to_cpup(data); break;
275 case 3: b |= ((u64)end[2]) << 16; /* fall through */
276 case 2: b |= le16_to_cpup(data); break;
277 case 1: b |= end[0];
278 }
279#endif
280 HPOSTAMBLE
281}
282EXPORT_SYMBOL(__hsiphash_aligned);
283
284#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
285u32 __hsiphash_unaligned(const void *data, size_t len,
286 const hsiphash_key_t *key)
287{
288 const u8 *end = data + len - (len % sizeof(u64));
289 const u8 left = len & (sizeof(u64) - 1);
290 u64 m;
291 HPREAMBLE(len)
292 for (; data != end; data += sizeof(u64)) {
293 m = get_unaligned_le64(data);
294 v3 ^= m;
295 HSIPROUND;
296 v0 ^= m;
297 }
298#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64
299 if (left)
300 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &
301 bytemask_from_count(left)));
302#else
303 switch (left) {
304 case 7: b |= ((u64)end[6]) << 48; /* fall through */
305 case 6: b |= ((u64)end[5]) << 40; /* fall through */
306 case 5: b |= ((u64)end[4]) << 32; /* fall through */
307 case 4: b |= get_unaligned_le32(end); break;
308 case 3: b |= ((u64)end[2]) << 16; /* fall through */
309 case 2: b |= get_unaligned_le16(end); break;
310 case 1: b |= end[0];
311 }
312#endif
313 HPOSTAMBLE
314}
315EXPORT_SYMBOL(__hsiphash_unaligned);
316#endif
317
318/**
319 * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32
320 * @first: first u32
321 * @key: the hsiphash key
322 */
323u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
324{
325 HPREAMBLE(4)
326 b |= first;
327 HPOSTAMBLE
328}
329EXPORT_SYMBOL(hsiphash_1u32);
330
331/**
332 * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
333 * @first: first u32
334 * @second: second u32
335 * @key: the hsiphash key
336 */
337u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
338{
339 u64 combined = (u64)second << 32 | first;
340 HPREAMBLE(8)
341 v3 ^= combined;
342 HSIPROUND;
343 v0 ^= combined;
344 HPOSTAMBLE
345}
346EXPORT_SYMBOL(hsiphash_2u32);
347
348/**
349 * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
350 * @first: first u32
351 * @second: second u32
352 * @third: third u32
353 * @key: the hsiphash key
354 */
355u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
356 const hsiphash_key_t *key)
357{
358 u64 combined = (u64)second << 32 | first;
359 HPREAMBLE(12)
360 v3 ^= combined;
361 HSIPROUND;
362 v0 ^= combined;
363 b |= third;
364 HPOSTAMBLE
365}
366EXPORT_SYMBOL(hsiphash_3u32);
367
368/**
369 * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
370 * @first: first u32
371 * @second: second u32
372 * @third: third u32
373 * @forth: forth u32
374 * @key: the hsiphash key
375 */
376u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
377 const u32 forth, const hsiphash_key_t *key)
378{
379 u64 combined = (u64)second << 32 | first;
380 HPREAMBLE(16)
381 v3 ^= combined;
382 HSIPROUND;
383 v0 ^= combined;
384 combined = (u64)forth << 32 | third;
385 v3 ^= combined;
386 HSIPROUND;
387 v0 ^= combined;
388 HPOSTAMBLE
389}
390EXPORT_SYMBOL(hsiphash_4u32);
391#else
392#define HSIPROUND \
393 do { \
394 v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \
395 v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \
396 v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \
397 v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \
398 } while (0)
399
400#define HPREAMBLE(len) \
401 u32 v0 = 0; \
402 u32 v1 = 0; \
403 u32 v2 = 0x6c796765U; \
404 u32 v3 = 0x74656462U; \
405 u32 b = ((u32)(len)) << 24; \
406 v3 ^= key->key[1]; \
407 v2 ^= key->key[0]; \
408 v1 ^= key->key[1]; \
409 v0 ^= key->key[0];
410
411#define HPOSTAMBLE \
412 v3 ^= b; \
413 HSIPROUND; \
414 v0 ^= b; \
415 v2 ^= 0xff; \
416 HSIPROUND; \
417 HSIPROUND; \
418 HSIPROUND; \
419 return v1 ^ v3;
420
421u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)
422{
423 const u8 *end = data + len - (len % sizeof(u32));
424 const u8 left = len & (sizeof(u32) - 1);
425 u32 m;
426 HPREAMBLE(len)
427 for (; data != end; data += sizeof(u32)) {
428 m = le32_to_cpup(data);
429 v3 ^= m;
430 HSIPROUND;
431 v0 ^= m;
432 }
433 switch (left) {
434 case 3: b |= ((u32)end[2]) << 16; /* fall through */
435 case 2: b |= le16_to_cpup(data); break;
436 case 1: b |= end[0];
437 }
438 HPOSTAMBLE
439}
440EXPORT_SYMBOL(__hsiphash_aligned);
441
442#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
443u32 __hsiphash_unaligned(const void *data, size_t len,
444 const hsiphash_key_t *key)
445{
446 const u8 *end = data + len - (len % sizeof(u32));
447 const u8 left = len & (sizeof(u32) - 1);
448 u32 m;
449 HPREAMBLE(len)
450 for (; data != end; data += sizeof(u32)) {
451 m = get_unaligned_le32(data);
452 v3 ^= m;
453 HSIPROUND;
454 v0 ^= m;
455 }
456 switch (left) {
457 case 3: b |= ((u32)end[2]) << 16; /* fall through */
458 case 2: b |= get_unaligned_le16(end); break;
459 case 1: b |= end[0];
460 }
461 HPOSTAMBLE
462}
463EXPORT_SYMBOL(__hsiphash_unaligned);
464#endif
465
466/**
467 * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32
468 * @first: first u32
469 * @key: the hsiphash key
470 */
471u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)
472{
473 HPREAMBLE(4)
474 v3 ^= first;
475 HSIPROUND;
476 v0 ^= first;
477 HPOSTAMBLE
478}
479EXPORT_SYMBOL(hsiphash_1u32);
480
481/**
482 * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32
483 * @first: first u32
484 * @second: second u32
485 * @key: the hsiphash key
486 */
487u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)
488{
489 HPREAMBLE(8)
490 v3 ^= first;
491 HSIPROUND;
492 v0 ^= first;
493 v3 ^= second;
494 HSIPROUND;
495 v0 ^= second;
496 HPOSTAMBLE
497}
498EXPORT_SYMBOL(hsiphash_2u32);
499
500/**
501 * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32
502 * @first: first u32
503 * @second: second u32
504 * @third: third u32
505 * @key: the hsiphash key
506 */
507u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,
508 const hsiphash_key_t *key)
509{
510 HPREAMBLE(12)
511 v3 ^= first;
512 HSIPROUND;
513 v0 ^= first;
514 v3 ^= second;
515 HSIPROUND;
516 v0 ^= second;
517 v3 ^= third;
518 HSIPROUND;
519 v0 ^= third;
520 HPOSTAMBLE
521}
522EXPORT_SYMBOL(hsiphash_3u32);
523
524/**
525 * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32
526 * @first: first u32
527 * @second: second u32
528 * @third: third u32
529 * @forth: forth u32
530 * @key: the hsiphash key
531 */
532u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
533 const u32 forth, const hsiphash_key_t *key)
534{
535 HPREAMBLE(16)
536 v3 ^= first;
537 HSIPROUND;
538 v0 ^= first;
539 v3 ^= second;
540 HSIPROUND;
541 v0 ^= second;
542 v3 ^= third;
543 HSIPROUND;
544 v0 ^= third;
545 v3 ^= forth;
546 HSIPROUND;
547 v0 ^= forth;
548 HPOSTAMBLE
549}
550EXPORT_SYMBOL(hsiphash_4u32);
551#endif