Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 2016 Broadcom
4 */
5
6#include <linux/debugfs.h>
7
8#include "cipher.h"
9#include "util.h"
10
11/* offset of SPU_OFIFO_CTRL register */
12#define SPU_OFIFO_CTRL 0x40
13#define SPU_FIFO_WATERMARK 0x1FF
14
15/**
16 * spu_sg_at_offset() - Find the scatterlist entry at a given distance from the
17 * start of a scatterlist.
18 * @sg: [in] Start of a scatterlist
19 * @skip: [in] Distance from the start of the scatterlist, in bytes
20 * @sge: [out] Scatterlist entry at skip bytes from start
21 * @sge_offset: [out] Number of bytes from start of sge buffer to get to
22 * requested distance.
23 *
24 * Return: 0 if entry found at requested distance
25 * < 0 otherwise
26 */
27int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip,
28 struct scatterlist **sge, unsigned int *sge_offset)
29{
30 /* byte index from start of sg to the end of the previous entry */
31 unsigned int index = 0;
32 /* byte index from start of sg to the end of the current entry */
33 unsigned int next_index;
34
35 next_index = sg->length;
36 while (next_index <= skip) {
37 sg = sg_next(sg);
38 index = next_index;
39 if (!sg)
40 return -EINVAL;
41 next_index += sg->length;
42 }
43
44 *sge_offset = skip - index;
45 *sge = sg;
46 return 0;
47}
48
49/* Copy len bytes of sg data, starting at offset skip, to a dest buffer */
50void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest,
51 unsigned int len, unsigned int skip)
52{
53 size_t copied;
54 unsigned int nents = sg_nents(src);
55
56 copied = sg_pcopy_to_buffer(src, nents, dest, len, skip);
57 if (copied != len) {
58 flow_log("%s copied %u bytes of %u requested. ",
59 __func__, (u32)copied, len);
60 flow_log("sg with %u entries and skip %u\n", nents, skip);
61 }
62}
63
64/*
65 * Copy data into a scatterlist starting at a specified offset in the
66 * scatterlist. Specifically, copy len bytes of data in the buffer src
67 * into the scatterlist dest, starting skip bytes into the scatterlist.
68 */
69void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src,
70 unsigned int len, unsigned int skip)
71{
72 size_t copied;
73 unsigned int nents = sg_nents(dest);
74
75 copied = sg_pcopy_from_buffer(dest, nents, src, len, skip);
76 if (copied != len) {
77 flow_log("%s copied %u bytes of %u requested. ",
78 __func__, (u32)copied, len);
79 flow_log("sg with %u entries and skip %u\n", nents, skip);
80 }
81}
82
83/**
84 * spu_sg_count() - Determine number of elements in scatterlist to provide a
85 * specified number of bytes.
86 * @sg_list: scatterlist to examine
87 * @skip: index of starting point
88 * @nbytes: consider elements of scatterlist until reaching this number of
89 * bytes
90 *
91 * Return: the number of sg entries contributing to nbytes of data
92 */
93int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes)
94{
95 struct scatterlist *sg;
96 int sg_nents = 0;
97 unsigned int offset;
98
99 if (!sg_list)
100 return 0;
101
102 if (spu_sg_at_offset(sg_list, skip, &sg, &offset) < 0)
103 return 0;
104
105 while (sg && (nbytes > 0)) {
106 sg_nents++;
107 nbytes -= (sg->length - offset);
108 offset = 0;
109 sg = sg_next(sg);
110 }
111 return sg_nents;
112}
113
114/**
115 * spu_msg_sg_add() - Copy scatterlist entries from one sg to another, up to a
116 * given length.
117 * @to_sg: scatterlist to copy to
118 * @from_sg: scatterlist to copy from
119 * @from_skip: number of bytes to skip in from_sg. Non-zero when previous
120 * request included part of the buffer in entry in from_sg.
121 * Assumes from_skip < from_sg->length.
122 * @from_nents: number of entries in from_sg
123 * @length: number of bytes to copy. may reach this limit before exhausting
124 * from_sg.
125 *
126 * Copies the entries themselves, not the data in the entries. Assumes to_sg has
127 * enough entries. Does not limit the size of an individual buffer in to_sg.
128 *
129 * to_sg, from_sg, skip are all updated to end of copy
130 *
131 * Return: Number of bytes copied
132 */
133u32 spu_msg_sg_add(struct scatterlist **to_sg,
134 struct scatterlist **from_sg, u32 *from_skip,
135 u8 from_nents, u32 length)
136{
137 struct scatterlist *sg; /* an entry in from_sg */
138 struct scatterlist *to = *to_sg;
139 struct scatterlist *from = *from_sg;
140 u32 skip = *from_skip;
141 u32 offset;
142 int i;
143 u32 entry_len = 0;
144 u32 frag_len = 0; /* length of entry added to to_sg */
145 u32 copied = 0; /* number of bytes copied so far */
146
147 if (length == 0)
148 return 0;
149
150 for_each_sg(from, sg, from_nents, i) {
151 /* number of bytes in this from entry not yet used */
152 entry_len = sg->length - skip;
153 frag_len = min(entry_len, length - copied);
154 offset = sg->offset + skip;
155 if (frag_len)
156 sg_set_page(to++, sg_page(sg), frag_len, offset);
157 copied += frag_len;
158 if (copied == entry_len) {
159 /* used up all of from entry */
160 skip = 0; /* start at beginning of next entry */
161 }
162 if (copied == length)
163 break;
164 }
165 *to_sg = to;
166 *from_sg = sg;
167 if (frag_len < entry_len)
168 *from_skip = skip + frag_len;
169 else
170 *from_skip = 0;
171
172 return copied;
173}
174
175void add_to_ctr(u8 *ctr_pos, unsigned int increment)
176{
177 __be64 *high_be = (__be64 *)ctr_pos;
178 __be64 *low_be = high_be + 1;
179 u64 orig_low = __be64_to_cpu(*low_be);
180 u64 new_low = orig_low + (u64)increment;
181
182 *low_be = __cpu_to_be64(new_low);
183 if (new_low < orig_low)
184 /* there was a carry from the low 8 bytes */
185 *high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1);
186}
187
188struct sdesc {
189 struct shash_desc shash;
190 char ctx[];
191};
192
193/**
194 * do_shash() - Do a synchronous hash operation in software
195 * @name: The name of the hash algorithm
196 * @result: Buffer where digest is to be written
197 * @data1: First part of data to hash. May be NULL.
198 * @data1_len: Length of data1, in bytes
199 * @data2: Second part of data to hash. May be NULL.
200 * @data2_len: Length of data2, in bytes
201 * @key: Key (if keyed hash)
202 * @key_len: Length of key, in bytes (or 0 if non-keyed hash)
203 *
204 * Note that the crypto API will not select this driver's own transform because
205 * this driver only registers asynchronous algos.
206 *
207 * Return: 0 if hash successfully stored in result
208 * < 0 otherwise
209 */
210int do_shash(unsigned char *name, unsigned char *result,
211 const u8 *data1, unsigned int data1_len,
212 const u8 *data2, unsigned int data2_len,
213 const u8 *key, unsigned int key_len)
214{
215 int rc;
216 unsigned int size;
217 struct crypto_shash *hash;
218 struct sdesc *sdesc;
219
220 hash = crypto_alloc_shash(name, 0, 0);
221 if (IS_ERR(hash)) {
222 rc = PTR_ERR(hash);
223 pr_err("%s: Crypto %s allocation error %d\n", __func__, name, rc);
224 return rc;
225 }
226
227 size = sizeof(struct shash_desc) + crypto_shash_descsize(hash);
228 sdesc = kmalloc(size, GFP_KERNEL);
229 if (!sdesc) {
230 rc = -ENOMEM;
231 goto do_shash_err;
232 }
233 sdesc->shash.tfm = hash;
234
235 if (key_len > 0) {
236 rc = crypto_shash_setkey(hash, key, key_len);
237 if (rc) {
238 pr_err("%s: Could not setkey %s shash\n", __func__, name);
239 goto do_shash_err;
240 }
241 }
242
243 rc = crypto_shash_init(&sdesc->shash);
244 if (rc) {
245 pr_err("%s: Could not init %s shash\n", __func__, name);
246 goto do_shash_err;
247 }
248 rc = crypto_shash_update(&sdesc->shash, data1, data1_len);
249 if (rc) {
250 pr_err("%s: Could not update1\n", __func__);
251 goto do_shash_err;
252 }
253 if (data2 && data2_len) {
254 rc = crypto_shash_update(&sdesc->shash, data2, data2_len);
255 if (rc) {
256 pr_err("%s: Could not update2\n", __func__);
257 goto do_shash_err;
258 }
259 }
260 rc = crypto_shash_final(&sdesc->shash, result);
261 if (rc)
262 pr_err("%s: Could not generate %s hash\n", __func__, name);
263
264do_shash_err:
265 crypto_free_shash(hash);
266 kfree(sdesc);
267
268 return rc;
269}
270
271#ifdef DEBUG
272/* Dump len bytes of a scatterlist starting at skip bytes into the sg */
273void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len)
274{
275 u8 dbuf[16];
276 unsigned int idx = skip;
277 unsigned int num_out = 0; /* number of bytes dumped so far */
278 unsigned int count;
279
280 if (packet_debug_logging) {
281 while (num_out < len) {
282 count = (len - num_out > 16) ? 16 : len - num_out;
283 sg_copy_part_to_buf(sg, dbuf, count, idx);
284 num_out += count;
285 print_hex_dump(KERN_ALERT, " sg: ", DUMP_PREFIX_NONE,
286 4, 1, dbuf, count, false);
287 idx += 16;
288 }
289 }
290 if (debug_logging_sleep)
291 msleep(debug_logging_sleep);
292}
293#endif
294
295/* Returns the name for a given cipher alg/mode */
296char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode)
297{
298 switch (alg) {
299 case CIPHER_ALG_RC4:
300 return "rc4";
301 case CIPHER_ALG_AES:
302 switch (mode) {
303 case CIPHER_MODE_CBC:
304 return "cbc(aes)";
305 case CIPHER_MODE_ECB:
306 return "ecb(aes)";
307 case CIPHER_MODE_OFB:
308 return "ofb(aes)";
309 case CIPHER_MODE_CFB:
310 return "cfb(aes)";
311 case CIPHER_MODE_CTR:
312 return "ctr(aes)";
313 case CIPHER_MODE_XTS:
314 return "xts(aes)";
315 case CIPHER_MODE_GCM:
316 return "gcm(aes)";
317 default:
318 return "aes";
319 }
320 break;
321 case CIPHER_ALG_DES:
322 switch (mode) {
323 case CIPHER_MODE_CBC:
324 return "cbc(des)";
325 case CIPHER_MODE_ECB:
326 return "ecb(des)";
327 case CIPHER_MODE_CTR:
328 return "ctr(des)";
329 default:
330 return "des";
331 }
332 break;
333 case CIPHER_ALG_3DES:
334 switch (mode) {
335 case CIPHER_MODE_CBC:
336 return "cbc(des3_ede)";
337 case CIPHER_MODE_ECB:
338 return "ecb(des3_ede)";
339 case CIPHER_MODE_CTR:
340 return "ctr(des3_ede)";
341 default:
342 return "3des";
343 }
344 break;
345 default:
346 return "other";
347 }
348}
349
350static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf,
351 size_t count, loff_t *offp)
352{
353 struct bcm_device_private *ipriv;
354 char *buf;
355 ssize_t ret, out_offset, out_count;
356 int i;
357 u32 fifo_len;
358 u32 spu_ofifo_ctrl;
359 u32 alg;
360 u32 mode;
361 u32 op_cnt;
362
363 out_count = 2048;
364
365 buf = kmalloc(out_count, GFP_KERNEL);
366 if (!buf)
367 return -ENOMEM;
368
369 ipriv = filp->private_data;
370 out_offset = 0;
371 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
372 "Number of SPUs.........%u\n",
373 ipriv->spu.num_spu);
374 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
375 "Current sessions.......%u\n",
376 atomic_read(&ipriv->session_count));
377 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
378 "Session count..........%u\n",
379 atomic_read(&ipriv->stream_count));
380 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
381 "Cipher setkey..........%u\n",
382 atomic_read(&ipriv->setkey_cnt[SPU_OP_CIPHER]));
383 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
384 "Cipher Ops.............%u\n",
385 atomic_read(&ipriv->op_counts[SPU_OP_CIPHER]));
386 for (alg = 0; alg < CIPHER_ALG_LAST; alg++) {
387 for (mode = 0; mode < CIPHER_MODE_LAST; mode++) {
388 op_cnt = atomic_read(&ipriv->cipher_cnt[alg][mode]);
389 if (op_cnt) {
390 out_offset += scnprintf(buf + out_offset,
391 out_count - out_offset,
392 " %-13s%11u\n",
393 spu_alg_name(alg, mode), op_cnt);
394 }
395 }
396 }
397 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
398 "Hash Ops...............%u\n",
399 atomic_read(&ipriv->op_counts[SPU_OP_HASH]));
400 for (alg = 0; alg < HASH_ALG_LAST; alg++) {
401 op_cnt = atomic_read(&ipriv->hash_cnt[alg]);
402 if (op_cnt) {
403 out_offset += scnprintf(buf + out_offset,
404 out_count - out_offset,
405 " %-13s%11u\n",
406 hash_alg_name[alg], op_cnt);
407 }
408 }
409 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
410 "HMAC setkey............%u\n",
411 atomic_read(&ipriv->setkey_cnt[SPU_OP_HMAC]));
412 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
413 "HMAC Ops...............%u\n",
414 atomic_read(&ipriv->op_counts[SPU_OP_HMAC]));
415 for (alg = 0; alg < HASH_ALG_LAST; alg++) {
416 op_cnt = atomic_read(&ipriv->hmac_cnt[alg]);
417 if (op_cnt) {
418 out_offset += scnprintf(buf + out_offset,
419 out_count - out_offset,
420 " %-13s%11u\n",
421 hash_alg_name[alg], op_cnt);
422 }
423 }
424 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
425 "AEAD setkey............%u\n",
426 atomic_read(&ipriv->setkey_cnt[SPU_OP_AEAD]));
427
428 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
429 "AEAD Ops...............%u\n",
430 atomic_read(&ipriv->op_counts[SPU_OP_AEAD]));
431 for (alg = 0; alg < AEAD_TYPE_LAST; alg++) {
432 op_cnt = atomic_read(&ipriv->aead_cnt[alg]);
433 if (op_cnt) {
434 out_offset += scnprintf(buf + out_offset,
435 out_count - out_offset,
436 " %-13s%11u\n",
437 aead_alg_name[alg], op_cnt);
438 }
439 }
440 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
441 "Bytes of req data......%llu\n",
442 (u64)atomic64_read(&ipriv->bytes_out));
443 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
444 "Bytes of resp data.....%llu\n",
445 (u64)atomic64_read(&ipriv->bytes_in));
446 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
447 "Mailbox full...........%u\n",
448 atomic_read(&ipriv->mb_no_spc));
449 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
450 "Mailbox send failures..%u\n",
451 atomic_read(&ipriv->mb_send_fail));
452 out_offset += scnprintf(buf + out_offset, out_count - out_offset,
453 "Check ICV errors.......%u\n",
454 atomic_read(&ipriv->bad_icv));
455 if (ipriv->spu.spu_type == SPU_TYPE_SPUM)
456 for (i = 0; i < ipriv->spu.num_spu; i++) {
457 spu_ofifo_ctrl = ioread32(ipriv->spu.reg_vbase[i] +
458 SPU_OFIFO_CTRL);
459 fifo_len = spu_ofifo_ctrl & SPU_FIFO_WATERMARK;
460 out_offset += scnprintf(buf + out_offset,
461 out_count - out_offset,
462 "SPU %d output FIFO high water.....%u\n",
463 i, fifo_len);
464 }
465
466 if (out_offset > out_count)
467 out_offset = out_count;
468
469 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
470 kfree(buf);
471 return ret;
472}
473
474static const struct file_operations spu_debugfs_stats = {
475 .owner = THIS_MODULE,
476 .open = simple_open,
477 .read = spu_debugfs_read,
478};
479
480/*
481 * Create the debug FS directories. If the top-level directory has not yet
482 * been created, create it now. Create a stats file in this directory for
483 * a SPU.
484 */
485void spu_setup_debugfs(void)
486{
487 if (!debugfs_initialized())
488 return;
489
490 if (!iproc_priv.debugfs_dir)
491 iproc_priv.debugfs_dir = debugfs_create_dir(KBUILD_MODNAME,
492 NULL);
493
494 if (!iproc_priv.debugfs_stats)
495 /* Create file with permissions S_IRUSR */
496 debugfs_create_file("stats", 0400, iproc_priv.debugfs_dir,
497 &iproc_priv, &spu_debugfs_stats);
498}
499
500void spu_free_debugfs(void)
501{
502 debugfs_remove_recursive(iproc_priv.debugfs_dir);
503 iproc_priv.debugfs_dir = NULL;
504}
505
506/**
507 * format_value_ccm() - Format a value into a buffer, using a specified number
508 * of bytes (i.e. maybe writing value X into a 4 byte
509 * buffer, or maybe into a 12 byte buffer), as per the
510 * SPU CCM spec.
511 *
512 * @val: value to write (up to max of unsigned int)
513 * @buf: (pointer to) buffer to write the value
514 * @len: number of bytes to use (0 to 255)
515 *
516 */
517void format_value_ccm(unsigned int val, u8 *buf, u8 len)
518{
519 int i;
520
521 /* First clear full output buffer */
522 memset(buf, 0, len);
523
524 /* Then, starting from right side, fill in with data */
525 for (i = 0; i < len; i++) {
526 buf[len - i - 1] = (val >> (8 * i)) & 0xff;
527 if (i >= 3)
528 break; /* Only handle up to 32 bits of 'val' */
529 }
530}
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#include <linux/debugfs.h>
18
19#include "cipher.h"
20#include "util.h"
21
22/* offset of SPU_OFIFO_CTRL register */
23#define SPU_OFIFO_CTRL 0x40
24#define SPU_FIFO_WATERMARK 0x1FF
25
26/**
27 * spu_sg_at_offset() - Find the scatterlist entry at a given distance from the
28 * start of a scatterlist.
29 * @sg: [in] Start of a scatterlist
30 * @skip: [in] Distance from the start of the scatterlist, in bytes
31 * @sge: [out] Scatterlist entry at skip bytes from start
32 * @sge_offset: [out] Number of bytes from start of sge buffer to get to
33 * requested distance.
34 *
35 * Return: 0 if entry found at requested distance
36 * < 0 otherwise
37 */
38int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip,
39 struct scatterlist **sge, unsigned int *sge_offset)
40{
41 /* byte index from start of sg to the end of the previous entry */
42 unsigned int index = 0;
43 /* byte index from start of sg to the end of the current entry */
44 unsigned int next_index;
45
46 next_index = sg->length;
47 while (next_index <= skip) {
48 sg = sg_next(sg);
49 index = next_index;
50 if (!sg)
51 return -EINVAL;
52 next_index += sg->length;
53 }
54
55 *sge_offset = skip - index;
56 *sge = sg;
57 return 0;
58}
59
60/* Copy len bytes of sg data, starting at offset skip, to a dest buffer */
61void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest,
62 unsigned int len, unsigned int skip)
63{
64 size_t copied;
65 unsigned int nents = sg_nents(src);
66
67 copied = sg_pcopy_to_buffer(src, nents, dest, len, skip);
68 if (copied != len) {
69 flow_log("%s copied %u bytes of %u requested. ",
70 __func__, (u32)copied, len);
71 flow_log("sg with %u entries and skip %u\n", nents, skip);
72 }
73}
74
75/*
76 * Copy data into a scatterlist starting at a specified offset in the
77 * scatterlist. Specifically, copy len bytes of data in the buffer src
78 * into the scatterlist dest, starting skip bytes into the scatterlist.
79 */
80void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src,
81 unsigned int len, unsigned int skip)
82{
83 size_t copied;
84 unsigned int nents = sg_nents(dest);
85
86 copied = sg_pcopy_from_buffer(dest, nents, src, len, skip);
87 if (copied != len) {
88 flow_log("%s copied %u bytes of %u requested. ",
89 __func__, (u32)copied, len);
90 flow_log("sg with %u entries and skip %u\n", nents, skip);
91 }
92}
93
94/**
95 * spu_sg_count() - Determine number of elements in scatterlist to provide a
96 * specified number of bytes.
97 * @sg_list: scatterlist to examine
98 * @skip: index of starting point
99 * @nbytes: consider elements of scatterlist until reaching this number of
100 * bytes
101 *
102 * Return: the number of sg entries contributing to nbytes of data
103 */
104int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes)
105{
106 struct scatterlist *sg;
107 int sg_nents = 0;
108 unsigned int offset;
109
110 if (!sg_list)
111 return 0;
112
113 if (spu_sg_at_offset(sg_list, skip, &sg, &offset) < 0)
114 return 0;
115
116 while (sg && (nbytes > 0)) {
117 sg_nents++;
118 nbytes -= (sg->length - offset);
119 offset = 0;
120 sg = sg_next(sg);
121 }
122 return sg_nents;
123}
124
125/**
126 * spu_msg_sg_add() - Copy scatterlist entries from one sg to another, up to a
127 * given length.
128 * @to_sg: scatterlist to copy to
129 * @from_sg: scatterlist to copy from
130 * @from_skip: number of bytes to skip in from_sg. Non-zero when previous
131 * request included part of the buffer in entry in from_sg.
132 * Assumes from_skip < from_sg->length.
133 * @from_nents number of entries in from_sg
134 * @length number of bytes to copy. may reach this limit before exhausting
135 * from_sg.
136 *
137 * Copies the entries themselves, not the data in the entries. Assumes to_sg has
138 * enough entries. Does not limit the size of an individual buffer in to_sg.
139 *
140 * to_sg, from_sg, skip are all updated to end of copy
141 *
142 * Return: Number of bytes copied
143 */
144u32 spu_msg_sg_add(struct scatterlist **to_sg,
145 struct scatterlist **from_sg, u32 *from_skip,
146 u8 from_nents, u32 length)
147{
148 struct scatterlist *sg; /* an entry in from_sg */
149 struct scatterlist *to = *to_sg;
150 struct scatterlist *from = *from_sg;
151 u32 skip = *from_skip;
152 u32 offset;
153 int i;
154 u32 entry_len = 0;
155 u32 frag_len = 0; /* length of entry added to to_sg */
156 u32 copied = 0; /* number of bytes copied so far */
157
158 if (length == 0)
159 return 0;
160
161 for_each_sg(from, sg, from_nents, i) {
162 /* number of bytes in this from entry not yet used */
163 entry_len = sg->length - skip;
164 frag_len = min(entry_len, length - copied);
165 offset = sg->offset + skip;
166 if (frag_len)
167 sg_set_page(to++, sg_page(sg), frag_len, offset);
168 copied += frag_len;
169 if (copied == entry_len) {
170 /* used up all of from entry */
171 skip = 0; /* start at beginning of next entry */
172 }
173 if (copied == length)
174 break;
175 }
176 *to_sg = to;
177 *from_sg = sg;
178 if (frag_len < entry_len)
179 *from_skip = skip + frag_len;
180 else
181 *from_skip = 0;
182
183 return copied;
184}
185
186void add_to_ctr(u8 *ctr_pos, unsigned int increment)
187{
188 __be64 *high_be = (__be64 *)ctr_pos;
189 __be64 *low_be = high_be + 1;
190 u64 orig_low = __be64_to_cpu(*low_be);
191 u64 new_low = orig_low + (u64)increment;
192
193 *low_be = __cpu_to_be64(new_low);
194 if (new_low < orig_low)
195 /* there was a carry from the low 8 bytes */
196 *high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1);
197}
198
199struct sdesc {
200 struct shash_desc shash;
201 char ctx[];
202};
203
204/* do a synchronous decrypt operation */
205int do_decrypt(char *alg_name,
206 void *key_ptr, unsigned int key_len,
207 void *iv_ptr, void *src_ptr, void *dst_ptr,
208 unsigned int block_len)
209{
210 struct scatterlist sg_in[1], sg_out[1];
211 struct crypto_blkcipher *tfm =
212 crypto_alloc_blkcipher(alg_name, 0, CRYPTO_ALG_ASYNC);
213 struct blkcipher_desc desc = {.tfm = tfm, .flags = 0 };
214 int ret = 0;
215 void *iv;
216 int ivsize;
217
218 flow_log("%s() name:%s block_len:%u\n", __func__, alg_name, block_len);
219
220 if (IS_ERR(tfm))
221 return PTR_ERR(tfm);
222
223 crypto_blkcipher_setkey((void *)tfm, key_ptr, key_len);
224
225 sg_init_table(sg_in, 1);
226 sg_set_buf(sg_in, src_ptr, block_len);
227
228 sg_init_table(sg_out, 1);
229 sg_set_buf(sg_out, dst_ptr, block_len);
230
231 iv = crypto_blkcipher_crt(tfm)->iv;
232 ivsize = crypto_blkcipher_ivsize(tfm);
233 memcpy(iv, iv_ptr, ivsize);
234
235 ret = crypto_blkcipher_decrypt(&desc, sg_out, sg_in, block_len);
236 crypto_free_blkcipher(tfm);
237
238 if (ret < 0)
239 pr_err("aes_decrypt failed %d\n", ret);
240
241 return ret;
242}
243
244/**
245 * do_shash() - Do a synchronous hash operation in software
246 * @name: The name of the hash algorithm
247 * @result: Buffer where digest is to be written
248 * @data1: First part of data to hash. May be NULL.
249 * @data1_len: Length of data1, in bytes
250 * @data2: Second part of data to hash. May be NULL.
251 * @data2_len: Length of data2, in bytes
252 * @key: Key (if keyed hash)
253 * @key_len: Length of key, in bytes (or 0 if non-keyed hash)
254 *
255 * Note that the crypto API will not select this driver's own transform because
256 * this driver only registers asynchronous algos.
257 *
258 * Return: 0 if hash successfully stored in result
259 * < 0 otherwise
260 */
261int do_shash(unsigned char *name, unsigned char *result,
262 const u8 *data1, unsigned int data1_len,
263 const u8 *data2, unsigned int data2_len,
264 const u8 *key, unsigned int key_len)
265{
266 int rc;
267 unsigned int size;
268 struct crypto_shash *hash;
269 struct sdesc *sdesc;
270
271 hash = crypto_alloc_shash(name, 0, 0);
272 if (IS_ERR(hash)) {
273 rc = PTR_ERR(hash);
274 pr_err("%s: Crypto %s allocation error %d\n", __func__, name, rc);
275 return rc;
276 }
277
278 size = sizeof(struct shash_desc) + crypto_shash_descsize(hash);
279 sdesc = kmalloc(size, GFP_KERNEL);
280 if (!sdesc) {
281 rc = -ENOMEM;
282 goto do_shash_err;
283 }
284 sdesc->shash.tfm = hash;
285 sdesc->shash.flags = 0x0;
286
287 if (key_len > 0) {
288 rc = crypto_shash_setkey(hash, key, key_len);
289 if (rc) {
290 pr_err("%s: Could not setkey %s shash\n", __func__, name);
291 goto do_shash_err;
292 }
293 }
294
295 rc = crypto_shash_init(&sdesc->shash);
296 if (rc) {
297 pr_err("%s: Could not init %s shash\n", __func__, name);
298 goto do_shash_err;
299 }
300 rc = crypto_shash_update(&sdesc->shash, data1, data1_len);
301 if (rc) {
302 pr_err("%s: Could not update1\n", __func__);
303 goto do_shash_err;
304 }
305 if (data2 && data2_len) {
306 rc = crypto_shash_update(&sdesc->shash, data2, data2_len);
307 if (rc) {
308 pr_err("%s: Could not update2\n", __func__);
309 goto do_shash_err;
310 }
311 }
312 rc = crypto_shash_final(&sdesc->shash, result);
313 if (rc)
314 pr_err("%s: Could not generate %s hash\n", __func__, name);
315
316do_shash_err:
317 crypto_free_shash(hash);
318 kfree(sdesc);
319
320 return rc;
321}
322
323/* Dump len bytes of a scatterlist starting at skip bytes into the sg */
324void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len)
325{
326 u8 dbuf[16];
327 unsigned int idx = skip;
328 unsigned int num_out = 0; /* number of bytes dumped so far */
329 unsigned int count;
330
331 if (packet_debug_logging) {
332 while (num_out < len) {
333 count = (len - num_out > 16) ? 16 : len - num_out;
334 sg_copy_part_to_buf(sg, dbuf, count, idx);
335 num_out += count;
336 print_hex_dump(KERN_ALERT, " sg: ", DUMP_PREFIX_NONE,
337 4, 1, dbuf, count, false);
338 idx += 16;
339 }
340 }
341 if (debug_logging_sleep)
342 msleep(debug_logging_sleep);
343}
344
345/* Returns the name for a given cipher alg/mode */
346char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode)
347{
348 switch (alg) {
349 case CIPHER_ALG_RC4:
350 return "rc4";
351 case CIPHER_ALG_AES:
352 switch (mode) {
353 case CIPHER_MODE_CBC:
354 return "cbc(aes)";
355 case CIPHER_MODE_ECB:
356 return "ecb(aes)";
357 case CIPHER_MODE_OFB:
358 return "ofb(aes)";
359 case CIPHER_MODE_CFB:
360 return "cfb(aes)";
361 case CIPHER_MODE_CTR:
362 return "ctr(aes)";
363 case CIPHER_MODE_XTS:
364 return "xts(aes)";
365 case CIPHER_MODE_GCM:
366 return "gcm(aes)";
367 default:
368 return "aes";
369 }
370 break;
371 case CIPHER_ALG_DES:
372 switch (mode) {
373 case CIPHER_MODE_CBC:
374 return "cbc(des)";
375 case CIPHER_MODE_ECB:
376 return "ecb(des)";
377 case CIPHER_MODE_CTR:
378 return "ctr(des)";
379 default:
380 return "des";
381 }
382 break;
383 case CIPHER_ALG_3DES:
384 switch (mode) {
385 case CIPHER_MODE_CBC:
386 return "cbc(des3_ede)";
387 case CIPHER_MODE_ECB:
388 return "ecb(des3_ede)";
389 case CIPHER_MODE_CTR:
390 return "ctr(des3_ede)";
391 default:
392 return "3des";
393 }
394 break;
395 default:
396 return "other";
397 }
398}
399
400static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf,
401 size_t count, loff_t *offp)
402{
403 struct device_private *ipriv;
404 char *buf;
405 ssize_t ret, out_offset, out_count;
406 int i;
407 u32 fifo_len;
408 u32 spu_ofifo_ctrl;
409 u32 alg;
410 u32 mode;
411 u32 op_cnt;
412
413 out_count = 2048;
414
415 buf = kmalloc(out_count, GFP_KERNEL);
416 if (!buf)
417 return -ENOMEM;
418
419 ipriv = filp->private_data;
420 out_offset = 0;
421 out_offset += snprintf(buf + out_offset, out_count - out_offset,
422 "Number of SPUs.........%u\n",
423 ipriv->spu.num_spu);
424 out_offset += snprintf(buf + out_offset, out_count - out_offset,
425 "Current sessions.......%u\n",
426 atomic_read(&ipriv->session_count));
427 out_offset += snprintf(buf + out_offset, out_count - out_offset,
428 "Session count..........%u\n",
429 atomic_read(&ipriv->stream_count));
430 out_offset += snprintf(buf + out_offset, out_count - out_offset,
431 "Cipher setkey..........%u\n",
432 atomic_read(&ipriv->setkey_cnt[SPU_OP_CIPHER]));
433 out_offset += snprintf(buf + out_offset, out_count - out_offset,
434 "Cipher Ops.............%u\n",
435 atomic_read(&ipriv->op_counts[SPU_OP_CIPHER]));
436 for (alg = 0; alg < CIPHER_ALG_LAST; alg++) {
437 for (mode = 0; mode < CIPHER_MODE_LAST; mode++) {
438 op_cnt = atomic_read(&ipriv->cipher_cnt[alg][mode]);
439 if (op_cnt) {
440 out_offset += snprintf(buf + out_offset,
441 out_count - out_offset,
442 " %-13s%11u\n",
443 spu_alg_name(alg, mode), op_cnt);
444 }
445 }
446 }
447 out_offset += snprintf(buf + out_offset, out_count - out_offset,
448 "Hash Ops...............%u\n",
449 atomic_read(&ipriv->op_counts[SPU_OP_HASH]));
450 for (alg = 0; alg < HASH_ALG_LAST; alg++) {
451 op_cnt = atomic_read(&ipriv->hash_cnt[alg]);
452 if (op_cnt) {
453 out_offset += snprintf(buf + out_offset,
454 out_count - out_offset,
455 " %-13s%11u\n",
456 hash_alg_name[alg], op_cnt);
457 }
458 }
459 out_offset += snprintf(buf + out_offset, out_count - out_offset,
460 "HMAC setkey............%u\n",
461 atomic_read(&ipriv->setkey_cnt[SPU_OP_HMAC]));
462 out_offset += snprintf(buf + out_offset, out_count - out_offset,
463 "HMAC Ops...............%u\n",
464 atomic_read(&ipriv->op_counts[SPU_OP_HMAC]));
465 for (alg = 0; alg < HASH_ALG_LAST; alg++) {
466 op_cnt = atomic_read(&ipriv->hmac_cnt[alg]);
467 if (op_cnt) {
468 out_offset += snprintf(buf + out_offset,
469 out_count - out_offset,
470 " %-13s%11u\n",
471 hash_alg_name[alg], op_cnt);
472 }
473 }
474 out_offset += snprintf(buf + out_offset, out_count - out_offset,
475 "AEAD setkey............%u\n",
476 atomic_read(&ipriv->setkey_cnt[SPU_OP_AEAD]));
477
478 out_offset += snprintf(buf + out_offset, out_count - out_offset,
479 "AEAD Ops...............%u\n",
480 atomic_read(&ipriv->op_counts[SPU_OP_AEAD]));
481 for (alg = 0; alg < AEAD_TYPE_LAST; alg++) {
482 op_cnt = atomic_read(&ipriv->aead_cnt[alg]);
483 if (op_cnt) {
484 out_offset += snprintf(buf + out_offset,
485 out_count - out_offset,
486 " %-13s%11u\n",
487 aead_alg_name[alg], op_cnt);
488 }
489 }
490 out_offset += snprintf(buf + out_offset, out_count - out_offset,
491 "Bytes of req data......%llu\n",
492 (u64)atomic64_read(&ipriv->bytes_out));
493 out_offset += snprintf(buf + out_offset, out_count - out_offset,
494 "Bytes of resp data.....%llu\n",
495 (u64)atomic64_read(&ipriv->bytes_in));
496 out_offset += snprintf(buf + out_offset, out_count - out_offset,
497 "Mailbox full...........%u\n",
498 atomic_read(&ipriv->mb_no_spc));
499 out_offset += snprintf(buf + out_offset, out_count - out_offset,
500 "Mailbox send failures..%u\n",
501 atomic_read(&ipriv->mb_send_fail));
502 out_offset += snprintf(buf + out_offset, out_count - out_offset,
503 "Check ICV errors.......%u\n",
504 atomic_read(&ipriv->bad_icv));
505 if (ipriv->spu.spu_type == SPU_TYPE_SPUM)
506 for (i = 0; i < ipriv->spu.num_spu; i++) {
507 spu_ofifo_ctrl = ioread32(ipriv->spu.reg_vbase[i] +
508 SPU_OFIFO_CTRL);
509 fifo_len = spu_ofifo_ctrl & SPU_FIFO_WATERMARK;
510 out_offset += snprintf(buf + out_offset,
511 out_count - out_offset,
512 "SPU %d output FIFO high water.....%u\n",
513 i, fifo_len);
514 }
515
516 if (out_offset > out_count)
517 out_offset = out_count;
518
519 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
520 kfree(buf);
521 return ret;
522}
523
524static const struct file_operations spu_debugfs_stats = {
525 .owner = THIS_MODULE,
526 .open = simple_open,
527 .read = spu_debugfs_read,
528};
529
530/*
531 * Create the debug FS directories. If the top-level directory has not yet
532 * been created, create it now. Create a stats file in this directory for
533 * a SPU.
534 */
535void spu_setup_debugfs(void)
536{
537 if (!debugfs_initialized())
538 return;
539
540 if (!iproc_priv.debugfs_dir)
541 iproc_priv.debugfs_dir = debugfs_create_dir(KBUILD_MODNAME,
542 NULL);
543
544 if (!iproc_priv.debugfs_stats)
545 /* Create file with permissions S_IRUSR */
546 debugfs_create_file("stats", 0400, iproc_priv.debugfs_dir,
547 &iproc_priv, &spu_debugfs_stats);
548}
549
550void spu_free_debugfs(void)
551{
552 debugfs_remove_recursive(iproc_priv.debugfs_dir);
553 iproc_priv.debugfs_dir = NULL;
554}
555
556/**
557 * format_value_ccm() - Format a value into a buffer, using a specified number
558 * of bytes (i.e. maybe writing value X into a 4 byte
559 * buffer, or maybe into a 12 byte buffer), as per the
560 * SPU CCM spec.
561 *
562 * @val: value to write (up to max of unsigned int)
563 * @buf: (pointer to) buffer to write the value
564 * @len: number of bytes to use (0 to 255)
565 *
566 */
567void format_value_ccm(unsigned int val, u8 *buf, u8 len)
568{
569 int i;
570
571 /* First clear full output buffer */
572 memset(buf, 0, len);
573
574 /* Then, starting from right side, fill in with data */
575 for (i = 0; i < len; i++) {
576 buf[len - i - 1] = (val >> (8 * i)) & 0xff;
577 if (i >= 3)
578 break; /* Only handle up to 32 bits of 'val' */
579 }
580}