Loading...
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/pci.h>
16#include <linux/kthread.h>
17#include <linux/dma-mapping.h>
18#include <linux/interrupt.h>
19#include <linux/compiler.h>
20#include <linux/ccp.h>
21
22#include "ccp-dev.h"
23
24/* Allocate the requested number of contiguous LSB slots
25 * from the LSB bitmap. Look in the private range for this
26 * queue first; failing that, check the public area.
27 * If no space is available, wait around.
28 * Return: first slot number
29 */
30static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
31{
32 struct ccp_device *ccp;
33 int start;
34
35 /* First look at the map for the queue */
36 if (cmd_q->lsb >= 0) {
37 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
38 LSB_SIZE,
39 0, count, 0);
40 if (start < LSB_SIZE) {
41 bitmap_set(cmd_q->lsbmap, start, count);
42 return start + cmd_q->lsb * LSB_SIZE;
43 }
44 }
45
46 /* No joy; try to get an entry from the shared blocks */
47 ccp = cmd_q->ccp;
48 for (;;) {
49 mutex_lock(&ccp->sb_mutex);
50
51 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
52 MAX_LSB_CNT * LSB_SIZE,
53 0,
54 count, 0);
55 if (start <= MAX_LSB_CNT * LSB_SIZE) {
56 bitmap_set(ccp->lsbmap, start, count);
57
58 mutex_unlock(&ccp->sb_mutex);
59 return start;
60 }
61
62 ccp->sb_avail = 0;
63
64 mutex_unlock(&ccp->sb_mutex);
65
66 /* Wait for KSB entries to become available */
67 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
68 return 0;
69 }
70}
71
72/* Free a number of LSB slots from the bitmap, starting at
73 * the indicated starting slot number.
74 */
75static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
76 unsigned int count)
77{
78 if (!start)
79 return;
80
81 if (cmd_q->lsb == start) {
82 /* An entry from the private LSB */
83 bitmap_clear(cmd_q->lsbmap, start, count);
84 } else {
85 /* From the shared LSBs */
86 struct ccp_device *ccp = cmd_q->ccp;
87
88 mutex_lock(&ccp->sb_mutex);
89 bitmap_clear(ccp->lsbmap, start, count);
90 ccp->sb_avail = 1;
91 mutex_unlock(&ccp->sb_mutex);
92 wake_up_interruptible_all(&ccp->sb_queue);
93 }
94}
95
96/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
97union ccp_function {
98 struct {
99 u16 size:7;
100 u16 encrypt:1;
101 u16 mode:5;
102 u16 type:2;
103 } aes;
104 struct {
105 u16 size:7;
106 u16 encrypt:1;
107 u16 rsvd:5;
108 u16 type:2;
109 } aes_xts;
110 struct {
111 u16 rsvd1:10;
112 u16 type:4;
113 u16 rsvd2:1;
114 } sha;
115 struct {
116 u16 mode:3;
117 u16 size:12;
118 } rsa;
119 struct {
120 u16 byteswap:2;
121 u16 bitwise:3;
122 u16 reflect:2;
123 u16 rsvd:8;
124 } pt;
125 struct {
126 u16 rsvd:13;
127 } zlib;
128 struct {
129 u16 size:10;
130 u16 type:2;
131 u16 mode:3;
132 } ecc;
133 u16 raw;
134};
135
136#define CCP_AES_SIZE(p) ((p)->aes.size)
137#define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
138#define CCP_AES_MODE(p) ((p)->aes.mode)
139#define CCP_AES_TYPE(p) ((p)->aes.type)
140#define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
141#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
142#define CCP_SHA_TYPE(p) ((p)->sha.type)
143#define CCP_RSA_SIZE(p) ((p)->rsa.size)
144#define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
145#define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
146#define CCP_ECC_MODE(p) ((p)->ecc.mode)
147#define CCP_ECC_AFFINE(p) ((p)->ecc.one)
148
149/* Word 0 */
150#define CCP5_CMD_DW0(p) ((p)->dw0)
151#define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
152#define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
153#define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
154#define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
155#define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
156#define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
157#define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
158
159/* Word 1 */
160#define CCP5_CMD_DW1(p) ((p)->length)
161#define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
162
163/* Word 2 */
164#define CCP5_CMD_DW2(p) ((p)->src_lo)
165#define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
166
167/* Word 3 */
168#define CCP5_CMD_DW3(p) ((p)->dw3)
169#define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
170#define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
171#define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
172#define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
173
174/* Words 4/5 */
175#define CCP5_CMD_DW4(p) ((p)->dw4)
176#define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
177#define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
178#define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
179#define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
180#define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
181#define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
182#define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
183
184/* Word 6/7 */
185#define CCP5_CMD_DW6(p) ((p)->key_lo)
186#define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
187#define CCP5_CMD_DW7(p) ((p)->dw7)
188#define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
189#define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
190
191static inline u32 low_address(unsigned long addr)
192{
193 return (u64)addr & 0x0ffffffff;
194}
195
196static inline u32 high_address(unsigned long addr)
197{
198 return ((u64)addr >> 32) & 0x00000ffff;
199}
200
201static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
202{
203 unsigned int head_idx, n;
204 u32 head_lo, queue_start;
205
206 queue_start = low_address(cmd_q->qdma_tail);
207 head_lo = ioread32(cmd_q->reg_head_lo);
208 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
209
210 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
211
212 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
213}
214
215static int ccp5_do_cmd(struct ccp5_desc *desc,
216 struct ccp_cmd_queue *cmd_q)
217{
218 u32 *mP;
219 __le32 *dP;
220 u32 tail;
221 int i;
222 int ret = 0;
223
224 if (CCP5_CMD_SOC(desc)) {
225 CCP5_CMD_IOC(desc) = 1;
226 CCP5_CMD_SOC(desc) = 0;
227 }
228 mutex_lock(&cmd_q->q_mutex);
229
230 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
231 dP = (__le32 *) desc;
232 for (i = 0; i < 8; i++)
233 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
234
235 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
236
237 /* The data used by this command must be flushed to memory */
238 wmb();
239
240 /* Write the new tail address back to the queue register */
241 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
242 iowrite32(tail, cmd_q->reg_tail_lo);
243
244 /* Turn the queue back on using our cached control register */
245 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
246 mutex_unlock(&cmd_q->q_mutex);
247
248 if (CCP5_CMD_IOC(desc)) {
249 /* Wait for the job to complete */
250 ret = wait_event_interruptible(cmd_q->int_queue,
251 cmd_q->int_rcvd);
252 if (ret || cmd_q->cmd_error) {
253 if (cmd_q->cmd_error)
254 ccp_log_error(cmd_q->ccp,
255 cmd_q->cmd_error);
256 /* A version 5 device doesn't use Job IDs... */
257 if (!ret)
258 ret = -EIO;
259 }
260 cmd_q->int_rcvd = 0;
261 }
262
263 return 0;
264}
265
266static int ccp5_perform_aes(struct ccp_op *op)
267{
268 struct ccp5_desc desc;
269 union ccp_function function;
270 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
271
272 /* Zero out all the fields of the command desc */
273 memset(&desc, 0, Q_DESC_SIZE);
274
275 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
276
277 CCP5_CMD_SOC(&desc) = op->soc;
278 CCP5_CMD_IOC(&desc) = 1;
279 CCP5_CMD_INIT(&desc) = op->init;
280 CCP5_CMD_EOM(&desc) = op->eom;
281 CCP5_CMD_PROT(&desc) = 0;
282
283 function.raw = 0;
284 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
285 CCP_AES_MODE(&function) = op->u.aes.mode;
286 CCP_AES_TYPE(&function) = op->u.aes.type;
287 if (op->u.aes.mode == CCP_AES_MODE_CFB)
288 CCP_AES_SIZE(&function) = 0x7f;
289
290 CCP5_CMD_FUNCTION(&desc) = function.raw;
291
292 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
293
294 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
295 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
296 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
297
298 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
299 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
300 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
301
302 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
303 CCP5_CMD_KEY_HI(&desc) = 0;
304 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
305 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
306
307 return ccp5_do_cmd(&desc, op->cmd_q);
308}
309
310static int ccp5_perform_xts_aes(struct ccp_op *op)
311{
312 struct ccp5_desc desc;
313 union ccp_function function;
314 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
315
316 /* Zero out all the fields of the command desc */
317 memset(&desc, 0, Q_DESC_SIZE);
318
319 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
320
321 CCP5_CMD_SOC(&desc) = op->soc;
322 CCP5_CMD_IOC(&desc) = 1;
323 CCP5_CMD_INIT(&desc) = op->init;
324 CCP5_CMD_EOM(&desc) = op->eom;
325 CCP5_CMD_PROT(&desc) = 0;
326
327 function.raw = 0;
328 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
329 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
330 CCP5_CMD_FUNCTION(&desc) = function.raw;
331
332 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
333
334 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
335 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
336 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
337
338 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
339 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
340 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
341
342 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
343 CCP5_CMD_KEY_HI(&desc) = 0;
344 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
345 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
346
347 return ccp5_do_cmd(&desc, op->cmd_q);
348}
349
350static int ccp5_perform_sha(struct ccp_op *op)
351{
352 struct ccp5_desc desc;
353 union ccp_function function;
354
355 /* Zero out all the fields of the command desc */
356 memset(&desc, 0, Q_DESC_SIZE);
357
358 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
359
360 CCP5_CMD_SOC(&desc) = op->soc;
361 CCP5_CMD_IOC(&desc) = 1;
362 CCP5_CMD_INIT(&desc) = 1;
363 CCP5_CMD_EOM(&desc) = op->eom;
364 CCP5_CMD_PROT(&desc) = 0;
365
366 function.raw = 0;
367 CCP_SHA_TYPE(&function) = op->u.sha.type;
368 CCP5_CMD_FUNCTION(&desc) = function.raw;
369
370 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
371
372 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
373 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
374 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
375
376 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
377
378 if (op->eom) {
379 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
380 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
381 } else {
382 CCP5_CMD_SHA_LO(&desc) = 0;
383 CCP5_CMD_SHA_HI(&desc) = 0;
384 }
385
386 return ccp5_do_cmd(&desc, op->cmd_q);
387}
388
389static int ccp5_perform_rsa(struct ccp_op *op)
390{
391 struct ccp5_desc desc;
392 union ccp_function function;
393
394 /* Zero out all the fields of the command desc */
395 memset(&desc, 0, Q_DESC_SIZE);
396
397 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
398
399 CCP5_CMD_SOC(&desc) = op->soc;
400 CCP5_CMD_IOC(&desc) = 1;
401 CCP5_CMD_INIT(&desc) = 0;
402 CCP5_CMD_EOM(&desc) = 1;
403 CCP5_CMD_PROT(&desc) = 0;
404
405 function.raw = 0;
406 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3;
407 CCP5_CMD_FUNCTION(&desc) = function.raw;
408
409 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
410
411 /* Source is from external memory */
412 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
413 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
414 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
415
416 /* Destination is in external memory */
417 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
418 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
419 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
420
421 /* Exponent is in LSB memory */
422 CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE;
423 CCP5_CMD_KEY_HI(&desc) = 0;
424 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
425
426 return ccp5_do_cmd(&desc, op->cmd_q);
427}
428
429static int ccp5_perform_passthru(struct ccp_op *op)
430{
431 struct ccp5_desc desc;
432 union ccp_function function;
433 struct ccp_dma_info *saddr = &op->src.u.dma;
434 struct ccp_dma_info *daddr = &op->dst.u.dma;
435
436 memset(&desc, 0, Q_DESC_SIZE);
437
438 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
439
440 CCP5_CMD_SOC(&desc) = 0;
441 CCP5_CMD_IOC(&desc) = 1;
442 CCP5_CMD_INIT(&desc) = 0;
443 CCP5_CMD_EOM(&desc) = op->eom;
444 CCP5_CMD_PROT(&desc) = 0;
445
446 function.raw = 0;
447 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
448 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
449 CCP5_CMD_FUNCTION(&desc) = function.raw;
450
451 /* Length of source data is always 256 bytes */
452 if (op->src.type == CCP_MEMTYPE_SYSTEM)
453 CCP5_CMD_LEN(&desc) = saddr->length;
454 else
455 CCP5_CMD_LEN(&desc) = daddr->length;
456
457 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
458 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
459 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
460 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
461
462 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
463 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
464 } else {
465 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
466
467 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
468 CCP5_CMD_SRC_HI(&desc) = 0;
469 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
470 }
471
472 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
473 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
474 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
475 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
476 } else {
477 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
478
479 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
480 CCP5_CMD_DST_HI(&desc) = 0;
481 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
482 }
483
484 return ccp5_do_cmd(&desc, op->cmd_q);
485}
486
487static int ccp5_perform_ecc(struct ccp_op *op)
488{
489 struct ccp5_desc desc;
490 union ccp_function function;
491
492 /* Zero out all the fields of the command desc */
493 memset(&desc, 0, Q_DESC_SIZE);
494
495 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
496
497 CCP5_CMD_SOC(&desc) = 0;
498 CCP5_CMD_IOC(&desc) = 1;
499 CCP5_CMD_INIT(&desc) = 0;
500 CCP5_CMD_EOM(&desc) = 1;
501 CCP5_CMD_PROT(&desc) = 0;
502
503 function.raw = 0;
504 function.ecc.mode = op->u.ecc.function;
505 CCP5_CMD_FUNCTION(&desc) = function.raw;
506
507 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
508
509 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
510 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
511 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
512
513 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
514 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
515 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
516
517 return ccp5_do_cmd(&desc, op->cmd_q);
518}
519
520static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
521{
522 int q_mask = 1 << cmd_q->id;
523 int queues = 0;
524 int j;
525
526 /* Build a bit mask to know which LSBs this queue has access to.
527 * Don't bother with segment 0 as it has special privileges.
528 */
529 for (j = 1; j < MAX_LSB_CNT; j++) {
530 if (status & q_mask)
531 bitmap_set(cmd_q->lsbmask, j, 1);
532 status >>= LSB_REGION_WIDTH;
533 }
534 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
535 dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
536 cmd_q->id, queues);
537
538 return queues ? 0 : -EINVAL;
539}
540
541
542static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
543 int lsb_cnt, int n_lsbs,
544 unsigned long *lsb_pub)
545{
546 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
547 int bitno;
548 int qlsb_wgt;
549 int i;
550
551 /* For each queue:
552 * If the count of potential LSBs available to a queue matches the
553 * ordinal given to us in lsb_cnt:
554 * Copy the mask of possible LSBs for this queue into "qlsb";
555 * For each bit in qlsb, see if the corresponding bit in the
556 * aggregation mask is set; if so, we have a match.
557 * If we have a match, clear the bit in the aggregation to
558 * mark it as no longer available.
559 * If there is no match, clear the bit in qlsb and keep looking.
560 */
561 for (i = 0; i < ccp->cmd_q_count; i++) {
562 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
563
564 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
565
566 if (qlsb_wgt == lsb_cnt) {
567 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
568
569 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
570 while (bitno < MAX_LSB_CNT) {
571 if (test_bit(bitno, lsb_pub)) {
572 /* We found an available LSB
573 * that this queue can access
574 */
575 cmd_q->lsb = bitno;
576 bitmap_clear(lsb_pub, bitno, 1);
577 dev_info(ccp->dev,
578 "Queue %d gets LSB %d\n",
579 i, bitno);
580 break;
581 }
582 bitmap_clear(qlsb, bitno, 1);
583 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
584 }
585 if (bitno >= MAX_LSB_CNT)
586 return -EINVAL;
587 n_lsbs--;
588 }
589 }
590 return n_lsbs;
591}
592
593/* For each queue, from the most- to least-constrained:
594 * find an LSB that can be assigned to the queue. If there are N queues that
595 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
596 * dedicated LSB. Remaining LSB regions become a shared resource.
597 * If we have fewer LSBs than queues, all LSB regions become shared resources.
598 */
599static int ccp_assign_lsbs(struct ccp_device *ccp)
600{
601 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
602 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
603 int n_lsbs = 0;
604 int bitno;
605 int i, lsb_cnt;
606 int rc = 0;
607
608 bitmap_zero(lsb_pub, MAX_LSB_CNT);
609
610 /* Create an aggregate bitmap to get a total count of available LSBs */
611 for (i = 0; i < ccp->cmd_q_count; i++)
612 bitmap_or(lsb_pub,
613 lsb_pub, ccp->cmd_q[i].lsbmask,
614 MAX_LSB_CNT);
615
616 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
617
618 if (n_lsbs >= ccp->cmd_q_count) {
619 /* We have enough LSBS to give every queue a private LSB.
620 * Brute force search to start with the queues that are more
621 * constrained in LSB choice. When an LSB is privately
622 * assigned, it is removed from the public mask.
623 * This is an ugly N squared algorithm with some optimization.
624 */
625 for (lsb_cnt = 1;
626 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
627 lsb_cnt++) {
628 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
629 lsb_pub);
630 if (rc < 0)
631 return -EINVAL;
632 n_lsbs = rc;
633 }
634 }
635
636 rc = 0;
637 /* What's left of the LSBs, according to the public mask, now become
638 * shared. Any zero bits in the lsb_pub mask represent an LSB region
639 * that can't be used as a shared resource, so mark the LSB slots for
640 * them as "in use".
641 */
642 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
643
644 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
645 while (bitno < MAX_LSB_CNT) {
646 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
647 bitmap_set(qlsb, bitno, 1);
648 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
649 }
650
651 return rc;
652}
653
654static int ccp5_init(struct ccp_device *ccp)
655{
656 struct device *dev = ccp->dev;
657 struct ccp_cmd_queue *cmd_q;
658 struct dma_pool *dma_pool;
659 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
660 unsigned int qmr, qim, i;
661 u64 status;
662 u32 status_lo, status_hi;
663 int ret;
664
665 /* Find available queues */
666 qim = 0;
667 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
668 for (i = 0; i < MAX_HW_QUEUES; i++) {
669
670 if (!(qmr & (1 << i)))
671 continue;
672
673 /* Allocate a dma pool for this queue */
674 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
675 ccp->name, i);
676 dma_pool = dma_pool_create(dma_pool_name, dev,
677 CCP_DMAPOOL_MAX_SIZE,
678 CCP_DMAPOOL_ALIGN, 0);
679 if (!dma_pool) {
680 dev_err(dev, "unable to allocate dma pool\n");
681 ret = -ENOMEM;
682 }
683
684 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
685 ccp->cmd_q_count++;
686
687 cmd_q->ccp = ccp;
688 cmd_q->id = i;
689 cmd_q->dma_pool = dma_pool;
690 mutex_init(&cmd_q->q_mutex);
691
692 /* Page alignment satisfies our needs for N <= 128 */
693 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
694 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
695 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
696 &cmd_q->qbase_dma,
697 GFP_KERNEL);
698 if (!cmd_q->qbase) {
699 dev_err(dev, "unable to allocate command queue\n");
700 ret = -ENOMEM;
701 goto e_pool;
702 }
703
704 cmd_q->qidx = 0;
705 /* Preset some register values and masks that are queue
706 * number dependent
707 */
708 cmd_q->reg_control = ccp->io_regs +
709 CMD5_Q_STATUS_INCR * (i + 1);
710 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
711 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
712 cmd_q->reg_int_enable = cmd_q->reg_control +
713 CMD5_Q_INT_ENABLE_BASE;
714 cmd_q->reg_interrupt_status = cmd_q->reg_control +
715 CMD5_Q_INTERRUPT_STATUS_BASE;
716 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
717 cmd_q->reg_int_status = cmd_q->reg_control +
718 CMD5_Q_INT_STATUS_BASE;
719 cmd_q->reg_dma_status = cmd_q->reg_control +
720 CMD5_Q_DMA_STATUS_BASE;
721 cmd_q->reg_dma_read_status = cmd_q->reg_control +
722 CMD5_Q_DMA_READ_STATUS_BASE;
723 cmd_q->reg_dma_write_status = cmd_q->reg_control +
724 CMD5_Q_DMA_WRITE_STATUS_BASE;
725
726 init_waitqueue_head(&cmd_q->int_queue);
727
728 dev_dbg(dev, "queue #%u available\n", i);
729 }
730 if (ccp->cmd_q_count == 0) {
731 dev_notice(dev, "no command queues available\n");
732 ret = -EIO;
733 goto e_pool;
734 }
735 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
736
737 /* Turn off the queues and disable interrupts until ready */
738 for (i = 0; i < ccp->cmd_q_count; i++) {
739 cmd_q = &ccp->cmd_q[i];
740
741 cmd_q->qcontrol = 0; /* Start with nothing */
742 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
743
744 /* Disable the interrupts */
745 iowrite32(0x00, cmd_q->reg_int_enable);
746 ioread32(cmd_q->reg_int_status);
747 ioread32(cmd_q->reg_status);
748
749 /* Clear the interrupts */
750 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
751 }
752
753 dev_dbg(dev, "Requesting an IRQ...\n");
754 /* Request an irq */
755 ret = ccp->get_irq(ccp);
756 if (ret) {
757 dev_err(dev, "unable to allocate an IRQ\n");
758 goto e_pool;
759 }
760
761 dev_dbg(dev, "Loading LSB map...\n");
762 /* Copy the private LSB mask to the public registers */
763 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
764 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
765 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
766 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
767 status = ((u64)status_hi<<30) | (u64)status_lo;
768
769 dev_dbg(dev, "Configuring virtual queues...\n");
770 /* Configure size of each virtual queue accessible to host */
771 for (i = 0; i < ccp->cmd_q_count; i++) {
772 u32 dma_addr_lo;
773 u32 dma_addr_hi;
774
775 cmd_q = &ccp->cmd_q[i];
776
777 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
778 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
779
780 cmd_q->qdma_tail = cmd_q->qbase_dma;
781 dma_addr_lo = low_address(cmd_q->qdma_tail);
782 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
783 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
784
785 dma_addr_hi = high_address(cmd_q->qdma_tail);
786 cmd_q->qcontrol |= (dma_addr_hi << 16);
787 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
788
789 /* Find the LSB regions accessible to the queue */
790 ccp_find_lsb_regions(cmd_q, status);
791 cmd_q->lsb = -1; /* Unassigned value */
792 }
793
794 dev_dbg(dev, "Assigning LSBs...\n");
795 ret = ccp_assign_lsbs(ccp);
796 if (ret) {
797 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
798 goto e_irq;
799 }
800
801 /* Optimization: pre-allocate LSB slots for each queue */
802 for (i = 0; i < ccp->cmd_q_count; i++) {
803 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
804 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
805 }
806
807 dev_dbg(dev, "Starting threads...\n");
808 /* Create a kthread for each queue */
809 for (i = 0; i < ccp->cmd_q_count; i++) {
810 struct task_struct *kthread;
811
812 cmd_q = &ccp->cmd_q[i];
813
814 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
815 "%s-q%u", ccp->name, cmd_q->id);
816 if (IS_ERR(kthread)) {
817 dev_err(dev, "error creating queue thread (%ld)\n",
818 PTR_ERR(kthread));
819 ret = PTR_ERR(kthread);
820 goto e_kthread;
821 }
822
823 cmd_q->kthread = kthread;
824 wake_up_process(kthread);
825 }
826
827 dev_dbg(dev, "Enabling interrupts...\n");
828 /* Enable interrupts */
829 for (i = 0; i < ccp->cmd_q_count; i++) {
830 cmd_q = &ccp->cmd_q[i];
831 iowrite32(ALL_INTERRUPTS, cmd_q->reg_int_enable);
832 }
833
834 dev_dbg(dev, "Registering device...\n");
835 /* Put this on the unit list to make it available */
836 ccp_add_device(ccp);
837
838 ret = ccp_register_rng(ccp);
839 if (ret)
840 goto e_kthread;
841
842 /* Register the DMA engine support */
843 ret = ccp_dmaengine_register(ccp);
844 if (ret)
845 goto e_hwrng;
846
847 return 0;
848
849e_hwrng:
850 ccp_unregister_rng(ccp);
851
852e_kthread:
853 for (i = 0; i < ccp->cmd_q_count; i++)
854 if (ccp->cmd_q[i].kthread)
855 kthread_stop(ccp->cmd_q[i].kthread);
856
857e_irq:
858 ccp->free_irq(ccp);
859
860e_pool:
861 for (i = 0; i < ccp->cmd_q_count; i++)
862 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
863
864 return ret;
865}
866
867static void ccp5_destroy(struct ccp_device *ccp)
868{
869 struct device *dev = ccp->dev;
870 struct ccp_cmd_queue *cmd_q;
871 struct ccp_cmd *cmd;
872 unsigned int i;
873
874 /* Unregister the DMA engine */
875 ccp_dmaengine_unregister(ccp);
876
877 /* Unregister the RNG */
878 ccp_unregister_rng(ccp);
879
880 /* Remove this device from the list of available units first */
881 ccp_del_device(ccp);
882
883 /* Disable and clear interrupts */
884 for (i = 0; i < ccp->cmd_q_count; i++) {
885 cmd_q = &ccp->cmd_q[i];
886
887 /* Turn off the run bit */
888 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
889
890 /* Disable the interrupts */
891 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
892
893 /* Clear the interrupt status */
894 iowrite32(0x00, cmd_q->reg_int_enable);
895 ioread32(cmd_q->reg_int_status);
896 ioread32(cmd_q->reg_status);
897 }
898
899 /* Stop the queue kthreads */
900 for (i = 0; i < ccp->cmd_q_count; i++)
901 if (ccp->cmd_q[i].kthread)
902 kthread_stop(ccp->cmd_q[i].kthread);
903
904 ccp->free_irq(ccp);
905
906 for (i = 0; i < ccp->cmd_q_count; i++) {
907 cmd_q = &ccp->cmd_q[i];
908 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
909 cmd_q->qbase_dma);
910 }
911
912 /* Flush the cmd and backlog queue */
913 while (!list_empty(&ccp->cmd)) {
914 /* Invoke the callback directly with an error code */
915 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
916 list_del(&cmd->entry);
917 cmd->callback(cmd->data, -ENODEV);
918 }
919 while (!list_empty(&ccp->backlog)) {
920 /* Invoke the callback directly with an error code */
921 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
922 list_del(&cmd->entry);
923 cmd->callback(cmd->data, -ENODEV);
924 }
925}
926
927static irqreturn_t ccp5_irq_handler(int irq, void *data)
928{
929 struct device *dev = data;
930 struct ccp_device *ccp = dev_get_drvdata(dev);
931 u32 status;
932 unsigned int i;
933
934 for (i = 0; i < ccp->cmd_q_count; i++) {
935 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
936
937 status = ioread32(cmd_q->reg_interrupt_status);
938
939 if (status) {
940 cmd_q->int_status = status;
941 cmd_q->q_status = ioread32(cmd_q->reg_status);
942 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
943
944 /* On error, only save the first error value */
945 if ((status & INT_ERROR) && !cmd_q->cmd_error)
946 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
947
948 cmd_q->int_rcvd = 1;
949
950 /* Acknowledge the interrupt and wake the kthread */
951 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
952 wake_up_interruptible(&cmd_q->int_queue);
953 }
954 }
955
956 return IRQ_HANDLED;
957}
958
959static void ccp5_config(struct ccp_device *ccp)
960{
961 /* Public side */
962 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
963}
964
965static void ccp5other_config(struct ccp_device *ccp)
966{
967 int i;
968 u32 rnd;
969
970 /* We own all of the queues on the NTB CCP */
971
972 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
973 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
974 for (i = 0; i < 12; i++) {
975 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
976 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
977 }
978
979 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
980 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
981 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
982
983 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
984 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
985
986 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
987
988 ccp5_config(ccp);
989}
990
991/* Version 5 adds some function, but is essentially the same as v5 */
992static const struct ccp_actions ccp5_actions = {
993 .aes = ccp5_perform_aes,
994 .xts_aes = ccp5_perform_xts_aes,
995 .sha = ccp5_perform_sha,
996 .rsa = ccp5_perform_rsa,
997 .passthru = ccp5_perform_passthru,
998 .ecc = ccp5_perform_ecc,
999 .sballoc = ccp_lsb_alloc,
1000 .sbfree = ccp_lsb_free,
1001 .init = ccp5_init,
1002 .destroy = ccp5_destroy,
1003 .get_free_slots = ccp5_get_free_slots,
1004 .irqhandler = ccp5_irq_handler,
1005};
1006
1007const struct ccp_vdata ccpv5a = {
1008 .version = CCP_VERSION(5, 0),
1009 .setup = ccp5_config,
1010 .perform = &ccp5_actions,
1011 .bar = 2,
1012 .offset = 0x0,
1013};
1014
1015const struct ccp_vdata ccpv5b = {
1016 .version = CCP_VERSION(5, 0),
1017 .dma_chan_attr = DMA_PRIVATE,
1018 .setup = ccp5other_config,
1019 .perform = &ccp5_actions,
1020 .bar = 2,
1021 .offset = 0x0,
1022};
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AMD Cryptographic Coprocessor (CCP) driver
4 *
5 * Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Gary R Hook <gary.hook@amd.com>
8 */
9
10#include <linux/kernel.h>
11#include <linux/kthread.h>
12#include <linux/dma-mapping.h>
13#include <linux/interrupt.h>
14#include <linux/compiler.h>
15#include <linux/ccp.h>
16
17#include "ccp-dev.h"
18
19/* Allocate the requested number of contiguous LSB slots
20 * from the LSB bitmap. Look in the private range for this
21 * queue first; failing that, check the public area.
22 * If no space is available, wait around.
23 * Return: first slot number
24 */
25static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
26{
27 struct ccp_device *ccp;
28 int start;
29
30 /* First look at the map for the queue */
31 if (cmd_q->lsb >= 0) {
32 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
33 LSB_SIZE,
34 0, count, 0);
35 if (start < LSB_SIZE) {
36 bitmap_set(cmd_q->lsbmap, start, count);
37 return start + cmd_q->lsb * LSB_SIZE;
38 }
39 }
40
41 /* No joy; try to get an entry from the shared blocks */
42 ccp = cmd_q->ccp;
43 for (;;) {
44 mutex_lock(&ccp->sb_mutex);
45
46 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
47 MAX_LSB_CNT * LSB_SIZE,
48 0,
49 count, 0);
50 if (start <= MAX_LSB_CNT * LSB_SIZE) {
51 bitmap_set(ccp->lsbmap, start, count);
52
53 mutex_unlock(&ccp->sb_mutex);
54 return start;
55 }
56
57 ccp->sb_avail = 0;
58
59 mutex_unlock(&ccp->sb_mutex);
60
61 /* Wait for KSB entries to become available */
62 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
63 return 0;
64 }
65}
66
67/* Free a number of LSB slots from the bitmap, starting at
68 * the indicated starting slot number.
69 */
70static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
71 unsigned int count)
72{
73 if (!start)
74 return;
75
76 if (cmd_q->lsb == start) {
77 /* An entry from the private LSB */
78 bitmap_clear(cmd_q->lsbmap, start, count);
79 } else {
80 /* From the shared LSBs */
81 struct ccp_device *ccp = cmd_q->ccp;
82
83 mutex_lock(&ccp->sb_mutex);
84 bitmap_clear(ccp->lsbmap, start, count);
85 ccp->sb_avail = 1;
86 mutex_unlock(&ccp->sb_mutex);
87 wake_up_interruptible_all(&ccp->sb_queue);
88 }
89}
90
91/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
92union ccp_function {
93 struct {
94 u16 size:7;
95 u16 encrypt:1;
96 u16 mode:5;
97 u16 type:2;
98 } aes;
99 struct {
100 u16 size:7;
101 u16 encrypt:1;
102 u16 rsvd:5;
103 u16 type:2;
104 } aes_xts;
105 struct {
106 u16 size:7;
107 u16 encrypt:1;
108 u16 mode:5;
109 u16 type:2;
110 } des3;
111 struct {
112 u16 rsvd1:10;
113 u16 type:4;
114 u16 rsvd2:1;
115 } sha;
116 struct {
117 u16 mode:3;
118 u16 size:12;
119 } rsa;
120 struct {
121 u16 byteswap:2;
122 u16 bitwise:3;
123 u16 reflect:2;
124 u16 rsvd:8;
125 } pt;
126 struct {
127 u16 rsvd:13;
128 } zlib;
129 struct {
130 u16 size:10;
131 u16 type:2;
132 u16 mode:3;
133 } ecc;
134 u16 raw;
135};
136
137#define CCP_AES_SIZE(p) ((p)->aes.size)
138#define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
139#define CCP_AES_MODE(p) ((p)->aes.mode)
140#define CCP_AES_TYPE(p) ((p)->aes.type)
141#define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
142#define CCP_XTS_TYPE(p) ((p)->aes_xts.type)
143#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
144#define CCP_DES3_SIZE(p) ((p)->des3.size)
145#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt)
146#define CCP_DES3_MODE(p) ((p)->des3.mode)
147#define CCP_DES3_TYPE(p) ((p)->des3.type)
148#define CCP_SHA_TYPE(p) ((p)->sha.type)
149#define CCP_RSA_SIZE(p) ((p)->rsa.size)
150#define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
151#define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
152#define CCP_ECC_MODE(p) ((p)->ecc.mode)
153#define CCP_ECC_AFFINE(p) ((p)->ecc.one)
154
155/* Word 0 */
156#define CCP5_CMD_DW0(p) ((p)->dw0)
157#define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
158#define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
159#define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
160#define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
161#define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
162#define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
163#define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
164
165/* Word 1 */
166#define CCP5_CMD_DW1(p) ((p)->length)
167#define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
168
169/* Word 2 */
170#define CCP5_CMD_DW2(p) ((p)->src_lo)
171#define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
172
173/* Word 3 */
174#define CCP5_CMD_DW3(p) ((p)->dw3)
175#define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
176#define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
177#define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
178#define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
179
180/* Words 4/5 */
181#define CCP5_CMD_DW4(p) ((p)->dw4)
182#define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
183#define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
184#define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
185#define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
186#define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
187#define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
188#define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
189
190/* Word 6/7 */
191#define CCP5_CMD_DW6(p) ((p)->key_lo)
192#define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
193#define CCP5_CMD_DW7(p) ((p)->dw7)
194#define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
195#define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
196
197static inline u32 low_address(unsigned long addr)
198{
199 return (u64)addr & 0x0ffffffff;
200}
201
202static inline u32 high_address(unsigned long addr)
203{
204 return ((u64)addr >> 32) & 0x00000ffff;
205}
206
207static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
208{
209 unsigned int head_idx, n;
210 u32 head_lo, queue_start;
211
212 queue_start = low_address(cmd_q->qdma_tail);
213 head_lo = ioread32(cmd_q->reg_head_lo);
214 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
215
216 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
217
218 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
219}
220
221static int ccp5_do_cmd(struct ccp5_desc *desc,
222 struct ccp_cmd_queue *cmd_q)
223{
224 __le32 *mP;
225 u32 *dP;
226 u32 tail;
227 int i;
228 int ret = 0;
229
230 cmd_q->total_ops++;
231
232 if (CCP5_CMD_SOC(desc)) {
233 CCP5_CMD_IOC(desc) = 1;
234 CCP5_CMD_SOC(desc) = 0;
235 }
236 mutex_lock(&cmd_q->q_mutex);
237
238 mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx];
239 dP = (u32 *)desc;
240 for (i = 0; i < 8; i++)
241 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
242
243 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
244
245 /* The data used by this command must be flushed to memory */
246 wmb();
247
248 /* Write the new tail address back to the queue register */
249 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
250 iowrite32(tail, cmd_q->reg_tail_lo);
251
252 /* Turn the queue back on using our cached control register */
253 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
254 mutex_unlock(&cmd_q->q_mutex);
255
256 if (CCP5_CMD_IOC(desc)) {
257 /* Wait for the job to complete */
258 ret = wait_event_interruptible(cmd_q->int_queue,
259 cmd_q->int_rcvd);
260 if (ret || cmd_q->cmd_error) {
261 /* Log the error and flush the queue by
262 * moving the head pointer
263 */
264 if (cmd_q->cmd_error)
265 ccp_log_error(cmd_q->ccp,
266 cmd_q->cmd_error);
267 iowrite32(tail, cmd_q->reg_head_lo);
268 if (!ret)
269 ret = -EIO;
270 }
271 cmd_q->int_rcvd = 0;
272 }
273
274 return ret;
275}
276
277static int ccp5_perform_aes(struct ccp_op *op)
278{
279 struct ccp5_desc desc;
280 union ccp_function function;
281 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
282
283 op->cmd_q->total_aes_ops++;
284
285 /* Zero out all the fields of the command desc */
286 memset(&desc, 0, Q_DESC_SIZE);
287
288 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
289
290 CCP5_CMD_SOC(&desc) = op->soc;
291 CCP5_CMD_IOC(&desc) = 1;
292 CCP5_CMD_INIT(&desc) = op->init;
293 CCP5_CMD_EOM(&desc) = op->eom;
294 CCP5_CMD_PROT(&desc) = 0;
295
296 function.raw = 0;
297 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
298 CCP_AES_MODE(&function) = op->u.aes.mode;
299 CCP_AES_TYPE(&function) = op->u.aes.type;
300 CCP_AES_SIZE(&function) = op->u.aes.size;
301
302 CCP5_CMD_FUNCTION(&desc) = function.raw;
303
304 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
305
306 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
307 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
308 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
309
310 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
311 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
312 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
313
314 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
315 CCP5_CMD_KEY_HI(&desc) = 0;
316 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
317 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
318
319 return ccp5_do_cmd(&desc, op->cmd_q);
320}
321
322static int ccp5_perform_xts_aes(struct ccp_op *op)
323{
324 struct ccp5_desc desc;
325 union ccp_function function;
326 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
327
328 op->cmd_q->total_xts_aes_ops++;
329
330 /* Zero out all the fields of the command desc */
331 memset(&desc, 0, Q_DESC_SIZE);
332
333 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
334
335 CCP5_CMD_SOC(&desc) = op->soc;
336 CCP5_CMD_IOC(&desc) = 1;
337 CCP5_CMD_INIT(&desc) = op->init;
338 CCP5_CMD_EOM(&desc) = op->eom;
339 CCP5_CMD_PROT(&desc) = 0;
340
341 function.raw = 0;
342 CCP_XTS_TYPE(&function) = op->u.xts.type;
343 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
344 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
345 CCP5_CMD_FUNCTION(&desc) = function.raw;
346
347 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
348
349 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
350 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
351 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
352
353 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
354 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
355 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
356
357 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
358 CCP5_CMD_KEY_HI(&desc) = 0;
359 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
360 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
361
362 return ccp5_do_cmd(&desc, op->cmd_q);
363}
364
365static int ccp5_perform_sha(struct ccp_op *op)
366{
367 struct ccp5_desc desc;
368 union ccp_function function;
369
370 op->cmd_q->total_sha_ops++;
371
372 /* Zero out all the fields of the command desc */
373 memset(&desc, 0, Q_DESC_SIZE);
374
375 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
376
377 CCP5_CMD_SOC(&desc) = op->soc;
378 CCP5_CMD_IOC(&desc) = 1;
379 CCP5_CMD_INIT(&desc) = 1;
380 CCP5_CMD_EOM(&desc) = op->eom;
381 CCP5_CMD_PROT(&desc) = 0;
382
383 function.raw = 0;
384 CCP_SHA_TYPE(&function) = op->u.sha.type;
385 CCP5_CMD_FUNCTION(&desc) = function.raw;
386
387 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
388
389 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
390 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
391 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
392
393 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
394
395 if (op->eom) {
396 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
397 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
398 } else {
399 CCP5_CMD_SHA_LO(&desc) = 0;
400 CCP5_CMD_SHA_HI(&desc) = 0;
401 }
402
403 return ccp5_do_cmd(&desc, op->cmd_q);
404}
405
406static int ccp5_perform_des3(struct ccp_op *op)
407{
408 struct ccp5_desc desc;
409 union ccp_function function;
410 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
411
412 op->cmd_q->total_3des_ops++;
413
414 /* Zero out all the fields of the command desc */
415 memset(&desc, 0, sizeof(struct ccp5_desc));
416
417 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3;
418
419 CCP5_CMD_SOC(&desc) = op->soc;
420 CCP5_CMD_IOC(&desc) = 1;
421 CCP5_CMD_INIT(&desc) = op->init;
422 CCP5_CMD_EOM(&desc) = op->eom;
423 CCP5_CMD_PROT(&desc) = 0;
424
425 function.raw = 0;
426 CCP_DES3_ENCRYPT(&function) = op->u.des3.action;
427 CCP_DES3_MODE(&function) = op->u.des3.mode;
428 CCP_DES3_TYPE(&function) = op->u.des3.type;
429 CCP5_CMD_FUNCTION(&desc) = function.raw;
430
431 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
432
433 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
434 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
435 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
436
437 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
438 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
439 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
440
441 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
442 CCP5_CMD_KEY_HI(&desc) = 0;
443 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
444 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
445
446 return ccp5_do_cmd(&desc, op->cmd_q);
447}
448
449static int ccp5_perform_rsa(struct ccp_op *op)
450{
451 struct ccp5_desc desc;
452 union ccp_function function;
453
454 op->cmd_q->total_rsa_ops++;
455
456 /* Zero out all the fields of the command desc */
457 memset(&desc, 0, Q_DESC_SIZE);
458
459 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
460
461 CCP5_CMD_SOC(&desc) = op->soc;
462 CCP5_CMD_IOC(&desc) = 1;
463 CCP5_CMD_INIT(&desc) = 0;
464 CCP5_CMD_EOM(&desc) = 1;
465 CCP5_CMD_PROT(&desc) = 0;
466
467 function.raw = 0;
468 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
469 CCP5_CMD_FUNCTION(&desc) = function.raw;
470
471 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
472
473 /* Source is from external memory */
474 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
475 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
476 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
477
478 /* Destination is in external memory */
479 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
480 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
481 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
482
483 /* Key (Exponent) is in external memory */
484 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
485 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
486 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
487
488 return ccp5_do_cmd(&desc, op->cmd_q);
489}
490
491static int ccp5_perform_passthru(struct ccp_op *op)
492{
493 struct ccp5_desc desc;
494 union ccp_function function;
495 struct ccp_dma_info *saddr = &op->src.u.dma;
496 struct ccp_dma_info *daddr = &op->dst.u.dma;
497
498
499 op->cmd_q->total_pt_ops++;
500
501 memset(&desc, 0, Q_DESC_SIZE);
502
503 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
504
505 CCP5_CMD_SOC(&desc) = 0;
506 CCP5_CMD_IOC(&desc) = 1;
507 CCP5_CMD_INIT(&desc) = 0;
508 CCP5_CMD_EOM(&desc) = op->eom;
509 CCP5_CMD_PROT(&desc) = 0;
510
511 function.raw = 0;
512 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
513 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
514 CCP5_CMD_FUNCTION(&desc) = function.raw;
515
516 /* Length of source data is always 256 bytes */
517 if (op->src.type == CCP_MEMTYPE_SYSTEM)
518 CCP5_CMD_LEN(&desc) = saddr->length;
519 else
520 CCP5_CMD_LEN(&desc) = daddr->length;
521
522 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
523 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
524 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
525 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
526
527 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
528 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
529 } else {
530 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
531
532 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
533 CCP5_CMD_SRC_HI(&desc) = 0;
534 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
535 }
536
537 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
538 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
539 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
540 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
541 } else {
542 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
543
544 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
545 CCP5_CMD_DST_HI(&desc) = 0;
546 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
547 }
548
549 return ccp5_do_cmd(&desc, op->cmd_q);
550}
551
552static int ccp5_perform_ecc(struct ccp_op *op)
553{
554 struct ccp5_desc desc;
555 union ccp_function function;
556
557 op->cmd_q->total_ecc_ops++;
558
559 /* Zero out all the fields of the command desc */
560 memset(&desc, 0, Q_DESC_SIZE);
561
562 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
563
564 CCP5_CMD_SOC(&desc) = 0;
565 CCP5_CMD_IOC(&desc) = 1;
566 CCP5_CMD_INIT(&desc) = 0;
567 CCP5_CMD_EOM(&desc) = 1;
568 CCP5_CMD_PROT(&desc) = 0;
569
570 function.raw = 0;
571 function.ecc.mode = op->u.ecc.function;
572 CCP5_CMD_FUNCTION(&desc) = function.raw;
573
574 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
575
576 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
577 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
578 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
579
580 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
581 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
582 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
583
584 return ccp5_do_cmd(&desc, op->cmd_q);
585}
586
587static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
588{
589 int q_mask = 1 << cmd_q->id;
590 int queues = 0;
591 int j;
592
593 /* Build a bit mask to know which LSBs this queue has access to.
594 * Don't bother with segment 0 as it has special privileges.
595 */
596 for (j = 1; j < MAX_LSB_CNT; j++) {
597 if (status & q_mask)
598 bitmap_set(cmd_q->lsbmask, j, 1);
599 status >>= LSB_REGION_WIDTH;
600 }
601 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
602 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
603 cmd_q->id, queues);
604
605 return queues ? 0 : -EINVAL;
606}
607
608static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
609 int lsb_cnt, int n_lsbs,
610 unsigned long *lsb_pub)
611{
612 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
613 int bitno;
614 int qlsb_wgt;
615 int i;
616
617 /* For each queue:
618 * If the count of potential LSBs available to a queue matches the
619 * ordinal given to us in lsb_cnt:
620 * Copy the mask of possible LSBs for this queue into "qlsb";
621 * For each bit in qlsb, see if the corresponding bit in the
622 * aggregation mask is set; if so, we have a match.
623 * If we have a match, clear the bit in the aggregation to
624 * mark it as no longer available.
625 * If there is no match, clear the bit in qlsb and keep looking.
626 */
627 for (i = 0; i < ccp->cmd_q_count; i++) {
628 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
629
630 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
631
632 if (qlsb_wgt == lsb_cnt) {
633 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
634
635 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
636 while (bitno < MAX_LSB_CNT) {
637 if (test_bit(bitno, lsb_pub)) {
638 /* We found an available LSB
639 * that this queue can access
640 */
641 cmd_q->lsb = bitno;
642 bitmap_clear(lsb_pub, bitno, 1);
643 dev_dbg(ccp->dev,
644 "Queue %d gets LSB %d\n",
645 i, bitno);
646 break;
647 }
648 bitmap_clear(qlsb, bitno, 1);
649 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
650 }
651 if (bitno >= MAX_LSB_CNT)
652 return -EINVAL;
653 n_lsbs--;
654 }
655 }
656 return n_lsbs;
657}
658
659/* For each queue, from the most- to least-constrained:
660 * find an LSB that can be assigned to the queue. If there are N queues that
661 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
662 * dedicated LSB. Remaining LSB regions become a shared resource.
663 * If we have fewer LSBs than queues, all LSB regions become shared resources.
664 */
665static int ccp_assign_lsbs(struct ccp_device *ccp)
666{
667 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
668 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
669 int n_lsbs = 0;
670 int bitno;
671 int i, lsb_cnt;
672 int rc = 0;
673
674 bitmap_zero(lsb_pub, MAX_LSB_CNT);
675
676 /* Create an aggregate bitmap to get a total count of available LSBs */
677 for (i = 0; i < ccp->cmd_q_count; i++)
678 bitmap_or(lsb_pub,
679 lsb_pub, ccp->cmd_q[i].lsbmask,
680 MAX_LSB_CNT);
681
682 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
683
684 if (n_lsbs >= ccp->cmd_q_count) {
685 /* We have enough LSBS to give every queue a private LSB.
686 * Brute force search to start with the queues that are more
687 * constrained in LSB choice. When an LSB is privately
688 * assigned, it is removed from the public mask.
689 * This is an ugly N squared algorithm with some optimization.
690 */
691 for (lsb_cnt = 1;
692 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
693 lsb_cnt++) {
694 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
695 lsb_pub);
696 if (rc < 0)
697 return -EINVAL;
698 n_lsbs = rc;
699 }
700 }
701
702 rc = 0;
703 /* What's left of the LSBs, according to the public mask, now become
704 * shared. Any zero bits in the lsb_pub mask represent an LSB region
705 * that can't be used as a shared resource, so mark the LSB slots for
706 * them as "in use".
707 */
708 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
709
710 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
711 while (bitno < MAX_LSB_CNT) {
712 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
713 bitmap_set(qlsb, bitno, 1);
714 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
715 }
716
717 return rc;
718}
719
720static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
721{
722 unsigned int i;
723
724 for (i = 0; i < ccp->cmd_q_count; i++)
725 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
726}
727
728static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
729{
730 unsigned int i;
731
732 for (i = 0; i < ccp->cmd_q_count; i++)
733 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
734}
735
736static void ccp5_irq_bh(unsigned long data)
737{
738 struct ccp_device *ccp = (struct ccp_device *)data;
739 u32 status;
740 unsigned int i;
741
742 for (i = 0; i < ccp->cmd_q_count; i++) {
743 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
744
745 status = ioread32(cmd_q->reg_interrupt_status);
746
747 if (status) {
748 cmd_q->int_status = status;
749 cmd_q->q_status = ioread32(cmd_q->reg_status);
750 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
751
752 /* On error, only save the first error value */
753 if ((status & INT_ERROR) && !cmd_q->cmd_error)
754 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
755
756 cmd_q->int_rcvd = 1;
757
758 /* Acknowledge the interrupt and wake the kthread */
759 iowrite32(status, cmd_q->reg_interrupt_status);
760 wake_up_interruptible(&cmd_q->int_queue);
761 }
762 }
763 ccp5_enable_queue_interrupts(ccp);
764}
765
766static irqreturn_t ccp5_irq_handler(int irq, void *data)
767{
768 struct ccp_device *ccp = (struct ccp_device *)data;
769
770 ccp5_disable_queue_interrupts(ccp);
771 ccp->total_interrupts++;
772 if (ccp->use_tasklet)
773 tasklet_schedule(&ccp->irq_tasklet);
774 else
775 ccp5_irq_bh((unsigned long)ccp);
776 return IRQ_HANDLED;
777}
778
779static int ccp5_init(struct ccp_device *ccp)
780{
781 struct device *dev = ccp->dev;
782 struct ccp_cmd_queue *cmd_q;
783 struct dma_pool *dma_pool;
784 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
785 unsigned int qmr, i;
786 u64 status;
787 u32 status_lo, status_hi;
788 int ret;
789
790 /* Find available queues */
791 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
792 /*
793 * Check for a access to the registers. If this read returns
794 * 0xffffffff, it's likely that the system is running a broken
795 * BIOS which disallows access to the device. Stop here and fail
796 * the initialization (but not the load, as the PSP could get
797 * properly initialized).
798 */
799 if (qmr == 0xffffffff) {
800 dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n");
801 return 1;
802 }
803
804 for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
805 if (!(qmr & (1 << i)))
806 continue;
807
808 /* Allocate a dma pool for this queue */
809 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
810 ccp->name, i);
811 dma_pool = dma_pool_create(dma_pool_name, dev,
812 CCP_DMAPOOL_MAX_SIZE,
813 CCP_DMAPOOL_ALIGN, 0);
814 if (!dma_pool) {
815 dev_err(dev, "unable to allocate dma pool\n");
816 ret = -ENOMEM;
817 goto e_pool;
818 }
819
820 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
821 ccp->cmd_q_count++;
822
823 cmd_q->ccp = ccp;
824 cmd_q->id = i;
825 cmd_q->dma_pool = dma_pool;
826 mutex_init(&cmd_q->q_mutex);
827
828 /* Page alignment satisfies our needs for N <= 128 */
829 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
830 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
831 cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,
832 &cmd_q->qbase_dma,
833 GFP_KERNEL);
834 if (!cmd_q->qbase) {
835 dev_err(dev, "unable to allocate command queue\n");
836 ret = -ENOMEM;
837 goto e_pool;
838 }
839
840 cmd_q->qidx = 0;
841 /* Preset some register values and masks that are queue
842 * number dependent
843 */
844 cmd_q->reg_control = ccp->io_regs +
845 CMD5_Q_STATUS_INCR * (i + 1);
846 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
847 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
848 cmd_q->reg_int_enable = cmd_q->reg_control +
849 CMD5_Q_INT_ENABLE_BASE;
850 cmd_q->reg_interrupt_status = cmd_q->reg_control +
851 CMD5_Q_INTERRUPT_STATUS_BASE;
852 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
853 cmd_q->reg_int_status = cmd_q->reg_control +
854 CMD5_Q_INT_STATUS_BASE;
855 cmd_q->reg_dma_status = cmd_q->reg_control +
856 CMD5_Q_DMA_STATUS_BASE;
857 cmd_q->reg_dma_read_status = cmd_q->reg_control +
858 CMD5_Q_DMA_READ_STATUS_BASE;
859 cmd_q->reg_dma_write_status = cmd_q->reg_control +
860 CMD5_Q_DMA_WRITE_STATUS_BASE;
861
862 init_waitqueue_head(&cmd_q->int_queue);
863
864 dev_dbg(dev, "queue #%u available\n", i);
865 }
866
867 if (ccp->cmd_q_count == 0) {
868 dev_notice(dev, "no command queues available\n");
869 ret = 1;
870 goto e_pool;
871 }
872
873 /* Turn off the queues and disable interrupts until ready */
874 ccp5_disable_queue_interrupts(ccp);
875 for (i = 0; i < ccp->cmd_q_count; i++) {
876 cmd_q = &ccp->cmd_q[i];
877
878 cmd_q->qcontrol = 0; /* Start with nothing */
879 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
880
881 ioread32(cmd_q->reg_int_status);
882 ioread32(cmd_q->reg_status);
883
884 /* Clear the interrupt status */
885 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
886 }
887
888 dev_dbg(dev, "Requesting an IRQ...\n");
889 /* Request an irq */
890 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
891 if (ret) {
892 dev_err(dev, "unable to allocate an IRQ\n");
893 goto e_pool;
894 }
895 /* Initialize the ISR tasklet */
896 if (ccp->use_tasklet)
897 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
898 (unsigned long)ccp);
899
900 dev_dbg(dev, "Loading LSB map...\n");
901 /* Copy the private LSB mask to the public registers */
902 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
903 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
904 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
905 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
906 status = ((u64)status_hi<<30) | (u64)status_lo;
907
908 dev_dbg(dev, "Configuring virtual queues...\n");
909 /* Configure size of each virtual queue accessible to host */
910 for (i = 0; i < ccp->cmd_q_count; i++) {
911 u32 dma_addr_lo;
912 u32 dma_addr_hi;
913
914 cmd_q = &ccp->cmd_q[i];
915
916 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
917 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
918
919 cmd_q->qdma_tail = cmd_q->qbase_dma;
920 dma_addr_lo = low_address(cmd_q->qdma_tail);
921 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
922 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
923
924 dma_addr_hi = high_address(cmd_q->qdma_tail);
925 cmd_q->qcontrol |= (dma_addr_hi << 16);
926 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
927
928 /* Find the LSB regions accessible to the queue */
929 ccp_find_lsb_regions(cmd_q, status);
930 cmd_q->lsb = -1; /* Unassigned value */
931 }
932
933 dev_dbg(dev, "Assigning LSBs...\n");
934 ret = ccp_assign_lsbs(ccp);
935 if (ret) {
936 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
937 goto e_irq;
938 }
939
940 /* Optimization: pre-allocate LSB slots for each queue */
941 for (i = 0; i < ccp->cmd_q_count; i++) {
942 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
943 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
944 }
945
946 dev_dbg(dev, "Starting threads...\n");
947 /* Create a kthread for each queue */
948 for (i = 0; i < ccp->cmd_q_count; i++) {
949 struct task_struct *kthread;
950
951 cmd_q = &ccp->cmd_q[i];
952
953 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
954 "%s-q%u", ccp->name, cmd_q->id);
955 if (IS_ERR(kthread)) {
956 dev_err(dev, "error creating queue thread (%ld)\n",
957 PTR_ERR(kthread));
958 ret = PTR_ERR(kthread);
959 goto e_kthread;
960 }
961
962 cmd_q->kthread = kthread;
963 wake_up_process(kthread);
964 }
965
966 dev_dbg(dev, "Enabling interrupts...\n");
967 ccp5_enable_queue_interrupts(ccp);
968
969 dev_dbg(dev, "Registering device...\n");
970 /* Put this on the unit list to make it available */
971 ccp_add_device(ccp);
972
973 ret = ccp_register_rng(ccp);
974 if (ret)
975 goto e_kthread;
976
977 /* Register the DMA engine support */
978 ret = ccp_dmaengine_register(ccp);
979 if (ret)
980 goto e_hwrng;
981
982#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
983 /* Set up debugfs entries */
984 ccp5_debugfs_setup(ccp);
985#endif
986
987 return 0;
988
989e_hwrng:
990 ccp_unregister_rng(ccp);
991
992e_kthread:
993 for (i = 0; i < ccp->cmd_q_count; i++)
994 if (ccp->cmd_q[i].kthread)
995 kthread_stop(ccp->cmd_q[i].kthread);
996
997e_irq:
998 sp_free_ccp_irq(ccp->sp, ccp);
999
1000e_pool:
1001 for (i = 0; i < ccp->cmd_q_count; i++)
1002 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
1003
1004 return ret;
1005}
1006
1007static void ccp5_destroy(struct ccp_device *ccp)
1008{
1009 struct ccp_cmd_queue *cmd_q;
1010 struct ccp_cmd *cmd;
1011 unsigned int i;
1012
1013 /* Unregister the DMA engine */
1014 ccp_dmaengine_unregister(ccp);
1015
1016 /* Unregister the RNG */
1017 ccp_unregister_rng(ccp);
1018
1019 /* Remove this device from the list of available units first */
1020 ccp_del_device(ccp);
1021
1022#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
1023 /* We're in the process of tearing down the entire driver;
1024 * when all the devices are gone clean up debugfs
1025 */
1026 if (ccp_present())
1027 ccp5_debugfs_destroy();
1028#endif
1029
1030 /* Disable and clear interrupts */
1031 ccp5_disable_queue_interrupts(ccp);
1032 for (i = 0; i < ccp->cmd_q_count; i++) {
1033 cmd_q = &ccp->cmd_q[i];
1034
1035 /* Turn off the run bit */
1036 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
1037
1038 /* Clear the interrupt status */
1039 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
1040 ioread32(cmd_q->reg_int_status);
1041 ioread32(cmd_q->reg_status);
1042 }
1043
1044 /* Stop the queue kthreads */
1045 for (i = 0; i < ccp->cmd_q_count; i++)
1046 if (ccp->cmd_q[i].kthread)
1047 kthread_stop(ccp->cmd_q[i].kthread);
1048
1049 sp_free_ccp_irq(ccp->sp, ccp);
1050
1051 /* Flush the cmd and backlog queue */
1052 while (!list_empty(&ccp->cmd)) {
1053 /* Invoke the callback directly with an error code */
1054 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
1055 list_del(&cmd->entry);
1056 cmd->callback(cmd->data, -ENODEV);
1057 }
1058 while (!list_empty(&ccp->backlog)) {
1059 /* Invoke the callback directly with an error code */
1060 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
1061 list_del(&cmd->entry);
1062 cmd->callback(cmd->data, -ENODEV);
1063 }
1064}
1065
1066static void ccp5_config(struct ccp_device *ccp)
1067{
1068 /* Public side */
1069 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
1070}
1071
1072static void ccp5other_config(struct ccp_device *ccp)
1073{
1074 int i;
1075 u32 rnd;
1076
1077 /* We own all of the queues on the NTB CCP */
1078
1079 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
1080 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
1081 for (i = 0; i < 12; i++) {
1082 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
1083 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
1084 }
1085
1086 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
1087 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
1088 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
1089
1090 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
1091 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
1092
1093 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
1094
1095 ccp5_config(ccp);
1096}
1097
1098/* Version 5 adds some function, but is essentially the same as v5 */
1099static const struct ccp_actions ccp5_actions = {
1100 .aes = ccp5_perform_aes,
1101 .xts_aes = ccp5_perform_xts_aes,
1102 .sha = ccp5_perform_sha,
1103 .des3 = ccp5_perform_des3,
1104 .rsa = ccp5_perform_rsa,
1105 .passthru = ccp5_perform_passthru,
1106 .ecc = ccp5_perform_ecc,
1107 .sballoc = ccp_lsb_alloc,
1108 .sbfree = ccp_lsb_free,
1109 .init = ccp5_init,
1110 .destroy = ccp5_destroy,
1111 .get_free_slots = ccp5_get_free_slots,
1112};
1113
1114const struct ccp_vdata ccpv5a = {
1115 .version = CCP_VERSION(5, 0),
1116 .setup = ccp5_config,
1117 .perform = &ccp5_actions,
1118 .offset = 0x0,
1119 .rsamax = CCP5_RSA_MAX_WIDTH,
1120};
1121
1122const struct ccp_vdata ccpv5b = {
1123 .version = CCP_VERSION(5, 0),
1124 .dma_chan_attr = DMA_PRIVATE,
1125 .setup = ccp5other_config,
1126 .perform = &ccp5_actions,
1127 .offset = 0x0,
1128 .rsamax = CCP5_RSA_MAX_WIDTH,
1129};