Loading...
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#ifndef __CCP_DEV_H__
14#define __CCP_DEV_H__
15
16#include <linux/device.h>
17#include <linux/pci.h>
18#include <linux/spinlock.h>
19#include <linux/mutex.h>
20#include <linux/list.h>
21#include <linux/wait.h>
22#include <linux/dmapool.h>
23#include <linux/hw_random.h>
24#include <linux/bitops.h>
25
26#define MAX_CCP_NAME_LEN 16
27#define MAX_DMAPOOL_NAME_LEN 32
28
29#define MAX_HW_QUEUES 5
30#define MAX_CMD_QLEN 100
31
32#define TRNG_RETRIES 10
33
34#define CACHE_NONE 0x00
35#define CACHE_WB_NO_ALLOC 0xb7
36
37/****** Register Mappings ******/
38#define Q_MASK_REG 0x000
39#define TRNG_OUT_REG 0x00c
40#define IRQ_MASK_REG 0x040
41#define IRQ_STATUS_REG 0x200
42
43#define DEL_CMD_Q_JOB 0x124
44#define DEL_Q_ACTIVE 0x00000200
45#define DEL_Q_ID_SHIFT 6
46
47#define CMD_REQ0 0x180
48#define CMD_REQ_INCR 0x04
49
50#define CMD_Q_STATUS_BASE 0x210
51#define CMD_Q_INT_STATUS_BASE 0x214
52#define CMD_Q_STATUS_INCR 0x20
53
54#define CMD_Q_CACHE_BASE 0x228
55#define CMD_Q_CACHE_INC 0x20
56
57#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)
58#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f)
59
60/****** REQ0 Related Values ******/
61#define REQ0_WAIT_FOR_WRITE 0x00000004
62#define REQ0_INT_ON_COMPLETE 0x00000002
63#define REQ0_STOP_ON_COMPLETE 0x00000001
64
65#define REQ0_CMD_Q_SHIFT 9
66#define REQ0_JOBID_SHIFT 3
67
68/****** REQ1 Related Values ******/
69#define REQ1_PROTECT_SHIFT 27
70#define REQ1_ENGINE_SHIFT 23
71#define REQ1_KEY_KSB_SHIFT 2
72
73#define REQ1_EOM 0x00000002
74#define REQ1_INIT 0x00000001
75
76/* AES Related Values */
77#define REQ1_AES_TYPE_SHIFT 21
78#define REQ1_AES_MODE_SHIFT 18
79#define REQ1_AES_ACTION_SHIFT 17
80#define REQ1_AES_CFB_SIZE_SHIFT 10
81
82/* XTS-AES Related Values */
83#define REQ1_XTS_AES_SIZE_SHIFT 10
84
85/* SHA Related Values */
86#define REQ1_SHA_TYPE_SHIFT 21
87
88/* RSA Related Values */
89#define REQ1_RSA_MOD_SIZE_SHIFT 10
90
91/* Pass-Through Related Values */
92#define REQ1_PT_BW_SHIFT 12
93#define REQ1_PT_BS_SHIFT 10
94
95/* ECC Related Values */
96#define REQ1_ECC_AFFINE_CONVERT 0x00200000
97#define REQ1_ECC_FUNCTION_SHIFT 18
98
99/****** REQ4 Related Values ******/
100#define REQ4_KSB_SHIFT 18
101#define REQ4_MEMTYPE_SHIFT 16
102
103/****** REQ6 Related Values ******/
104#define REQ6_MEMTYPE_SHIFT 16
105
106/****** Key Storage Block ******/
107#define KSB_START 77
108#define KSB_END 127
109#define KSB_COUNT (KSB_END - KSB_START + 1)
110#define CCP_KSB_BITS 256
111#define CCP_KSB_BYTES 32
112
113#define CCP_JOBID_MASK 0x0000003f
114
115#define CCP_DMAPOOL_MAX_SIZE 64
116#define CCP_DMAPOOL_ALIGN BIT(5)
117
118#define CCP_REVERSE_BUF_SIZE 64
119
120#define CCP_AES_KEY_KSB_COUNT 1
121#define CCP_AES_CTX_KSB_COUNT 1
122
123#define CCP_XTS_AES_KEY_KSB_COUNT 1
124#define CCP_XTS_AES_CTX_KSB_COUNT 1
125
126#define CCP_SHA_KSB_COUNT 1
127
128#define CCP_RSA_MAX_WIDTH 4096
129
130#define CCP_PASSTHRU_BLOCKSIZE 256
131#define CCP_PASSTHRU_MASKSIZE 32
132#define CCP_PASSTHRU_KSB_COUNT 1
133
134#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */
135#define CCP_ECC_MAX_OPERANDS 6
136#define CCP_ECC_MAX_OUTPUTS 3
137#define CCP_ECC_SRC_BUF_SIZE 448
138#define CCP_ECC_DST_BUF_SIZE 192
139#define CCP_ECC_OPERAND_SIZE 64
140#define CCP_ECC_OUTPUT_SIZE 64
141#define CCP_ECC_RESULT_OFFSET 60
142#define CCP_ECC_RESULT_SUCCESS 0x0001
143
144struct ccp_op;
145
146/* Structure for computation functions that are device-specific */
147struct ccp_actions {
148 int (*perform_aes)(struct ccp_op *);
149 int (*perform_xts_aes)(struct ccp_op *);
150 int (*perform_sha)(struct ccp_op *);
151 int (*perform_rsa)(struct ccp_op *);
152 int (*perform_passthru)(struct ccp_op *);
153 int (*perform_ecc)(struct ccp_op *);
154 int (*init)(struct ccp_device *);
155 void (*destroy)(struct ccp_device *);
156 irqreturn_t (*irqhandler)(int, void *);
157};
158
159/* Structure to hold CCP version-specific values */
160struct ccp_vdata {
161 unsigned int version;
162 struct ccp_actions *perform;
163};
164
165extern struct ccp_vdata ccpv3;
166
167struct ccp_device;
168struct ccp_cmd;
169
170struct ccp_cmd_queue {
171 struct ccp_device *ccp;
172
173 /* Queue identifier */
174 u32 id;
175
176 /* Queue dma pool */
177 struct dma_pool *dma_pool;
178
179 /* Queue reserved KSB regions */
180 u32 ksb_key;
181 u32 ksb_ctx;
182
183 /* Queue processing thread */
184 struct task_struct *kthread;
185 unsigned int active;
186 unsigned int suspended;
187
188 /* Number of free command slots available */
189 unsigned int free_slots;
190
191 /* Interrupt masks */
192 u32 int_ok;
193 u32 int_err;
194
195 /* Register addresses for queue */
196 void __iomem *reg_status;
197 void __iomem *reg_int_status;
198
199 /* Status values from job */
200 u32 int_status;
201 u32 q_status;
202 u32 q_int_status;
203 u32 cmd_error;
204
205 /* Interrupt wait queue */
206 wait_queue_head_t int_queue;
207 unsigned int int_rcvd;
208} ____cacheline_aligned;
209
210struct ccp_device {
211 struct list_head entry;
212
213 struct ccp_vdata *vdata;
214 unsigned int ord;
215 char name[MAX_CCP_NAME_LEN];
216 char rngname[MAX_CCP_NAME_LEN];
217
218 struct device *dev;
219
220 /*
221 * Bus specific device information
222 */
223 void *dev_specific;
224 int (*get_irq)(struct ccp_device *ccp);
225 void (*free_irq)(struct ccp_device *ccp);
226 unsigned int irq;
227
228 /*
229 * I/O area used for device communication. The register mapping
230 * starts at an offset into the mapped bar.
231 * The CMD_REQx registers and the Delete_Cmd_Queue_Job register
232 * need to be protected while a command queue thread is accessing
233 * them.
234 */
235 struct mutex req_mutex ____cacheline_aligned;
236 void __iomem *io_map;
237 void __iomem *io_regs;
238
239 /*
240 * Master lists that all cmds are queued on. Because there can be
241 * more than one CCP command queue that can process a cmd a separate
242 * backlog list is neeeded so that the backlog completion call
243 * completes before the cmd is available for execution.
244 */
245 spinlock_t cmd_lock ____cacheline_aligned;
246 unsigned int cmd_count;
247 struct list_head cmd;
248 struct list_head backlog;
249
250 /*
251 * The command queues. These represent the queues available on the
252 * CCP that are available for processing cmds
253 */
254 struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
255 unsigned int cmd_q_count;
256
257 /*
258 * Support for the CCP True RNG
259 */
260 struct hwrng hwrng;
261 unsigned int hwrng_retries;
262
263 /*
264 * A counter used to generate job-ids for cmds submitted to the CCP
265 */
266 atomic_t current_id ____cacheline_aligned;
267
268 /*
269 * The CCP uses key storage blocks (KSB) to maintain context for certain
270 * operations. To prevent multiple cmds from using the same KSB range
271 * a command queue reserves a KSB range for the duration of the cmd.
272 * Each queue, will however, reserve 2 KSB blocks for operations that
273 * only require single KSB entries (eg. AES context/iv and key) in order
274 * to avoid allocation contention. This will reserve at most 10 KSB
275 * entries, leaving 40 KSB entries available for dynamic allocation.
276 */
277 struct mutex ksb_mutex ____cacheline_aligned;
278 DECLARE_BITMAP(ksb, KSB_COUNT);
279 wait_queue_head_t ksb_queue;
280 unsigned int ksb_avail;
281 unsigned int ksb_count;
282 u32 ksb_start;
283
284 /* Suspend support */
285 unsigned int suspending;
286 wait_queue_head_t suspend_queue;
287
288 /* DMA caching attribute support */
289 unsigned int axcache;
290};
291
292enum ccp_memtype {
293 CCP_MEMTYPE_SYSTEM = 0,
294 CCP_MEMTYPE_KSB,
295 CCP_MEMTYPE_LOCAL,
296 CCP_MEMTYPE__LAST,
297};
298
299struct ccp_dma_info {
300 dma_addr_t address;
301 unsigned int offset;
302 unsigned int length;
303 enum dma_data_direction dir;
304};
305
306struct ccp_dm_workarea {
307 struct device *dev;
308 struct dma_pool *dma_pool;
309 unsigned int length;
310
311 u8 *address;
312 struct ccp_dma_info dma;
313};
314
315struct ccp_sg_workarea {
316 struct scatterlist *sg;
317 int nents;
318
319 struct scatterlist *dma_sg;
320 struct device *dma_dev;
321 unsigned int dma_count;
322 enum dma_data_direction dma_dir;
323
324 unsigned int sg_used;
325
326 u64 bytes_left;
327};
328
329struct ccp_data {
330 struct ccp_sg_workarea sg_wa;
331 struct ccp_dm_workarea dm_wa;
332};
333
334struct ccp_mem {
335 enum ccp_memtype type;
336 union {
337 struct ccp_dma_info dma;
338 u32 ksb;
339 } u;
340};
341
342struct ccp_aes_op {
343 enum ccp_aes_type type;
344 enum ccp_aes_mode mode;
345 enum ccp_aes_action action;
346};
347
348struct ccp_xts_aes_op {
349 enum ccp_aes_action action;
350 enum ccp_xts_aes_unit_size unit_size;
351};
352
353struct ccp_sha_op {
354 enum ccp_sha_type type;
355 u64 msg_bits;
356};
357
358struct ccp_rsa_op {
359 u32 mod_size;
360 u32 input_len;
361};
362
363struct ccp_passthru_op {
364 enum ccp_passthru_bitwise bit_mod;
365 enum ccp_passthru_byteswap byte_swap;
366};
367
368struct ccp_ecc_op {
369 enum ccp_ecc_function function;
370};
371
372struct ccp_op {
373 struct ccp_cmd_queue *cmd_q;
374
375 u32 jobid;
376 u32 ioc;
377 u32 soc;
378 u32 ksb_key;
379 u32 ksb_ctx;
380 u32 init;
381 u32 eom;
382
383 struct ccp_mem src;
384 struct ccp_mem dst;
385
386 union {
387 struct ccp_aes_op aes;
388 struct ccp_xts_aes_op xts;
389 struct ccp_sha_op sha;
390 struct ccp_rsa_op rsa;
391 struct ccp_passthru_op passthru;
392 struct ccp_ecc_op ecc;
393 } u;
394};
395
396static inline u32 ccp_addr_lo(struct ccp_dma_info *info)
397{
398 return lower_32_bits(info->address + info->offset);
399}
400
401static inline u32 ccp_addr_hi(struct ccp_dma_info *info)
402{
403 return upper_32_bits(info->address + info->offset) & 0x0000ffff;
404}
405
406int ccp_pci_init(void);
407void ccp_pci_exit(void);
408
409int ccp_platform_init(void);
410void ccp_platform_exit(void);
411
412void ccp_add_device(struct ccp_device *ccp);
413void ccp_del_device(struct ccp_device *ccp);
414
415struct ccp_device *ccp_alloc_struct(struct device *dev);
416bool ccp_queues_suspended(struct ccp_device *ccp);
417int ccp_cmd_queue_thread(void *data);
418
419int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd);
420
421#endif
1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#ifndef __CCP_DEV_H__
14#define __CCP_DEV_H__
15
16#include <linux/device.h>
17#include <linux/pci.h>
18#include <linux/spinlock.h>
19#include <linux/mutex.h>
20#include <linux/list.h>
21#include <linux/wait.h>
22#include <linux/dmapool.h>
23#include <linux/hw_random.h>
24
25
26#define IO_OFFSET 0x20000
27
28#define MAX_DMAPOOL_NAME_LEN 32
29
30#define MAX_HW_QUEUES 5
31#define MAX_CMD_QLEN 100
32
33#define TRNG_RETRIES 10
34
35
36/****** Register Mappings ******/
37#define Q_MASK_REG 0x000
38#define TRNG_OUT_REG 0x00c
39#define IRQ_MASK_REG 0x040
40#define IRQ_STATUS_REG 0x200
41
42#define DEL_CMD_Q_JOB 0x124
43#define DEL_Q_ACTIVE 0x00000200
44#define DEL_Q_ID_SHIFT 6
45
46#define CMD_REQ0 0x180
47#define CMD_REQ_INCR 0x04
48
49#define CMD_Q_STATUS_BASE 0x210
50#define CMD_Q_INT_STATUS_BASE 0x214
51#define CMD_Q_STATUS_INCR 0x20
52
53#define CMD_Q_CACHE 0x228
54#define CMD_Q_CACHE_INC 0x20
55
56#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f);
57#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f);
58
59/****** REQ0 Related Values ******/
60#define REQ0_WAIT_FOR_WRITE 0x00000004
61#define REQ0_INT_ON_COMPLETE 0x00000002
62#define REQ0_STOP_ON_COMPLETE 0x00000001
63
64#define REQ0_CMD_Q_SHIFT 9
65#define REQ0_JOBID_SHIFT 3
66
67/****** REQ1 Related Values ******/
68#define REQ1_PROTECT_SHIFT 27
69#define REQ1_ENGINE_SHIFT 23
70#define REQ1_KEY_KSB_SHIFT 2
71
72#define REQ1_EOM 0x00000002
73#define REQ1_INIT 0x00000001
74
75/* AES Related Values */
76#define REQ1_AES_TYPE_SHIFT 21
77#define REQ1_AES_MODE_SHIFT 18
78#define REQ1_AES_ACTION_SHIFT 17
79#define REQ1_AES_CFB_SIZE_SHIFT 10
80
81/* XTS-AES Related Values */
82#define REQ1_XTS_AES_SIZE_SHIFT 10
83
84/* SHA Related Values */
85#define REQ1_SHA_TYPE_SHIFT 21
86
87/* RSA Related Values */
88#define REQ1_RSA_MOD_SIZE_SHIFT 10
89
90/* Pass-Through Related Values */
91#define REQ1_PT_BW_SHIFT 12
92#define REQ1_PT_BS_SHIFT 10
93
94/* ECC Related Values */
95#define REQ1_ECC_AFFINE_CONVERT 0x00200000
96#define REQ1_ECC_FUNCTION_SHIFT 18
97
98/****** REQ4 Related Values ******/
99#define REQ4_KSB_SHIFT 18
100#define REQ4_MEMTYPE_SHIFT 16
101
102/****** REQ6 Related Values ******/
103#define REQ6_MEMTYPE_SHIFT 16
104
105
106/****** Key Storage Block ******/
107#define KSB_START 77
108#define KSB_END 127
109#define KSB_COUNT (KSB_END - KSB_START + 1)
110#define CCP_KSB_BITS 256
111#define CCP_KSB_BYTES 32
112
113#define CCP_JOBID_MASK 0x0000003f
114
115#define CCP_DMAPOOL_MAX_SIZE 64
116#define CCP_DMAPOOL_ALIGN (1 << 5)
117
118#define CCP_REVERSE_BUF_SIZE 64
119
120#define CCP_AES_KEY_KSB_COUNT 1
121#define CCP_AES_CTX_KSB_COUNT 1
122
123#define CCP_XTS_AES_KEY_KSB_COUNT 1
124#define CCP_XTS_AES_CTX_KSB_COUNT 1
125
126#define CCP_SHA_KSB_COUNT 1
127
128#define CCP_RSA_MAX_WIDTH 4096
129
130#define CCP_PASSTHRU_BLOCKSIZE 256
131#define CCP_PASSTHRU_MASKSIZE 32
132#define CCP_PASSTHRU_KSB_COUNT 1
133
134#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */
135#define CCP_ECC_MAX_OPERANDS 6
136#define CCP_ECC_MAX_OUTPUTS 3
137#define CCP_ECC_SRC_BUF_SIZE 448
138#define CCP_ECC_DST_BUF_SIZE 192
139#define CCP_ECC_OPERAND_SIZE 64
140#define CCP_ECC_OUTPUT_SIZE 64
141#define CCP_ECC_RESULT_OFFSET 60
142#define CCP_ECC_RESULT_SUCCESS 0x0001
143
144
145struct ccp_device;
146struct ccp_cmd;
147
148struct ccp_cmd_queue {
149 struct ccp_device *ccp;
150
151 /* Queue identifier */
152 u32 id;
153
154 /* Queue dma pool */
155 struct dma_pool *dma_pool;
156
157 /* Queue reserved KSB regions */
158 u32 ksb_key;
159 u32 ksb_ctx;
160
161 /* Queue processing thread */
162 struct task_struct *kthread;
163 unsigned int active;
164 unsigned int suspended;
165
166 /* Number of free command slots available */
167 unsigned int free_slots;
168
169 /* Interrupt masks */
170 u32 int_ok;
171 u32 int_err;
172
173 /* Register addresses for queue */
174 void __iomem *reg_status;
175 void __iomem *reg_int_status;
176
177 /* Status values from job */
178 u32 int_status;
179 u32 q_status;
180 u32 q_int_status;
181 u32 cmd_error;
182
183 /* Interrupt wait queue */
184 wait_queue_head_t int_queue;
185 unsigned int int_rcvd;
186} ____cacheline_aligned;
187
188struct ccp_device {
189 struct device *dev;
190
191 /*
192 * Bus specific device information
193 */
194 void *dev_specific;
195 int (*get_irq)(struct ccp_device *ccp);
196 void (*free_irq)(struct ccp_device *ccp);
197
198 /*
199 * I/O area used for device communication. The register mapping
200 * starts at an offset into the mapped bar.
201 * The CMD_REQx registers and the Delete_Cmd_Queue_Job register
202 * need to be protected while a command queue thread is accessing
203 * them.
204 */
205 struct mutex req_mutex ____cacheline_aligned;
206 void __iomem *io_map;
207 void __iomem *io_regs;
208
209 /*
210 * Master lists that all cmds are queued on. Because there can be
211 * more than one CCP command queue that can process a cmd a separate
212 * backlog list is neeeded so that the backlog completion call
213 * completes before the cmd is available for execution.
214 */
215 spinlock_t cmd_lock ____cacheline_aligned;
216 unsigned int cmd_count;
217 struct list_head cmd;
218 struct list_head backlog;
219
220 /*
221 * The command queues. These represent the queues available on the
222 * CCP that are available for processing cmds
223 */
224 struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
225 unsigned int cmd_q_count;
226
227 /*
228 * Support for the CCP True RNG
229 */
230 struct hwrng hwrng;
231 unsigned int hwrng_retries;
232
233 /*
234 * A counter used to generate job-ids for cmds submitted to the CCP
235 */
236 atomic_t current_id ____cacheline_aligned;
237
238 /*
239 * The CCP uses key storage blocks (KSB) to maintain context for certain
240 * operations. To prevent multiple cmds from using the same KSB range
241 * a command queue reserves a KSB range for the duration of the cmd.
242 * Each queue, will however, reserve 2 KSB blocks for operations that
243 * only require single KSB entries (eg. AES context/iv and key) in order
244 * to avoid allocation contention. This will reserve at most 10 KSB
245 * entries, leaving 40 KSB entries available for dynamic allocation.
246 */
247 struct mutex ksb_mutex ____cacheline_aligned;
248 DECLARE_BITMAP(ksb, KSB_COUNT);
249 wait_queue_head_t ksb_queue;
250 unsigned int ksb_avail;
251 unsigned int ksb_count;
252 u32 ksb_start;
253
254 /* Suspend support */
255 unsigned int suspending;
256 wait_queue_head_t suspend_queue;
257};
258
259
260int ccp_pci_init(void);
261void ccp_pci_exit(void);
262
263struct ccp_device *ccp_alloc_struct(struct device *dev);
264int ccp_init(struct ccp_device *ccp);
265void ccp_destroy(struct ccp_device *ccp);
266bool ccp_queues_suspended(struct ccp_device *ccp);
267
268irqreturn_t ccp_irq_handler(int irq, void *data);
269
270int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd);
271
272#endif