Loading...
1
2/* SPDX-License-Identifier: GPL-2.0-only */
3/*
4 * Copyright 2016 Broadcom
5 */
6
7#ifndef _CIPHER_H
8#define _CIPHER_H
9
10#include <linux/atomic.h>
11#include <linux/mailbox/brcm-message.h>
12#include <linux/mailbox_client.h>
13#include <crypto/aes.h>
14#include <crypto/internal/hash.h>
15#include <crypto/internal/skcipher.h>
16#include <crypto/aead.h>
17#include <crypto/arc4.h>
18#include <crypto/gcm.h>
19#include <crypto/sha1.h>
20#include <crypto/sha2.h>
21#include <crypto/sha3.h>
22
23#include "spu.h"
24#include "spum.h"
25#include "spu2.h"
26
27/* Driver supports up to MAX_SPUS SPU blocks */
28#define MAX_SPUS 16
29
30#define ARC4_STATE_SIZE 4
31
32#define CCM_AES_IV_SIZE 16
33#define CCM_ESP_IV_SIZE 8
34#define RFC4543_ICV_SIZE 16
35
36#define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE
37#define MAX_IV_SIZE AES_BLOCK_SIZE
38#define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE
39#define MAX_ASSOC_SIZE 512
40
41/* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
42#define GCM_ESP_SALT_SIZE 4
43#define CCM_ESP_SALT_SIZE 3
44#define MAX_SALT_SIZE GCM_ESP_SALT_SIZE
45#define GCM_ESP_SALT_OFFSET 0
46#define CCM_ESP_SALT_OFFSET 1
47
48#define GCM_ESP_DIGESTSIZE 16
49
50#define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
51
52/*
53 * Maximum number of bytes from a non-final hash request that can be deferred
54 * until more data is available. With new crypto API framework, this
55 * can be no more than one block of data.
56 */
57#define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE
58
59/* Force at least 4-byte alignment of all SPU message fields */
60#define SPU_MSG_ALIGN 4
61
62/* Number of times to resend mailbox message if mb queue is full */
63#define SPU_MB_RETRY_MAX 1000
64
65/* op_counts[] indexes */
66enum op_type {
67 SPU_OP_CIPHER,
68 SPU_OP_HASH,
69 SPU_OP_HMAC,
70 SPU_OP_AEAD,
71 SPU_OP_NUM
72};
73
74enum spu_spu_type {
75 SPU_TYPE_SPUM,
76 SPU_TYPE_SPU2,
77};
78
79/*
80 * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
81 * respectively.
82 */
83enum spu_spu_subtype {
84 SPU_SUBTYPE_SPUM_NS2,
85 SPU_SUBTYPE_SPUM_NSP,
86 SPU_SUBTYPE_SPU2_V1,
87 SPU_SUBTYPE_SPU2_V2
88};
89
90struct spu_type_subtype {
91 enum spu_spu_type type;
92 enum spu_spu_subtype subtype;
93};
94
95struct cipher_op {
96 enum spu_cipher_alg alg;
97 enum spu_cipher_mode mode;
98};
99
100struct auth_op {
101 enum hash_alg alg;
102 enum hash_mode mode;
103};
104
105struct iproc_alg_s {
106 u32 type;
107 union {
108 struct skcipher_alg skcipher;
109 struct ahash_alg hash;
110 struct aead_alg aead;
111 } alg;
112 struct cipher_op cipher_info;
113 struct auth_op auth_info;
114 bool auth_first;
115 bool registered;
116};
117
118/*
119 * Buffers for a SPU request/reply message pair. All part of one structure to
120 * allow a single alloc per request.
121 */
122struct spu_msg_buf {
123 /* Request message fragments */
124
125 /*
126 * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
127 * and BD header. For SPU2, holds FMD, OMD.
128 */
129 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
130
131 /* IV or counter. Size to include salt. Also used for XTS tweek. */
132 u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
133
134 /* Hash digest. request and response. */
135 u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
136
137 /* SPU request message padding */
138 u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
139
140 /* SPU-M request message STATUS field */
141 u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
142
143 /* Response message fragments */
144
145 /* SPU response message header */
146 u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
147
148 /* SPU response message STATUS field padding */
149 u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
150
151 /* SPU response message STATUS field */
152 u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
153
154 union {
155 /* Buffers only used for skcipher */
156 struct {
157 /*
158 * Field used for either SUPDT when RC4 is used
159 * -OR- tweak value when XTS/AES is used
160 */
161 u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
162 } c;
163
164 /* Buffers only used for aead */
165 struct {
166 /* SPU response pad for GCM data */
167 u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
168
169 /* SPU request msg padding for GCM AAD */
170 u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
171
172 /* SPU response data to be discarded */
173 u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
174 SPU_MSG_ALIGN)];
175 } a;
176 };
177};
178
179struct iproc_ctx_s {
180 u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
181 unsigned int enckeylen;
182
183 u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
184 unsigned int authkeylen;
185
186 u8 salt[MAX_SALT_SIZE];
187 unsigned int salt_len;
188 unsigned int salt_offset;
189 u8 iv[MAX_IV_SIZE];
190
191 unsigned int digestsize;
192
193 struct iproc_alg_s *alg;
194 bool is_esp;
195
196 struct cipher_op cipher;
197 enum spu_cipher_type cipher_type;
198
199 struct auth_op auth;
200 bool auth_first;
201
202 /*
203 * The maximum length in bytes of the payload in a SPU message for this
204 * context. For SPU-M, the payload is the combination of AAD and data.
205 * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
206 * indicates that there is no limit to the length of the SPU message
207 * payload.
208 */
209 unsigned int max_payload;
210
211 struct crypto_aead *fallback_cipher;
212
213 /* auth_type is determined during processing of request */
214
215 u8 ipad[MAX_HASH_BLOCK_SIZE];
216 u8 opad[MAX_HASH_BLOCK_SIZE];
217
218 /*
219 * Buffer to hold SPU message header template. Template is created at
220 * setkey time for skcipher requests, since most of the fields in the
221 * header are known at that time. At request time, just fill in a few
222 * missing pieces related to length of data in the request and IVs, etc.
223 */
224 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
225
226 /* Length of SPU request header */
227 u16 spu_req_hdr_len;
228
229 /* Expected length of SPU response header */
230 u16 spu_resp_hdr_len;
231
232 /*
233 * shash descriptor - needed to perform incremental hashing in
234 * software, when hw doesn't support it.
235 */
236 struct shash_desc *shash;
237
238 bool is_rfc4543; /* RFC 4543 style of GMAC */
239};
240
241/* state from iproc_reqctx_s necessary for hash state export/import */
242struct spu_hash_export_s {
243 unsigned int total_todo;
244 unsigned int total_sent;
245 u8 hash_carry[HASH_CARRY_MAX];
246 unsigned int hash_carry_len;
247 u8 incr_hash[MAX_DIGEST_SIZE];
248 bool is_sw_hmac;
249};
250
251struct iproc_reqctx_s {
252 /* general context */
253 struct crypto_async_request *parent;
254
255 /* only valid after enqueue() */
256 struct iproc_ctx_s *ctx;
257
258 u8 chan_idx; /* Mailbox channel to be used to submit this request */
259
260 /* total todo, rx'd, and sent for this request */
261 unsigned int total_todo;
262 unsigned int total_received; /* only valid for skcipher */
263 unsigned int total_sent;
264
265 /*
266 * num bytes sent to hw from the src sg in this request. This can differ
267 * from total_sent for incremental hashing. total_sent includes previous
268 * init() and update() data. src_sent does not.
269 */
270 unsigned int src_sent;
271
272 /*
273 * For AEAD requests, start of associated data. This will typically
274 * point to the beginning of the src scatterlist from the request,
275 * since assoc data is at the beginning of the src scatterlist rather
276 * than in its own sg.
277 */
278 struct scatterlist *assoc;
279
280 /*
281 * scatterlist entry and offset to start of data for next chunk. Crypto
282 * API src scatterlist for AEAD starts with AAD, if present. For first
283 * chunk, src_sg is sg entry at beginning of input data (after AAD).
284 * src_skip begins at the offset in that sg entry where data begins.
285 */
286 struct scatterlist *src_sg;
287 int src_nents; /* Number of src entries with data */
288 u32 src_skip; /* bytes of current sg entry already used */
289
290 /*
291 * Same for destination. For AEAD, if there is AAD, output data must
292 * be written at offset following AAD.
293 */
294 struct scatterlist *dst_sg;
295 int dst_nents; /* Number of dst entries with data */
296 u32 dst_skip; /* bytes of current sg entry already written */
297
298 /* Mailbox message used to send this request to PDC driver */
299 struct brcm_message mb_mssg;
300
301 bool bd_suppress; /* suppress BD field in SPU response? */
302
303 /* cipher context */
304 bool is_encrypt;
305
306 /*
307 * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA
308 * buffer for AEAD requests. So allocate as DMAable memory. If IV
309 * concatenated with salt, includes the salt.
310 */
311 u8 *iv_ctr;
312 /* Length of IV or counter, in bytes */
313 unsigned int iv_ctr_len;
314
315 /*
316 * Hash requests can be of any size, whether initial, update, or final.
317 * A non-final request must be submitted to the SPU as an integral
318 * number of blocks. This may leave data at the end of the request
319 * that is not a full block. Since the request is non-final, it cannot
320 * be padded. So, we write the remainder to this hash_carry buffer and
321 * hold it until the next request arrives. The carry data is then
322 * submitted at the beginning of the data in the next SPU msg.
323 * hash_carry_len is the number of bytes currently in hash_carry. These
324 * fields are only used for ahash requests.
325 */
326 u8 hash_carry[HASH_CARRY_MAX];
327 unsigned int hash_carry_len;
328 unsigned int is_final; /* is this the final for the hash op? */
329
330 /*
331 * Digest from incremental hash is saved here to include in next hash
332 * operation. Cannot be stored in req->result for truncated hashes,
333 * since result may be sized for final digest. Cannot be saved in
334 * msg_buf because that gets deleted between incremental hash ops
335 * and is not saved as part of export().
336 */
337 u8 incr_hash[MAX_DIGEST_SIZE];
338
339 /* hmac context */
340 bool is_sw_hmac;
341
342 gfp_t gfp;
343
344 /* Buffers used to build SPU request and response messages */
345 struct spu_msg_buf msg_buf;
346
347 struct aead_request req;
348};
349
350/*
351 * Structure encapsulates a set of function pointers specific to the type of
352 * SPU hardware running. These functions handling creation and parsing of
353 * SPU request messages and SPU response messages. Includes hardware-specific
354 * values read from device tree.
355 */
356struct spu_hw {
357 void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
358 u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
359 enum spu_cipher_mode cipher_mode,
360 unsigned int blocksize);
361 u32 (*spu_payload_length)(u8 *spu_hdr);
362 u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
363 bool is_hash);
364 u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
365 enum hash_mode hash_mode, u32 chunksize,
366 u16 hash_block_size);
367 u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
368 unsigned int data_size);
369 u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
370 unsigned int assoc_len,
371 unsigned int iv_len, bool is_encrypt);
372 u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
373 u16 iv_len);
374 enum hash_type (*spu_hash_type)(u32 src_sent);
375 u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
376 enum hash_type);
377 u32 (*spu_create_request)(u8 *spu_hdr,
378 struct spu_request_opts *req_opts,
379 struct spu_cipher_parms *cipher_parms,
380 struct spu_hash_parms *hash_parms,
381 struct spu_aead_parms *aead_parms,
382 unsigned int data_size);
383 u16 (*spu_cipher_req_init)(u8 *spu_hdr,
384 struct spu_cipher_parms *cipher_parms);
385 void (*spu_cipher_req_finish)(u8 *spu_hdr,
386 u16 spu_req_hdr_len,
387 unsigned int is_inbound,
388 struct spu_cipher_parms *cipher_parms,
389 unsigned int data_size);
390 void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
391 u32 hash_pad_len, enum hash_alg auth_alg,
392 enum hash_mode auth_mode,
393 unsigned int total_sent, u32 status_padding);
394 u8 (*spu_xts_tweak_in_payload)(void);
395 u8 (*spu_tx_status_len)(void);
396 u8 (*spu_rx_status_len)(void);
397 int (*spu_status_process)(u8 *statp);
398 void (*spu_ccm_update_iv)(unsigned int digestsize,
399 struct spu_cipher_parms *cipher_parms,
400 unsigned int assoclen, unsigned int chunksize,
401 bool is_encrypt, bool is_esp);
402 u32 (*spu_wordalign_padlen)(u32 data_size);
403
404 /* The base virtual address of the SPU hw registers */
405 void __iomem *reg_vbase[MAX_SPUS];
406
407 /* Version of the SPU hardware */
408 enum spu_spu_type spu_type;
409
410 /* Sub-version of the SPU hardware */
411 enum spu_spu_subtype spu_subtype;
412
413 /* The number of SPUs on this platform */
414 u32 num_spu;
415
416 /* The number of SPU channels on this platform */
417 u32 num_chan;
418};
419
420struct bcm_device_private {
421 struct platform_device *pdev;
422
423 struct spu_hw spu;
424
425 atomic_t session_count; /* number of streams active */
426 atomic_t stream_count; /* monotonic counter for streamID's */
427
428 /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
429 u8 bcm_hdr_len;
430
431 /* The index of the channel to use for the next crypto request */
432 atomic_t next_chan;
433
434 struct dentry *debugfs_dir;
435 struct dentry *debugfs_stats;
436
437 /* Number of request bytes processed and result bytes returned */
438 atomic64_t bytes_in;
439 atomic64_t bytes_out;
440
441 /* Number of operations of each type */
442 atomic_t op_counts[SPU_OP_NUM];
443
444 atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
445 atomic_t hash_cnt[HASH_ALG_LAST];
446 atomic_t hmac_cnt[HASH_ALG_LAST];
447 atomic_t aead_cnt[AEAD_TYPE_LAST];
448
449 /* Number of calls to setkey() for each operation type */
450 atomic_t setkey_cnt[SPU_OP_NUM];
451
452 /* Number of times request was resubmitted because mb was full */
453 atomic_t mb_no_spc;
454
455 /* Number of mailbox send failures */
456 atomic_t mb_send_fail;
457
458 /* Number of ICV check failures for AEAD messages */
459 atomic_t bad_icv;
460
461 struct mbox_client mcl;
462
463 /* Array of mailbox channel pointers, one for each channel */
464 struct mbox_chan **mbox;
465};
466
467extern struct bcm_device_private iproc_priv;
468
469#endif
1/*
2 * Copyright 2016 Broadcom
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation (the "GPL").
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License version 2 (GPLv2) for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * version 2 (GPLv2) along with this source code.
15 */
16
17#ifndef _CIPHER_H
18#define _CIPHER_H
19
20#include <linux/atomic.h>
21#include <linux/mailbox/brcm-message.h>
22#include <linux/mailbox_client.h>
23#include <crypto/aes.h>
24#include <crypto/internal/hash.h>
25#include <crypto/aead.h>
26#include <crypto/gcm.h>
27#include <crypto/sha.h>
28#include <crypto/sha3.h>
29
30#include "spu.h"
31#include "spum.h"
32#include "spu2.h"
33
34/* Driver supports up to MAX_SPUS SPU blocks */
35#define MAX_SPUS 16
36
37#define ARC4_MIN_KEY_SIZE 1
38#define ARC4_MAX_KEY_SIZE 256
39#define ARC4_BLOCK_SIZE 1
40#define ARC4_STATE_SIZE 4
41
42#define CCM_AES_IV_SIZE 16
43#define CCM_ESP_IV_SIZE 8
44#define RFC4543_ICV_SIZE 16
45
46#define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE
47#define MAX_IV_SIZE AES_BLOCK_SIZE
48#define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE
49#define MAX_ASSOC_SIZE 512
50
51/* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
52#define GCM_ESP_SALT_SIZE 4
53#define CCM_ESP_SALT_SIZE 3
54#define MAX_SALT_SIZE GCM_ESP_SALT_SIZE
55#define GCM_ESP_SALT_OFFSET 0
56#define CCM_ESP_SALT_OFFSET 1
57
58#define GCM_ESP_DIGESTSIZE 16
59
60#define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
61
62/*
63 * Maximum number of bytes from a non-final hash request that can be deferred
64 * until more data is available. With new crypto API framework, this
65 * can be no more than one block of data.
66 */
67#define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE
68
69/* Force at least 4-byte alignment of all SPU message fields */
70#define SPU_MSG_ALIGN 4
71
72/* Number of times to resend mailbox message if mb queue is full */
73#define SPU_MB_RETRY_MAX 1000
74
75/* op_counts[] indexes */
76enum op_type {
77 SPU_OP_CIPHER,
78 SPU_OP_HASH,
79 SPU_OP_HMAC,
80 SPU_OP_AEAD,
81 SPU_OP_NUM
82};
83
84enum spu_spu_type {
85 SPU_TYPE_SPUM,
86 SPU_TYPE_SPU2,
87};
88
89/*
90 * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
91 * respectively.
92 */
93enum spu_spu_subtype {
94 SPU_SUBTYPE_SPUM_NS2,
95 SPU_SUBTYPE_SPUM_NSP,
96 SPU_SUBTYPE_SPU2_V1,
97 SPU_SUBTYPE_SPU2_V2
98};
99
100struct spu_type_subtype {
101 enum spu_spu_type type;
102 enum spu_spu_subtype subtype;
103};
104
105struct cipher_op {
106 enum spu_cipher_alg alg;
107 enum spu_cipher_mode mode;
108};
109
110struct auth_op {
111 enum hash_alg alg;
112 enum hash_mode mode;
113};
114
115struct iproc_alg_s {
116 u32 type;
117 union {
118 struct crypto_alg crypto;
119 struct ahash_alg hash;
120 struct aead_alg aead;
121 } alg;
122 struct cipher_op cipher_info;
123 struct auth_op auth_info;
124 bool auth_first;
125 bool registered;
126};
127
128/*
129 * Buffers for a SPU request/reply message pair. All part of one structure to
130 * allow a single alloc per request.
131 */
132struct spu_msg_buf {
133 /* Request message fragments */
134
135 /*
136 * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
137 * and BD header. For SPU2, holds FMD, OMD.
138 */
139 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
140
141 /* IV or counter. Size to include salt. Also used for XTS tweek. */
142 u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
143
144 /* Hash digest. request and response. */
145 u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
146
147 /* SPU request message padding */
148 u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
149
150 /* SPU-M request message STATUS field */
151 u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
152
153 /* Response message fragments */
154
155 /* SPU response message header */
156 u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
157
158 /* SPU response message STATUS field padding */
159 u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
160
161 /* SPU response message STATUS field */
162 u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
163
164 union {
165 /* Buffers only used for ablkcipher */
166 struct {
167 /*
168 * Field used for either SUPDT when RC4 is used
169 * -OR- tweak value when XTS/AES is used
170 */
171 u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
172 } c;
173
174 /* Buffers only used for aead */
175 struct {
176 /* SPU response pad for GCM data */
177 u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
178
179 /* SPU request msg padding for GCM AAD */
180 u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
181
182 /* SPU response data to be discarded */
183 u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
184 SPU_MSG_ALIGN)];
185 } a;
186 };
187};
188
189struct iproc_ctx_s {
190 u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
191 unsigned int enckeylen;
192
193 u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
194 unsigned int authkeylen;
195
196 u8 salt[MAX_SALT_SIZE];
197 unsigned int salt_len;
198 unsigned int salt_offset;
199 u8 iv[MAX_IV_SIZE];
200
201 unsigned int digestsize;
202
203 struct iproc_alg_s *alg;
204 bool is_esp;
205
206 struct cipher_op cipher;
207 enum spu_cipher_type cipher_type;
208
209 struct auth_op auth;
210 bool auth_first;
211
212 /*
213 * The maximum length in bytes of the payload in a SPU message for this
214 * context. For SPU-M, the payload is the combination of AAD and data.
215 * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
216 * indicates that there is no limit to the length of the SPU message
217 * payload.
218 */
219 unsigned int max_payload;
220
221 struct crypto_aead *fallback_cipher;
222
223 /* auth_type is determined during processing of request */
224
225 u8 ipad[MAX_HASH_BLOCK_SIZE];
226 u8 opad[MAX_HASH_BLOCK_SIZE];
227
228 /*
229 * Buffer to hold SPU message header template. Template is created at
230 * setkey time for ablkcipher requests, since most of the fields in the
231 * header are known at that time. At request time, just fill in a few
232 * missing pieces related to length of data in the request and IVs, etc.
233 */
234 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
235
236 /* Length of SPU request header */
237 u16 spu_req_hdr_len;
238
239 /* Expected length of SPU response header */
240 u16 spu_resp_hdr_len;
241
242 /*
243 * shash descriptor - needed to perform incremental hashing in
244 * in software, when hw doesn't support it.
245 */
246 struct shash_desc *shash;
247
248 bool is_rfc4543; /* RFC 4543 style of GMAC */
249};
250
251/* state from iproc_reqctx_s necessary for hash state export/import */
252struct spu_hash_export_s {
253 unsigned int total_todo;
254 unsigned int total_sent;
255 u8 hash_carry[HASH_CARRY_MAX];
256 unsigned int hash_carry_len;
257 u8 incr_hash[MAX_DIGEST_SIZE];
258 bool is_sw_hmac;
259};
260
261struct iproc_reqctx_s {
262 /* general context */
263 struct crypto_async_request *parent;
264
265 /* only valid after enqueue() */
266 struct iproc_ctx_s *ctx;
267
268 u8 chan_idx; /* Mailbox channel to be used to submit this request */
269
270 /* total todo, rx'd, and sent for this request */
271 unsigned int total_todo;
272 unsigned int total_received; /* only valid for ablkcipher */
273 unsigned int total_sent;
274
275 /*
276 * num bytes sent to hw from the src sg in this request. This can differ
277 * from total_sent for incremental hashing. total_sent includes previous
278 * init() and update() data. src_sent does not.
279 */
280 unsigned int src_sent;
281
282 /*
283 * For AEAD requests, start of associated data. This will typically
284 * point to the beginning of the src scatterlist from the request,
285 * since assoc data is at the beginning of the src scatterlist rather
286 * than in its own sg.
287 */
288 struct scatterlist *assoc;
289
290 /*
291 * scatterlist entry and offset to start of data for next chunk. Crypto
292 * API src scatterlist for AEAD starts with AAD, if present. For first
293 * chunk, src_sg is sg entry at beginning of input data (after AAD).
294 * src_skip begins at the offset in that sg entry where data begins.
295 */
296 struct scatterlist *src_sg;
297 int src_nents; /* Number of src entries with data */
298 u32 src_skip; /* bytes of current sg entry already used */
299
300 /*
301 * Same for destination. For AEAD, if there is AAD, output data must
302 * be written at offset following AAD.
303 */
304 struct scatterlist *dst_sg;
305 int dst_nents; /* Number of dst entries with data */
306 u32 dst_skip; /* bytes of current sg entry already written */
307
308 /* Mailbox message used to send this request to PDC driver */
309 struct brcm_message mb_mssg;
310
311 bool bd_suppress; /* suppress BD field in SPU response? */
312
313 /* cipher context */
314 bool is_encrypt;
315
316 /*
317 * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA
318 * buffer for AEAD requests. So allocate as DMAable memory. If IV
319 * concatenated with salt, includes the salt.
320 */
321 u8 *iv_ctr;
322 /* Length of IV or counter, in bytes */
323 unsigned int iv_ctr_len;
324
325 /*
326 * Hash requests can be of any size, whether initial, update, or final.
327 * A non-final request must be submitted to the SPU as an integral
328 * number of blocks. This may leave data at the end of the request
329 * that is not a full block. Since the request is non-final, it cannot
330 * be padded. So, we write the remainder to this hash_carry buffer and
331 * hold it until the next request arrives. The carry data is then
332 * submitted at the beginning of the data in the next SPU msg.
333 * hash_carry_len is the number of bytes currently in hash_carry. These
334 * fields are only used for ahash requests.
335 */
336 u8 hash_carry[HASH_CARRY_MAX];
337 unsigned int hash_carry_len;
338 unsigned int is_final; /* is this the final for the hash op? */
339
340 /*
341 * Digest from incremental hash is saved here to include in next hash
342 * operation. Cannot be stored in req->result for truncated hashes,
343 * since result may be sized for final digest. Cannot be saved in
344 * msg_buf because that gets deleted between incremental hash ops
345 * and is not saved as part of export().
346 */
347 u8 incr_hash[MAX_DIGEST_SIZE];
348
349 /* hmac context */
350 bool is_sw_hmac;
351
352 /* aead context */
353 struct crypto_tfm *old_tfm;
354 crypto_completion_t old_complete;
355 void *old_data;
356
357 gfp_t gfp;
358
359 /* Buffers used to build SPU request and response messages */
360 struct spu_msg_buf msg_buf;
361};
362
363/*
364 * Structure encapsulates a set of function pointers specific to the type of
365 * SPU hardware running. These functions handling creation and parsing of
366 * SPU request messages and SPU response messages. Includes hardware-specific
367 * values read from device tree.
368 */
369struct spu_hw {
370 void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
371 u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
372 enum spu_cipher_mode cipher_mode,
373 unsigned int blocksize);
374 u32 (*spu_payload_length)(u8 *spu_hdr);
375 u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
376 bool is_hash);
377 u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
378 enum hash_mode hash_mode, u32 chunksize,
379 u16 hash_block_size);
380 u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
381 unsigned int data_size);
382 u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
383 unsigned int assoc_len,
384 unsigned int iv_len, bool is_encrypt);
385 u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
386 u16 iv_len);
387 enum hash_type (*spu_hash_type)(u32 src_sent);
388 u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
389 enum hash_type);
390 u32 (*spu_create_request)(u8 *spu_hdr,
391 struct spu_request_opts *req_opts,
392 struct spu_cipher_parms *cipher_parms,
393 struct spu_hash_parms *hash_parms,
394 struct spu_aead_parms *aead_parms,
395 unsigned int data_size);
396 u16 (*spu_cipher_req_init)(u8 *spu_hdr,
397 struct spu_cipher_parms *cipher_parms);
398 void (*spu_cipher_req_finish)(u8 *spu_hdr,
399 u16 spu_req_hdr_len,
400 unsigned int is_inbound,
401 struct spu_cipher_parms *cipher_parms,
402 bool update_key,
403 unsigned int data_size);
404 void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
405 u32 hash_pad_len, enum hash_alg auth_alg,
406 enum hash_mode auth_mode,
407 unsigned int total_sent, u32 status_padding);
408 u8 (*spu_xts_tweak_in_payload)(void);
409 u8 (*spu_tx_status_len)(void);
410 u8 (*spu_rx_status_len)(void);
411 int (*spu_status_process)(u8 *statp);
412 void (*spu_ccm_update_iv)(unsigned int digestsize,
413 struct spu_cipher_parms *cipher_parms,
414 unsigned int assoclen, unsigned int chunksize,
415 bool is_encrypt, bool is_esp);
416 u32 (*spu_wordalign_padlen)(u32 data_size);
417
418 /* The base virtual address of the SPU hw registers */
419 void __iomem *reg_vbase[MAX_SPUS];
420
421 /* Version of the SPU hardware */
422 enum spu_spu_type spu_type;
423
424 /* Sub-version of the SPU hardware */
425 enum spu_spu_subtype spu_subtype;
426
427 /* The number of SPUs on this platform */
428 u32 num_spu;
429
430 /* The number of SPU channels on this platform */
431 u32 num_chan;
432};
433
434struct device_private {
435 struct platform_device *pdev;
436
437 struct spu_hw spu;
438
439 atomic_t session_count; /* number of streams active */
440 atomic_t stream_count; /* monotonic counter for streamID's */
441
442 /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
443 u8 bcm_hdr_len;
444
445 /* The index of the channel to use for the next crypto request */
446 atomic_t next_chan;
447
448 struct dentry *debugfs_dir;
449 struct dentry *debugfs_stats;
450
451 /* Number of request bytes processed and result bytes returned */
452 atomic64_t bytes_in;
453 atomic64_t bytes_out;
454
455 /* Number of operations of each type */
456 atomic_t op_counts[SPU_OP_NUM];
457
458 atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
459 atomic_t hash_cnt[HASH_ALG_LAST];
460 atomic_t hmac_cnt[HASH_ALG_LAST];
461 atomic_t aead_cnt[AEAD_TYPE_LAST];
462
463 /* Number of calls to setkey() for each operation type */
464 atomic_t setkey_cnt[SPU_OP_NUM];
465
466 /* Number of times request was resubmitted because mb was full */
467 atomic_t mb_no_spc;
468
469 /* Number of mailbox send failures */
470 atomic_t mb_send_fail;
471
472 /* Number of ICV check failures for AEAD messages */
473 atomic_t bad_icv;
474
475 struct mbox_client mcl;
476
477 /* Array of mailbox channel pointers, one for each channel */
478 struct mbox_chan **mbox;
479};
480
481extern struct device_private iproc_priv;
482
483#endif