Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2014 - 2020 Intel Corporation */
3#include <linux/module.h>
4#include <linux/slab.h>
5#include <linux/crypto.h>
6#include <crypto/internal/aead.h>
7#include <crypto/internal/cipher.h>
8#include <crypto/internal/skcipher.h>
9#include <crypto/aes.h>
10#include <crypto/sha1.h>
11#include <crypto/sha2.h>
12#include <crypto/hash.h>
13#include <crypto/hmac.h>
14#include <crypto/algapi.h>
15#include <crypto/authenc.h>
16#include <crypto/scatterwalk.h>
17#include <crypto/xts.h>
18#include <linux/dma-mapping.h>
19#include "adf_accel_devices.h"
20#include "qat_algs_send.h"
21#include "adf_common_drv.h"
22#include "qat_crypto.h"
23#include "icp_qat_hw.h"
24#include "icp_qat_fw.h"
25#include "icp_qat_fw_la.h"
26#include "qat_bl.h"
27
28#define QAT_AES_HW_CONFIG_ENC(alg, mode) \
29 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
30 ICP_QAT_HW_CIPHER_NO_CONVERT, \
31 ICP_QAT_HW_CIPHER_ENCRYPT)
32
33#define QAT_AES_HW_CONFIG_DEC(alg, mode) \
34 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
35 ICP_QAT_HW_CIPHER_KEY_CONVERT, \
36 ICP_QAT_HW_CIPHER_DECRYPT)
37
38#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \
39 ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
40 ICP_QAT_HW_CIPHER_NO_CONVERT, \
41 ICP_QAT_HW_CIPHER_DECRYPT)
42
43#define HW_CAP_AES_V2(accel_dev) \
44 (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \
45 ICP_ACCEL_CAPABILITIES_AES_V2)
46
47static DEFINE_MUTEX(algs_lock);
48static unsigned int active_devs;
49
50/* Common content descriptor */
51struct qat_alg_cd {
52 union {
53 struct qat_enc { /* Encrypt content desc */
54 struct icp_qat_hw_cipher_algo_blk cipher;
55 struct icp_qat_hw_auth_algo_blk hash;
56 } qat_enc_cd;
57 struct qat_dec { /* Decrypt content desc */
58 struct icp_qat_hw_auth_algo_blk hash;
59 struct icp_qat_hw_cipher_algo_blk cipher;
60 } qat_dec_cd;
61 };
62} __aligned(64);
63
64struct qat_alg_aead_ctx {
65 struct qat_alg_cd *enc_cd;
66 struct qat_alg_cd *dec_cd;
67 dma_addr_t enc_cd_paddr;
68 dma_addr_t dec_cd_paddr;
69 struct icp_qat_fw_la_bulk_req enc_fw_req;
70 struct icp_qat_fw_la_bulk_req dec_fw_req;
71 struct crypto_shash *hash_tfm;
72 enum icp_qat_hw_auth_algo qat_hash_alg;
73 struct qat_crypto_instance *inst;
74 union {
75 struct sha1_state sha1;
76 struct sha256_state sha256;
77 struct sha512_state sha512;
78 };
79 char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */
80 char opad[SHA512_BLOCK_SIZE];
81};
82
83struct qat_alg_skcipher_ctx {
84 struct icp_qat_hw_cipher_algo_blk *enc_cd;
85 struct icp_qat_hw_cipher_algo_blk *dec_cd;
86 dma_addr_t enc_cd_paddr;
87 dma_addr_t dec_cd_paddr;
88 struct icp_qat_fw_la_bulk_req enc_fw_req;
89 struct icp_qat_fw_la_bulk_req dec_fw_req;
90 struct qat_crypto_instance *inst;
91 struct crypto_skcipher *ftfm;
92 struct crypto_cipher *tweak;
93 bool fallback;
94 int mode;
95};
96
97static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
98{
99 switch (qat_hash_alg) {
100 case ICP_QAT_HW_AUTH_ALGO_SHA1:
101 return ICP_QAT_HW_SHA1_STATE1_SZ;
102 case ICP_QAT_HW_AUTH_ALGO_SHA256:
103 return ICP_QAT_HW_SHA256_STATE1_SZ;
104 case ICP_QAT_HW_AUTH_ALGO_SHA512:
105 return ICP_QAT_HW_SHA512_STATE1_SZ;
106 default:
107 return -EFAULT;
108 }
109}
110
111static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
112 struct qat_alg_aead_ctx *ctx,
113 const u8 *auth_key,
114 unsigned int auth_keylen)
115{
116 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
117 int block_size = crypto_shash_blocksize(ctx->hash_tfm);
118 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
119 __be32 *hash_state_out;
120 __be64 *hash512_state_out;
121 int i, offset;
122
123 memset(ctx->ipad, 0, block_size);
124 memset(ctx->opad, 0, block_size);
125 shash->tfm = ctx->hash_tfm;
126
127 if (auth_keylen > block_size) {
128 int ret = crypto_shash_digest(shash, auth_key,
129 auth_keylen, ctx->ipad);
130 if (ret)
131 return ret;
132
133 memcpy(ctx->opad, ctx->ipad, digest_size);
134 } else {
135 memcpy(ctx->ipad, auth_key, auth_keylen);
136 memcpy(ctx->opad, auth_key, auth_keylen);
137 }
138
139 for (i = 0; i < block_size; i++) {
140 char *ipad_ptr = ctx->ipad + i;
141 char *opad_ptr = ctx->opad + i;
142 *ipad_ptr ^= HMAC_IPAD_VALUE;
143 *opad_ptr ^= HMAC_OPAD_VALUE;
144 }
145
146 if (crypto_shash_init(shash))
147 return -EFAULT;
148
149 if (crypto_shash_update(shash, ctx->ipad, block_size))
150 return -EFAULT;
151
152 hash_state_out = (__be32 *)hash->sha.state1;
153 hash512_state_out = (__be64 *)hash_state_out;
154
155 switch (ctx->qat_hash_alg) {
156 case ICP_QAT_HW_AUTH_ALGO_SHA1:
157 if (crypto_shash_export(shash, &ctx->sha1))
158 return -EFAULT;
159 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
160 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
161 break;
162 case ICP_QAT_HW_AUTH_ALGO_SHA256:
163 if (crypto_shash_export(shash, &ctx->sha256))
164 return -EFAULT;
165 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
166 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
167 break;
168 case ICP_QAT_HW_AUTH_ALGO_SHA512:
169 if (crypto_shash_export(shash, &ctx->sha512))
170 return -EFAULT;
171 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
172 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
173 break;
174 default:
175 return -EFAULT;
176 }
177
178 if (crypto_shash_init(shash))
179 return -EFAULT;
180
181 if (crypto_shash_update(shash, ctx->opad, block_size))
182 return -EFAULT;
183
184 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
185 if (offset < 0)
186 return -EFAULT;
187
188 hash_state_out = (__be32 *)(hash->sha.state1 + offset);
189 hash512_state_out = (__be64 *)hash_state_out;
190
191 switch (ctx->qat_hash_alg) {
192 case ICP_QAT_HW_AUTH_ALGO_SHA1:
193 if (crypto_shash_export(shash, &ctx->sha1))
194 return -EFAULT;
195 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
196 *hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
197 break;
198 case ICP_QAT_HW_AUTH_ALGO_SHA256:
199 if (crypto_shash_export(shash, &ctx->sha256))
200 return -EFAULT;
201 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
202 *hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
203 break;
204 case ICP_QAT_HW_AUTH_ALGO_SHA512:
205 if (crypto_shash_export(shash, &ctx->sha512))
206 return -EFAULT;
207 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
208 *hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
209 break;
210 default:
211 return -EFAULT;
212 }
213 memzero_explicit(ctx->ipad, block_size);
214 memzero_explicit(ctx->opad, block_size);
215 return 0;
216}
217
218static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
219{
220 header->hdr_flags =
221 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
222 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
223 header->comn_req_flags =
224 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
225 QAT_COMN_PTR_TYPE_SGL);
226 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
227 ICP_QAT_FW_LA_PARTIAL_NONE);
228 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
229 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
230 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
231 ICP_QAT_FW_LA_NO_PROTO);
232 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
233 ICP_QAT_FW_LA_NO_UPDATE_STATE);
234}
235
236static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
237 int alg,
238 struct crypto_authenc_keys *keys,
239 int mode)
240{
241 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
242 unsigned int digestsize = crypto_aead_authsize(aead_tfm);
243 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
244 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
245 struct icp_qat_hw_auth_algo_blk *hash =
246 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
247 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
248 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
249 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
250 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
251 void *ptr = &req_tmpl->cd_ctrl;
252 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
253 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
254
255 /* CD setup */
256 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
257 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
258 hash->sha.inner_setup.auth_config.config =
259 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
260 ctx->qat_hash_alg, digestsize);
261 hash->sha.inner_setup.auth_counter.counter =
262 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
263
264 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
265 return -EFAULT;
266
267 /* Request setup */
268 qat_alg_init_common_hdr(header);
269 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
270 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
271 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
272 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
273 ICP_QAT_FW_LA_RET_AUTH_RES);
274 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
275 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
276 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
277 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
278
279 /* Cipher CD config setup */
280 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
281 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
282 cipher_cd_ctrl->cipher_cfg_offset = 0;
283 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
284 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
285 /* Auth CD config setup */
286 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
287 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
288 hash_cd_ctrl->inner_res_sz = digestsize;
289 hash_cd_ctrl->final_sz = digestsize;
290
291 switch (ctx->qat_hash_alg) {
292 case ICP_QAT_HW_AUTH_ALGO_SHA1:
293 hash_cd_ctrl->inner_state1_sz =
294 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
295 hash_cd_ctrl->inner_state2_sz =
296 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
297 break;
298 case ICP_QAT_HW_AUTH_ALGO_SHA256:
299 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
300 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
301 break;
302 case ICP_QAT_HW_AUTH_ALGO_SHA512:
303 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
304 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
305 break;
306 default:
307 break;
308 }
309 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
310 ((sizeof(struct icp_qat_hw_auth_setup) +
311 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
312 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
313 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
314 return 0;
315}
316
317static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
318 int alg,
319 struct crypto_authenc_keys *keys,
320 int mode)
321{
322 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
323 unsigned int digestsize = crypto_aead_authsize(aead_tfm);
324 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
325 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
326 struct icp_qat_hw_cipher_algo_blk *cipher =
327 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
328 sizeof(struct icp_qat_hw_auth_setup) +
329 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
330 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
331 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
332 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
333 void *ptr = &req_tmpl->cd_ctrl;
334 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
335 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
336 struct icp_qat_fw_la_auth_req_params *auth_param =
337 (struct icp_qat_fw_la_auth_req_params *)
338 ((char *)&req_tmpl->serv_specif_rqpars +
339 sizeof(struct icp_qat_fw_la_cipher_req_params));
340
341 /* CD setup */
342 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
343 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
344 hash->sha.inner_setup.auth_config.config =
345 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
346 ctx->qat_hash_alg,
347 digestsize);
348 hash->sha.inner_setup.auth_counter.counter =
349 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
350
351 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
352 return -EFAULT;
353
354 /* Request setup */
355 qat_alg_init_common_hdr(header);
356 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
357 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
358 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
359 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
360 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
361 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
362 ICP_QAT_FW_LA_CMP_AUTH_RES);
363 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
364 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
365
366 /* Cipher CD config setup */
367 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
368 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
369 cipher_cd_ctrl->cipher_cfg_offset =
370 (sizeof(struct icp_qat_hw_auth_setup) +
371 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
372 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
373 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
374
375 /* Auth CD config setup */
376 hash_cd_ctrl->hash_cfg_offset = 0;
377 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
378 hash_cd_ctrl->inner_res_sz = digestsize;
379 hash_cd_ctrl->final_sz = digestsize;
380
381 switch (ctx->qat_hash_alg) {
382 case ICP_QAT_HW_AUTH_ALGO_SHA1:
383 hash_cd_ctrl->inner_state1_sz =
384 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
385 hash_cd_ctrl->inner_state2_sz =
386 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
387 break;
388 case ICP_QAT_HW_AUTH_ALGO_SHA256:
389 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
390 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
391 break;
392 case ICP_QAT_HW_AUTH_ALGO_SHA512:
393 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
394 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
395 break;
396 default:
397 break;
398 }
399
400 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
401 ((sizeof(struct icp_qat_hw_auth_setup) +
402 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
403 auth_param->auth_res_sz = digestsize;
404 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
405 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
406 return 0;
407}
408
409static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,
410 struct icp_qat_fw_la_bulk_req *req,
411 struct icp_qat_hw_cipher_algo_blk *cd,
412 const u8 *key, unsigned int keylen)
413{
414 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
415 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
416 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
417 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
418 int mode = ctx->mode;
419
420 qat_alg_init_common_hdr(header);
421 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
422 cd_pars->u.s.content_desc_params_sz =
423 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
424
425 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
426 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
427 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
428
429 /* Store both XTS keys in CD, only the first key is sent
430 * to the HW, the second key is used for tweak calculation
431 */
432 memcpy(cd->ucs_aes.key, key, keylen);
433 keylen = keylen / 2;
434 } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
435 ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
436 ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
437 memcpy(cd->ucs_aes.key, key, keylen);
438 keylen = round_up(keylen, 16);
439 } else {
440 memcpy(cd->aes.key, key, keylen);
441 }
442
443 /* Cipher CD config setup */
444 cd_ctrl->cipher_key_sz = keylen >> 3;
445 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
446 cd_ctrl->cipher_cfg_offset = 0;
447 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
448 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
449}
450
451static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
452 int alg, const u8 *key,
453 unsigned int keylen, int mode)
454{
455 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
456 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
457 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
458
459 qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen);
460 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
461 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
462}
463
464static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
465 u8 *key_reverse)
466{
467 struct crypto_aes_ctx aes_expanded;
468 int nrounds;
469 u8 *key;
470
471 aes_expandkey(&aes_expanded, key_forward, keylen);
472 if (keylen == AES_KEYSIZE_128) {
473 nrounds = 10;
474 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
475 memcpy(key_reverse, key, AES_BLOCK_SIZE);
476 } else {
477 /* AES_KEYSIZE_256 */
478 nrounds = 14;
479 key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
480 memcpy(key_reverse, key, AES_BLOCK_SIZE);
481 memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
482 AES_BLOCK_SIZE);
483 }
484}
485
486static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
487 int alg, const u8 *key,
488 unsigned int keylen, int mode)
489{
490 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
491 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
492 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
493 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
494
495 qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen);
496 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
497
498 if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
499 /* Key reversing not supported, set no convert */
500 dec_cd->aes.cipher_config.val =
501 QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode);
502
503 /* In-place key reversal */
504 qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2,
505 dec_cd->ucs_aes.key);
506 } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) {
507 dec_cd->aes.cipher_config.val =
508 QAT_AES_HW_CONFIG_DEC(alg, mode);
509 } else {
510 dec_cd->aes.cipher_config.val =
511 QAT_AES_HW_CONFIG_ENC(alg, mode);
512 }
513}
514
515static int qat_alg_validate_key(int key_len, int *alg, int mode)
516{
517 if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
518 switch (key_len) {
519 case AES_KEYSIZE_128:
520 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
521 break;
522 case AES_KEYSIZE_192:
523 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
524 break;
525 case AES_KEYSIZE_256:
526 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
527 break;
528 default:
529 return -EINVAL;
530 }
531 } else {
532 switch (key_len) {
533 case AES_KEYSIZE_128 << 1:
534 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
535 break;
536 case AES_KEYSIZE_256 << 1:
537 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
538 break;
539 default:
540 return -EINVAL;
541 }
542 }
543 return 0;
544}
545
546static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
547 unsigned int keylen, int mode)
548{
549 struct crypto_authenc_keys keys;
550 int alg;
551
552 if (crypto_authenc_extractkeys(&keys, key, keylen))
553 goto bad_key;
554
555 if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
556 goto bad_key;
557
558 if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
559 goto error;
560
561 if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
562 goto error;
563
564 memzero_explicit(&keys, sizeof(keys));
565 return 0;
566bad_key:
567 memzero_explicit(&keys, sizeof(keys));
568 return -EINVAL;
569error:
570 memzero_explicit(&keys, sizeof(keys));
571 return -EFAULT;
572}
573
574static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,
575 const u8 *key,
576 unsigned int keylen,
577 int mode)
578{
579 int alg;
580
581 if (qat_alg_validate_key(keylen, &alg, mode))
582 return -EINVAL;
583
584 qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);
585 qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);
586 return 0;
587}
588
589static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key,
590 unsigned int keylen)
591{
592 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
593
594 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
595 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
596 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
597 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
598
599 return qat_alg_aead_init_sessions(tfm, key, keylen,
600 ICP_QAT_HW_CIPHER_CBC_MODE);
601}
602
603static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key,
604 unsigned int keylen)
605{
606 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
607 struct qat_crypto_instance *inst = NULL;
608 int node = numa_node_id();
609 struct device *dev;
610 int ret;
611
612 inst = qat_crypto_get_instance_node(node);
613 if (!inst)
614 return -EINVAL;
615 dev = &GET_DEV(inst->accel_dev);
616 ctx->inst = inst;
617 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
618 &ctx->enc_cd_paddr,
619 GFP_ATOMIC);
620 if (!ctx->enc_cd) {
621 ret = -ENOMEM;
622 goto out_free_inst;
623 }
624 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
625 &ctx->dec_cd_paddr,
626 GFP_ATOMIC);
627 if (!ctx->dec_cd) {
628 ret = -ENOMEM;
629 goto out_free_enc;
630 }
631
632 ret = qat_alg_aead_init_sessions(tfm, key, keylen,
633 ICP_QAT_HW_CIPHER_CBC_MODE);
634 if (ret)
635 goto out_free_all;
636
637 return 0;
638
639out_free_all:
640 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
641 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
642 ctx->dec_cd, ctx->dec_cd_paddr);
643 ctx->dec_cd = NULL;
644out_free_enc:
645 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
646 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
647 ctx->enc_cd, ctx->enc_cd_paddr);
648 ctx->enc_cd = NULL;
649out_free_inst:
650 ctx->inst = NULL;
651 qat_crypto_put_instance(inst);
652 return ret;
653}
654
655static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key,
656 unsigned int keylen)
657{
658 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
659
660 if (ctx->enc_cd)
661 return qat_alg_aead_rekey(tfm, key, keylen);
662 else
663 return qat_alg_aead_newkey(tfm, key, keylen);
664}
665
666static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
667 struct qat_crypto_request *qat_req)
668{
669 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
670 struct qat_crypto_instance *inst = ctx->inst;
671 struct aead_request *areq = qat_req->aead_req;
672 u8 stat_filed = qat_resp->comn_resp.comn_status;
673 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
674
675 qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
676 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
677 res = -EBADMSG;
678 aead_request_complete(areq, res);
679}
680
681static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req)
682{
683 struct skcipher_request *sreq = qat_req->skcipher_req;
684 u64 iv_lo_prev;
685 u64 iv_lo;
686 u64 iv_hi;
687
688 memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE);
689
690 iv_lo = be64_to_cpu(qat_req->iv_lo);
691 iv_hi = be64_to_cpu(qat_req->iv_hi);
692
693 iv_lo_prev = iv_lo;
694 iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE);
695 if (iv_lo < iv_lo_prev)
696 iv_hi++;
697
698 qat_req->iv_lo = cpu_to_be64(iv_lo);
699 qat_req->iv_hi = cpu_to_be64(iv_hi);
700}
701
702static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req)
703{
704 struct skcipher_request *sreq = qat_req->skcipher_req;
705 int offset = sreq->cryptlen - AES_BLOCK_SIZE;
706 struct scatterlist *sgl;
707
708 if (qat_req->encryption)
709 sgl = sreq->dst;
710 else
711 sgl = sreq->src;
712
713 scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0);
714}
715
716static void qat_alg_update_iv(struct qat_crypto_request *qat_req)
717{
718 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
719 struct device *dev = &GET_DEV(ctx->inst->accel_dev);
720
721 switch (ctx->mode) {
722 case ICP_QAT_HW_CIPHER_CTR_MODE:
723 qat_alg_update_iv_ctr_mode(qat_req);
724 break;
725 case ICP_QAT_HW_CIPHER_CBC_MODE:
726 qat_alg_update_iv_cbc_mode(qat_req);
727 break;
728 case ICP_QAT_HW_CIPHER_XTS_MODE:
729 break;
730 default:
731 dev_warn(dev, "Unsupported IV update for cipher mode %d\n",
732 ctx->mode);
733 }
734}
735
736static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
737 struct qat_crypto_request *qat_req)
738{
739 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
740 struct qat_crypto_instance *inst = ctx->inst;
741 struct skcipher_request *sreq = qat_req->skcipher_req;
742 u8 stat_filed = qat_resp->comn_resp.comn_status;
743 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
744
745 qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
746 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
747 res = -EINVAL;
748
749 if (qat_req->encryption)
750 qat_alg_update_iv(qat_req);
751
752 memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
753
754 skcipher_request_complete(sreq, res);
755}
756
757void qat_alg_callback(void *resp)
758{
759 struct icp_qat_fw_la_resp *qat_resp = resp;
760 struct qat_crypto_request *qat_req =
761 (void *)(__force long)qat_resp->opaque_data;
762 struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
763
764 qat_req->cb(qat_resp, qat_req);
765
766 qat_alg_send_backlog(backlog);
767}
768
769static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,
770 struct qat_crypto_instance *inst,
771 struct crypto_async_request *base)
772{
773 struct qat_alg_req *alg_req = &qat_req->alg_req;
774
775 alg_req->fw_req = (u32 *)&qat_req->req;
776 alg_req->tx_ring = inst->sym_tx;
777 alg_req->base = base;
778 alg_req->backlog = &inst->backlog;
779
780 return qat_alg_send_message(alg_req);
781}
782
783static int qat_alg_aead_dec(struct aead_request *areq)
784{
785 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
786 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
787 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
788 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
789 struct icp_qat_fw_la_cipher_req_params *cipher_param;
790 struct icp_qat_fw_la_auth_req_params *auth_param;
791 struct icp_qat_fw_la_bulk_req *msg;
792 int digst_size = crypto_aead_authsize(aead_tfm);
793 gfp_t f = qat_algs_alloc_flags(&areq->base);
794 int ret;
795 u32 cipher_len;
796
797 cipher_len = areq->cryptlen - digst_size;
798 if (cipher_len % AES_BLOCK_SIZE != 0)
799 return -EINVAL;
800
801 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
802 &qat_req->buf, NULL, f);
803 if (unlikely(ret))
804 return ret;
805
806 msg = &qat_req->req;
807 *msg = ctx->dec_fw_req;
808 qat_req->aead_ctx = ctx;
809 qat_req->aead_req = areq;
810 qat_req->cb = qat_aead_alg_callback;
811 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
812 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
813 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
814 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
815 cipher_param->cipher_length = cipher_len;
816 cipher_param->cipher_offset = areq->assoclen;
817 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
818 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
819 auth_param->auth_off = 0;
820 auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
821
822 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
823 if (ret == -ENOSPC)
824 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
825
826 return ret;
827}
828
829static int qat_alg_aead_enc(struct aead_request *areq)
830{
831 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
832 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
833 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
834 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
835 struct icp_qat_fw_la_cipher_req_params *cipher_param;
836 struct icp_qat_fw_la_auth_req_params *auth_param;
837 gfp_t f = qat_algs_alloc_flags(&areq->base);
838 struct icp_qat_fw_la_bulk_req *msg;
839 u8 *iv = areq->iv;
840 int ret;
841
842 if (areq->cryptlen % AES_BLOCK_SIZE != 0)
843 return -EINVAL;
844
845 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
846 &qat_req->buf, NULL, f);
847 if (unlikely(ret))
848 return ret;
849
850 msg = &qat_req->req;
851 *msg = ctx->enc_fw_req;
852 qat_req->aead_ctx = ctx;
853 qat_req->aead_req = areq;
854 qat_req->cb = qat_aead_alg_callback;
855 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
856 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
857 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
858 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
859 auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
860
861 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
862 cipher_param->cipher_length = areq->cryptlen;
863 cipher_param->cipher_offset = areq->assoclen;
864
865 auth_param->auth_off = 0;
866 auth_param->auth_len = areq->assoclen + areq->cryptlen;
867
868 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
869 if (ret == -ENOSPC)
870 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
871
872 return ret;
873}
874
875static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
876 const u8 *key, unsigned int keylen,
877 int mode)
878{
879 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
880 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
881 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
882 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
883
884 return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
885}
886
887static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,
888 const u8 *key, unsigned int keylen,
889 int mode)
890{
891 struct qat_crypto_instance *inst = NULL;
892 struct device *dev;
893 int node = numa_node_id();
894 int ret;
895
896 inst = qat_crypto_get_instance_node(node);
897 if (!inst)
898 return -EINVAL;
899 dev = &GET_DEV(inst->accel_dev);
900 ctx->inst = inst;
901 ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
902 &ctx->enc_cd_paddr,
903 GFP_ATOMIC);
904 if (!ctx->enc_cd) {
905 ret = -ENOMEM;
906 goto out_free_instance;
907 }
908 ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
909 &ctx->dec_cd_paddr,
910 GFP_ATOMIC);
911 if (!ctx->dec_cd) {
912 ret = -ENOMEM;
913 goto out_free_enc;
914 }
915
916 ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
917 if (ret)
918 goto out_free_all;
919
920 return 0;
921
922out_free_all:
923 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
924 dma_free_coherent(dev, sizeof(*ctx->dec_cd),
925 ctx->dec_cd, ctx->dec_cd_paddr);
926 ctx->dec_cd = NULL;
927out_free_enc:
928 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
929 dma_free_coherent(dev, sizeof(*ctx->enc_cd),
930 ctx->enc_cd, ctx->enc_cd_paddr);
931 ctx->enc_cd = NULL;
932out_free_instance:
933 ctx->inst = NULL;
934 qat_crypto_put_instance(inst);
935 return ret;
936}
937
938static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,
939 const u8 *key, unsigned int keylen,
940 int mode)
941{
942 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
943
944 ctx->mode = mode;
945
946 if (ctx->enc_cd)
947 return qat_alg_skcipher_rekey(ctx, key, keylen, mode);
948 else
949 return qat_alg_skcipher_newkey(ctx, key, keylen, mode);
950}
951
952static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,
953 const u8 *key, unsigned int keylen)
954{
955 return qat_alg_skcipher_setkey(tfm, key, keylen,
956 ICP_QAT_HW_CIPHER_CBC_MODE);
957}
958
959static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,
960 const u8 *key, unsigned int keylen)
961{
962 return qat_alg_skcipher_setkey(tfm, key, keylen,
963 ICP_QAT_HW_CIPHER_CTR_MODE);
964}
965
966static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,
967 const u8 *key, unsigned int keylen)
968{
969 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
970 int ret;
971
972 ret = xts_verify_key(tfm, key, keylen);
973 if (ret)
974 return ret;
975
976 if (keylen >> 1 == AES_KEYSIZE_192) {
977 ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen);
978 if (ret)
979 return ret;
980
981 ctx->fallback = true;
982
983 return 0;
984 }
985
986 ctx->fallback = false;
987
988 ret = qat_alg_skcipher_setkey(tfm, key, keylen,
989 ICP_QAT_HW_CIPHER_XTS_MODE);
990 if (ret)
991 return ret;
992
993 if (HW_CAP_AES_V2(ctx->inst->accel_dev))
994 ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2),
995 keylen / 2);
996
997 return ret;
998}
999
1000static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req)
1001{
1002 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1003 struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
1004 bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
1005 u8 *iv = qat_req->skcipher_req->iv;
1006
1007 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1008
1009 if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE)
1010 crypto_cipher_encrypt_one(ctx->tweak,
1011 (u8 *)cipher_param->u.cipher_IV_array,
1012 iv);
1013 else
1014 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
1015}
1016
1017static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
1018{
1019 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1020 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1021 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1022 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1023 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1024 gfp_t f = qat_algs_alloc_flags(&req->base);
1025 struct icp_qat_fw_la_bulk_req *msg;
1026 int ret;
1027
1028 if (req->cryptlen == 0)
1029 return 0;
1030
1031 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,
1032 &qat_req->buf, NULL, f);
1033 if (unlikely(ret))
1034 return ret;
1035
1036 msg = &qat_req->req;
1037 *msg = ctx->enc_fw_req;
1038 qat_req->skcipher_ctx = ctx;
1039 qat_req->skcipher_req = req;
1040 qat_req->cb = qat_skcipher_alg_callback;
1041 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1042 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1043 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1044 qat_req->encryption = true;
1045 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1046 cipher_param->cipher_length = req->cryptlen;
1047 cipher_param->cipher_offset = 0;
1048
1049 qat_alg_set_req_iv(qat_req);
1050
1051 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
1052 if (ret == -ENOSPC)
1053 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
1054
1055 return ret;
1056}
1057
1058static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
1059{
1060 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1061 return -EINVAL;
1062
1063 return qat_alg_skcipher_encrypt(req);
1064}
1065
1066static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req)
1067{
1068 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1069 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1070 struct skcipher_request *nreq = skcipher_request_ctx(req);
1071
1072 if (req->cryptlen < XTS_BLOCK_SIZE)
1073 return -EINVAL;
1074
1075 if (ctx->fallback) {
1076 memcpy(nreq, req, sizeof(*req));
1077 skcipher_request_set_tfm(nreq, ctx->ftfm);
1078 return crypto_skcipher_encrypt(nreq);
1079 }
1080
1081 return qat_alg_skcipher_encrypt(req);
1082}
1083
1084static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
1085{
1086 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1087 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1088 struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1089 struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1090 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1091 gfp_t f = qat_algs_alloc_flags(&req->base);
1092 struct icp_qat_fw_la_bulk_req *msg;
1093 int ret;
1094
1095 if (req->cryptlen == 0)
1096 return 0;
1097
1098 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,
1099 &qat_req->buf, NULL, f);
1100 if (unlikely(ret))
1101 return ret;
1102
1103 msg = &qat_req->req;
1104 *msg = ctx->dec_fw_req;
1105 qat_req->skcipher_ctx = ctx;
1106 qat_req->skcipher_req = req;
1107 qat_req->cb = qat_skcipher_alg_callback;
1108 qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1109 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1110 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1111 qat_req->encryption = false;
1112 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1113 cipher_param->cipher_length = req->cryptlen;
1114 cipher_param->cipher_offset = 0;
1115
1116 qat_alg_set_req_iv(qat_req);
1117 qat_alg_update_iv(qat_req);
1118
1119 ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
1120 if (ret == -ENOSPC)
1121 qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
1122
1123 return ret;
1124}
1125
1126static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
1127{
1128 if (req->cryptlen % AES_BLOCK_SIZE != 0)
1129 return -EINVAL;
1130
1131 return qat_alg_skcipher_decrypt(req);
1132}
1133
1134static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)
1135{
1136 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1137 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1138 struct skcipher_request *nreq = skcipher_request_ctx(req);
1139
1140 if (req->cryptlen < XTS_BLOCK_SIZE)
1141 return -EINVAL;
1142
1143 if (ctx->fallback) {
1144 memcpy(nreq, req, sizeof(*req));
1145 skcipher_request_set_tfm(nreq, ctx->ftfm);
1146 return crypto_skcipher_decrypt(nreq);
1147 }
1148
1149 return qat_alg_skcipher_decrypt(req);
1150}
1151
1152static int qat_alg_aead_init(struct crypto_aead *tfm,
1153 enum icp_qat_hw_auth_algo hash,
1154 const char *hash_name)
1155{
1156 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1157
1158 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1159 if (IS_ERR(ctx->hash_tfm))
1160 return PTR_ERR(ctx->hash_tfm);
1161 ctx->qat_hash_alg = hash;
1162 crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1163 return 0;
1164}
1165
1166static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1167{
1168 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1169}
1170
1171static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1172{
1173 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1174}
1175
1176static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1177{
1178 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1179}
1180
1181static void qat_alg_aead_exit(struct crypto_aead *tfm)
1182{
1183 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1184 struct qat_crypto_instance *inst = ctx->inst;
1185 struct device *dev;
1186
1187 crypto_free_shash(ctx->hash_tfm);
1188
1189 if (!inst)
1190 return;
1191
1192 dev = &GET_DEV(inst->accel_dev);
1193 if (ctx->enc_cd) {
1194 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1195 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1196 ctx->enc_cd, ctx->enc_cd_paddr);
1197 }
1198 if (ctx->dec_cd) {
1199 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1200 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1201 ctx->dec_cd, ctx->dec_cd_paddr);
1202 }
1203 qat_crypto_put_instance(inst);
1204}
1205
1206static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
1207{
1208 crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1209 return 0;
1210}
1211
1212static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
1213{
1214 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1215 int reqsize;
1216
1217 ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0,
1218 CRYPTO_ALG_NEED_FALLBACK);
1219 if (IS_ERR(ctx->ftfm))
1220 return PTR_ERR(ctx->ftfm);
1221
1222 ctx->tweak = crypto_alloc_cipher("aes", 0, 0);
1223 if (IS_ERR(ctx->tweak)) {
1224 crypto_free_skcipher(ctx->ftfm);
1225 return PTR_ERR(ctx->tweak);
1226 }
1227
1228 reqsize = max(sizeof(struct qat_crypto_request),
1229 sizeof(struct skcipher_request) +
1230 crypto_skcipher_reqsize(ctx->ftfm));
1231 crypto_skcipher_set_reqsize(tfm, reqsize);
1232
1233 return 0;
1234}
1235
1236static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
1237{
1238 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1239 struct qat_crypto_instance *inst = ctx->inst;
1240 struct device *dev;
1241
1242 if (!inst)
1243 return;
1244
1245 dev = &GET_DEV(inst->accel_dev);
1246 if (ctx->enc_cd) {
1247 memset(ctx->enc_cd, 0,
1248 sizeof(struct icp_qat_hw_cipher_algo_blk));
1249 dma_free_coherent(dev,
1250 sizeof(struct icp_qat_hw_cipher_algo_blk),
1251 ctx->enc_cd, ctx->enc_cd_paddr);
1252 }
1253 if (ctx->dec_cd) {
1254 memset(ctx->dec_cd, 0,
1255 sizeof(struct icp_qat_hw_cipher_algo_blk));
1256 dma_free_coherent(dev,
1257 sizeof(struct icp_qat_hw_cipher_algo_blk),
1258 ctx->dec_cd, ctx->dec_cd_paddr);
1259 }
1260 qat_crypto_put_instance(inst);
1261}
1262
1263static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm)
1264{
1265 struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1266
1267 if (ctx->ftfm)
1268 crypto_free_skcipher(ctx->ftfm);
1269
1270 if (ctx->tweak)
1271 crypto_free_cipher(ctx->tweak);
1272
1273 qat_alg_skcipher_exit_tfm(tfm);
1274}
1275
1276static struct aead_alg qat_aeads[] = { {
1277 .base = {
1278 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1279 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1280 .cra_priority = 4001,
1281 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1282 .cra_blocksize = AES_BLOCK_SIZE,
1283 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1284 .cra_module = THIS_MODULE,
1285 },
1286 .init = qat_alg_aead_sha1_init,
1287 .exit = qat_alg_aead_exit,
1288 .setkey = qat_alg_aead_setkey,
1289 .decrypt = qat_alg_aead_dec,
1290 .encrypt = qat_alg_aead_enc,
1291 .ivsize = AES_BLOCK_SIZE,
1292 .maxauthsize = SHA1_DIGEST_SIZE,
1293}, {
1294 .base = {
1295 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1296 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1297 .cra_priority = 4001,
1298 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1299 .cra_blocksize = AES_BLOCK_SIZE,
1300 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1301 .cra_module = THIS_MODULE,
1302 },
1303 .init = qat_alg_aead_sha256_init,
1304 .exit = qat_alg_aead_exit,
1305 .setkey = qat_alg_aead_setkey,
1306 .decrypt = qat_alg_aead_dec,
1307 .encrypt = qat_alg_aead_enc,
1308 .ivsize = AES_BLOCK_SIZE,
1309 .maxauthsize = SHA256_DIGEST_SIZE,
1310}, {
1311 .base = {
1312 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1313 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1314 .cra_priority = 4001,
1315 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1316 .cra_blocksize = AES_BLOCK_SIZE,
1317 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1318 .cra_module = THIS_MODULE,
1319 },
1320 .init = qat_alg_aead_sha512_init,
1321 .exit = qat_alg_aead_exit,
1322 .setkey = qat_alg_aead_setkey,
1323 .decrypt = qat_alg_aead_dec,
1324 .encrypt = qat_alg_aead_enc,
1325 .ivsize = AES_BLOCK_SIZE,
1326 .maxauthsize = SHA512_DIGEST_SIZE,
1327} };
1328
1329static struct skcipher_alg qat_skciphers[] = { {
1330 .base.cra_name = "cbc(aes)",
1331 .base.cra_driver_name = "qat_aes_cbc",
1332 .base.cra_priority = 4001,
1333 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1334 .base.cra_blocksize = AES_BLOCK_SIZE,
1335 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1336 .base.cra_alignmask = 0,
1337 .base.cra_module = THIS_MODULE,
1338
1339 .init = qat_alg_skcipher_init_tfm,
1340 .exit = qat_alg_skcipher_exit_tfm,
1341 .setkey = qat_alg_skcipher_cbc_setkey,
1342 .decrypt = qat_alg_skcipher_blk_decrypt,
1343 .encrypt = qat_alg_skcipher_blk_encrypt,
1344 .min_keysize = AES_MIN_KEY_SIZE,
1345 .max_keysize = AES_MAX_KEY_SIZE,
1346 .ivsize = AES_BLOCK_SIZE,
1347}, {
1348 .base.cra_name = "ctr(aes)",
1349 .base.cra_driver_name = "qat_aes_ctr",
1350 .base.cra_priority = 4001,
1351 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1352 .base.cra_blocksize = 1,
1353 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1354 .base.cra_alignmask = 0,
1355 .base.cra_module = THIS_MODULE,
1356
1357 .init = qat_alg_skcipher_init_tfm,
1358 .exit = qat_alg_skcipher_exit_tfm,
1359 .setkey = qat_alg_skcipher_ctr_setkey,
1360 .decrypt = qat_alg_skcipher_decrypt,
1361 .encrypt = qat_alg_skcipher_encrypt,
1362 .min_keysize = AES_MIN_KEY_SIZE,
1363 .max_keysize = AES_MAX_KEY_SIZE,
1364 .ivsize = AES_BLOCK_SIZE,
1365}, {
1366 .base.cra_name = "xts(aes)",
1367 .base.cra_driver_name = "qat_aes_xts",
1368 .base.cra_priority = 4001,
1369 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
1370 CRYPTO_ALG_ALLOCATES_MEMORY,
1371 .base.cra_blocksize = AES_BLOCK_SIZE,
1372 .base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1373 .base.cra_alignmask = 0,
1374 .base.cra_module = THIS_MODULE,
1375
1376 .init = qat_alg_skcipher_init_xts_tfm,
1377 .exit = qat_alg_skcipher_exit_xts_tfm,
1378 .setkey = qat_alg_skcipher_xts_setkey,
1379 .decrypt = qat_alg_skcipher_xts_decrypt,
1380 .encrypt = qat_alg_skcipher_xts_encrypt,
1381 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1382 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1383 .ivsize = AES_BLOCK_SIZE,
1384} };
1385
1386int qat_algs_register(void)
1387{
1388 int ret = 0;
1389
1390 mutex_lock(&algs_lock);
1391 if (++active_devs != 1)
1392 goto unlock;
1393
1394 ret = crypto_register_skciphers(qat_skciphers,
1395 ARRAY_SIZE(qat_skciphers));
1396 if (ret)
1397 goto unlock;
1398
1399 ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1400 if (ret)
1401 goto unreg_algs;
1402
1403unlock:
1404 mutex_unlock(&algs_lock);
1405 return ret;
1406
1407unreg_algs:
1408 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1409 goto unlock;
1410}
1411
1412void qat_algs_unregister(void)
1413{
1414 mutex_lock(&algs_lock);
1415 if (--active_devs != 0)
1416 goto unlock;
1417
1418 crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1419 crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1420
1421unlock:
1422 mutex_unlock(&algs_lock);
1423}