Loading...
Note: File does not exist in v6.8.
1/*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Atul Gupta (atul.gupta@chelsio.com)
36 */
37
38#define pr_fmt(fmt) "chcr:" fmt
39
40#include <linux/kernel.h>
41#include <linux/module.h>
42#include <linux/crypto.h>
43#include <linux/cryptohash.h>
44#include <linux/skbuff.h>
45#include <linux/rtnetlink.h>
46#include <linux/highmem.h>
47#include <linux/if_vlan.h>
48#include <linux/ip.h>
49#include <linux/netdevice.h>
50#include <net/esp.h>
51#include <net/xfrm.h>
52#include <crypto/aes.h>
53#include <crypto/algapi.h>
54#include <crypto/hash.h>
55#include <crypto/sha.h>
56#include <crypto/authenc.h>
57#include <crypto/internal/aead.h>
58#include <crypto/null.h>
59#include <crypto/internal/skcipher.h>
60#include <crypto/aead.h>
61#include <crypto/scatterwalk.h>
62#include <crypto/internal/hash.h>
63
64#include "chcr_core.h"
65#include "chcr_algo.h"
66#include "chcr_crypto.h"
67
68/*
69 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
70 * into a WR.
71 */
72#define MAX_IMM_TX_PKT_LEN 256
73#define GCM_ESP_IV_SIZE 8
74
75static int chcr_xfrm_add_state(struct xfrm_state *x);
76static void chcr_xfrm_del_state(struct xfrm_state *x);
77static void chcr_xfrm_free_state(struct xfrm_state *x);
78static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
79
80static const struct xfrmdev_ops chcr_xfrmdev_ops = {
81 .xdo_dev_state_add = chcr_xfrm_add_state,
82 .xdo_dev_state_delete = chcr_xfrm_del_state,
83 .xdo_dev_state_free = chcr_xfrm_free_state,
84 .xdo_dev_offload_ok = chcr_ipsec_offload_ok,
85};
86
87/* Add offload xfrms to Chelsio Interface */
88void chcr_add_xfrmops(const struct cxgb4_lld_info *lld)
89{
90 struct net_device *netdev = NULL;
91 int i;
92
93 for (i = 0; i < lld->nports; i++) {
94 netdev = lld->ports[i];
95 if (!netdev)
96 continue;
97 netdev->xfrmdev_ops = &chcr_xfrmdev_ops;
98 netdev->hw_enc_features |= NETIF_F_HW_ESP;
99 netdev->features |= NETIF_F_HW_ESP;
100 rtnl_lock();
101 netdev_change_features(netdev);
102 rtnl_unlock();
103 }
104}
105
106static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
107 struct ipsec_sa_entry *sa_entry)
108{
109 int hmac_ctrl;
110 int authsize = x->aead->alg_icv_len / 8;
111
112 sa_entry->authsize = authsize;
113
114 switch (authsize) {
115 case ICV_8:
116 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
117 break;
118 case ICV_12:
119 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
120 break;
121 case ICV_16:
122 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
123 break;
124 default:
125 return -EINVAL;
126 }
127 return hmac_ctrl;
128}
129
130static inline int chcr_ipsec_setkey(struct xfrm_state *x,
131 struct ipsec_sa_entry *sa_entry)
132{
133 struct crypto_cipher *cipher;
134 int keylen = (x->aead->alg_key_len + 7) / 8;
135 unsigned char *key = x->aead->alg_key;
136 int ck_size, key_ctx_size = 0;
137 unsigned char ghash_h[AEAD_H_SIZE];
138 int ret = 0;
139
140 if (keylen > 3) {
141 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
142 memcpy(sa_entry->salt, key + keylen, 4);
143 }
144
145 if (keylen == AES_KEYSIZE_128) {
146 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
147 } else if (keylen == AES_KEYSIZE_192) {
148 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
149 } else if (keylen == AES_KEYSIZE_256) {
150 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
151 } else {
152 pr_err("GCM: Invalid key length %d\n", keylen);
153 ret = -EINVAL;
154 goto out;
155 }
156
157 memcpy(sa_entry->key, key, keylen);
158 sa_entry->enckey_len = keylen;
159 key_ctx_size = sizeof(struct _key_ctx) +
160 ((DIV_ROUND_UP(keylen, 16)) << 4) +
161 AEAD_H_SIZE;
162
163 sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
164 CHCR_KEYCTX_MAC_KEY_SIZE_128,
165 0, 0,
166 key_ctx_size >> 4);
167
168 /* Calculate the H = CIPH(K, 0 repeated 16 times).
169 * It will go in key context
170 */
171 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
172 if (IS_ERR(cipher)) {
173 sa_entry->enckey_len = 0;
174 ret = -ENOMEM;
175 goto out;
176 }
177
178 ret = crypto_cipher_setkey(cipher, key, keylen);
179 if (ret) {
180 sa_entry->enckey_len = 0;
181 goto out1;
182 }
183 memset(ghash_h, 0, AEAD_H_SIZE);
184 crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
185 memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
186 16), ghash_h, AEAD_H_SIZE);
187 sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
188 AEAD_H_SIZE;
189out1:
190 crypto_free_cipher(cipher);
191out:
192 return ret;
193}
194
195/*
196 * chcr_xfrm_add_state
197 * returns 0 on success, negative error if failed to send message to FPGA
198 * positive error if FPGA returned a bad response
199 */
200static int chcr_xfrm_add_state(struct xfrm_state *x)
201{
202 struct ipsec_sa_entry *sa_entry;
203 int res = 0;
204
205 if (x->props.aalgo != SADB_AALG_NONE) {
206 pr_debug("CHCR: Cannot offload authenticated xfrm states\n");
207 return -EINVAL;
208 }
209 if (x->props.calgo != SADB_X_CALG_NONE) {
210 pr_debug("CHCR: Cannot offload compressed xfrm states\n");
211 return -EINVAL;
212 }
213 if (x->props.flags & XFRM_STATE_ESN) {
214 pr_debug("CHCR: Cannot offload ESN xfrm states\n");
215 return -EINVAL;
216 }
217 if (x->props.family != AF_INET &&
218 x->props.family != AF_INET6) {
219 pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n");
220 return -EINVAL;
221 }
222 if (x->props.mode != XFRM_MODE_TRANSPORT &&
223 x->props.mode != XFRM_MODE_TUNNEL) {
224 pr_debug("CHCR: Only transport and tunnel xfrm offload\n");
225 return -EINVAL;
226 }
227 if (x->id.proto != IPPROTO_ESP) {
228 pr_debug("CHCR: Only ESP xfrm state offloaded\n");
229 return -EINVAL;
230 }
231 if (x->encap) {
232 pr_debug("CHCR: Encapsulated xfrm state not offloaded\n");
233 return -EINVAL;
234 }
235 if (!x->aead) {
236 pr_debug("CHCR: Cannot offload xfrm states without aead\n");
237 return -EINVAL;
238 }
239 if (x->aead->alg_icv_len != 128 &&
240 x->aead->alg_icv_len != 96) {
241 pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n");
242 return -EINVAL;
243 }
244 if ((x->aead->alg_key_len != 128 + 32) &&
245 (x->aead->alg_key_len != 256 + 32)) {
246 pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
247 return -EINVAL;
248 }
249 if (x->tfcpad) {
250 pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n");
251 return -EINVAL;
252 }
253 if (!x->geniv) {
254 pr_debug("CHCR: Cannot offload xfrm states without geniv\n");
255 return -EINVAL;
256 }
257 if (strcmp(x->geniv, "seqiv")) {
258 pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n");
259 return -EINVAL;
260 }
261
262 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
263 if (!sa_entry) {
264 res = -ENOMEM;
265 goto out;
266 }
267
268 sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry);
269 chcr_ipsec_setkey(x, sa_entry);
270 x->xso.offload_handle = (unsigned long)sa_entry;
271 try_module_get(THIS_MODULE);
272out:
273 return res;
274}
275
276static void chcr_xfrm_del_state(struct xfrm_state *x)
277{
278 /* do nothing */
279 if (!x->xso.offload_handle)
280 return;
281}
282
283static void chcr_xfrm_free_state(struct xfrm_state *x)
284{
285 struct ipsec_sa_entry *sa_entry;
286
287 if (!x->xso.offload_handle)
288 return;
289
290 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
291 kfree(sa_entry);
292 module_put(THIS_MODULE);
293}
294
295static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
296{
297 /* Offload with IP options is not supported yet */
298 if (ip_hdr(skb)->ihl > 5)
299 return false;
300
301 return true;
302}
303
304static inline int is_eth_imm(const struct sk_buff *skb, unsigned int kctx_len)
305{
306 int hdrlen = sizeof(struct chcr_ipsec_req) + kctx_len;
307
308 hdrlen += sizeof(struct cpl_tx_pkt);
309 if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
310 return hdrlen;
311 return 0;
312}
313
314static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
315 unsigned int kctx_len)
316{
317 unsigned int flits;
318 int hdrlen = is_eth_imm(skb, kctx_len);
319
320 /* If the skb is small enough, we can pump it out as a work request
321 * with only immediate data. In that case we just have to have the
322 * TX Packet header plus the skb data in the Work Request.
323 */
324
325 if (hdrlen)
326 return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
327
328 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
329
330 /* Otherwise, we're going to have to construct a Scatter gather list
331 * of the skb body and fragments. We also include the flits necessary
332 * for the TX Packet Work Request and CPL. We always have a firmware
333 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
334 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
335 * message or, if we're doing a Large Send Offload, an LSO CPL message
336 * with an embedded TX Packet Write CPL message.
337 */
338 flits += (sizeof(struct fw_ulptx_wr) +
339 sizeof(struct chcr_ipsec_req) +
340 kctx_len +
341 sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
342 return flits;
343}
344
345inline void *copy_cpltx_pktxt(struct sk_buff *skb,
346 struct net_device *dev,
347 void *pos)
348{
349 struct adapter *adap;
350 struct port_info *pi;
351 struct sge_eth_txq *q;
352 struct cpl_tx_pkt_core *cpl;
353 u64 cntrl = 0;
354 u32 ctrl0, qidx;
355
356 pi = netdev_priv(dev);
357 adap = pi->adapter;
358 qidx = skb->queue_mapping;
359 q = &adap->sge.ethtxq[qidx + pi->first_qset];
360
361 cpl = (struct cpl_tx_pkt_core *)pos;
362
363 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
364 ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
365 TXPKT_PF_V(adap->pf);
366 if (skb_vlan_tag_present(skb)) {
367 q->vlan_ins++;
368 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
369 }
370
371 cpl->ctrl0 = htonl(ctrl0);
372 cpl->pack = htons(0);
373 cpl->len = htons(skb->len);
374 cpl->ctrl1 = cpu_to_be64(cntrl);
375
376 pos += sizeof(struct cpl_tx_pkt_core);
377 return pos;
378}
379
380inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
381 struct net_device *dev,
382 void *pos,
383 struct ipsec_sa_entry *sa_entry)
384{
385 struct adapter *adap;
386 struct port_info *pi;
387 struct sge_eth_txq *q;
388 unsigned int len, qidx;
389 struct _key_ctx *key_ctx;
390 int left, eoq, key_len;
391
392 pi = netdev_priv(dev);
393 adap = pi->adapter;
394 qidx = skb->queue_mapping;
395 q = &adap->sge.ethtxq[qidx + pi->first_qset];
396 len = sa_entry->enckey_len + sizeof(struct cpl_tx_pkt_core);
397 key_len = sa_entry->kctx_len;
398
399 /* end of queue, reset pos to start of queue */
400 eoq = (void *)q->q.stat - pos;
401 left = eoq;
402 if (!eoq) {
403 pos = q->q.desc;
404 left = 64 * q->q.size;
405 }
406
407 /* Copy the Key context header */
408 key_ctx = (struct _key_ctx *)pos;
409 key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
410 memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
411 pos += sizeof(struct _key_ctx);
412 left -= sizeof(struct _key_ctx);
413
414 if (likely(len <= left)) {
415 memcpy(key_ctx->key, sa_entry->key, key_len);
416 pos += key_len;
417 } else {
418 if (key_len <= left) {
419 memcpy(pos, sa_entry->key, key_len);
420 pos += key_len;
421 } else {
422 memcpy(pos, sa_entry->key, left);
423 memcpy(q->q.desc, sa_entry->key + left,
424 key_len - left);
425 pos = (u8 *)q->q.desc + (key_len - left);
426 }
427 }
428 /* Copy CPL TX PKT XT */
429 pos = copy_cpltx_pktxt(skb, dev, pos);
430
431 return pos;
432}
433
434inline void *chcr_crypto_wreq(struct sk_buff *skb,
435 struct net_device *dev,
436 void *pos,
437 int credits,
438 struct ipsec_sa_entry *sa_entry)
439{
440 struct port_info *pi = netdev_priv(dev);
441 struct adapter *adap = pi->adapter;
442 unsigned int immdatalen = 0;
443 unsigned int ivsize = GCM_ESP_IV_SIZE;
444 struct chcr_ipsec_wr *wr;
445 unsigned int flits;
446 u32 wr_mid;
447 int qidx = skb_get_queue_mapping(skb);
448 struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
449 unsigned int kctx_len = sa_entry->kctx_len;
450 int qid = q->q.cntxt_id;
451
452 atomic_inc(&adap->chcr_stats.ipsec_cnt);
453
454 flits = calc_tx_sec_flits(skb, kctx_len);
455
456 if (is_eth_imm(skb, kctx_len))
457 immdatalen = skb->len;
458
459 /* WR Header */
460 wr = (struct chcr_ipsec_wr *)pos;
461 wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
462 wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
463
464 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
465 netif_tx_stop_queue(q->txq);
466 q->q.stops++;
467 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
468 }
469 wr_mid |= FW_ULPTX_WR_DATA_F;
470 wr->wreq.flowid_len16 = htonl(wr_mid);
471
472 /* ULPTX */
473 wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
474 wr->req.ulptx.len = htonl(DIV_ROUND_UP(flits, 2) - 1);
475
476 /* Sub-command */
477 wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen);
478 wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
479 sizeof(wr->req.key_ctx) +
480 kctx_len +
481 sizeof(struct cpl_tx_pkt_core) +
482 immdatalen);
483
484 /* CPL_SEC_PDU */
485 wr->req.sec_cpl.op_ivinsrtofst = htonl(
486 CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
487 CPL_TX_SEC_PDU_CPLLEN_V(2) |
488 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
489 CPL_TX_SEC_PDU_IVINSRTOFST_V(
490 (skb_transport_offset(skb) +
491 sizeof(struct ip_esp_hdr) + 1)));
492
493 wr->req.sec_cpl.pldlen = htonl(skb->len);
494
495 wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
496 (skb_transport_offset(skb) + 1),
497 (skb_transport_offset(skb) +
498 sizeof(struct ip_esp_hdr)),
499 (skb_transport_offset(skb) +
500 sizeof(struct ip_esp_hdr) +
501 GCM_ESP_IV_SIZE + 1), 0);
502
503 wr->req.sec_cpl.cipherstop_lo_authinsert =
504 FILL_SEC_CPL_AUTHINSERT(0, skb_transport_offset(skb) +
505 sizeof(struct ip_esp_hdr) +
506 GCM_ESP_IV_SIZE + 1,
507 sa_entry->authsize,
508 sa_entry->authsize);
509 wr->req.sec_cpl.seqno_numivs =
510 FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
511 CHCR_SCMD_CIPHER_MODE_AES_GCM,
512 CHCR_SCMD_AUTH_MODE_GHASH,
513 sa_entry->hmac_ctrl,
514 ivsize >> 1);
515 wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
516 0, 0, 0);
517
518 pos += sizeof(struct fw_ulptx_wr) +
519 sizeof(struct ulp_txpkt) +
520 sizeof(struct ulptx_idata) +
521 sizeof(struct cpl_tx_sec_pdu);
522
523 pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
524
525 return pos;
526}
527
528/**
529 * flits_to_desc - returns the num of Tx descriptors for the given flits
530 * @n: the number of flits
531 *
532 * Returns the number of Tx descriptors needed for the supplied number
533 * of flits.
534 */
535static inline unsigned int flits_to_desc(unsigned int n)
536{
537 WARN_ON(n > SGE_MAX_WR_LEN / 8);
538 return DIV_ROUND_UP(n, 8);
539}
540
541static inline unsigned int txq_avail(const struct sge_txq *q)
542{
543 return q->size - 1 - q->in_use;
544}
545
546static void eth_txq_stop(struct sge_eth_txq *q)
547{
548 netif_tx_stop_queue(q->txq);
549 q->q.stops++;
550}
551
552static inline void txq_advance(struct sge_txq *q, unsigned int n)
553{
554 q->in_use += n;
555 q->pidx += n;
556 if (q->pidx >= q->size)
557 q->pidx -= q->size;
558}
559
560/*
561 * chcr_ipsec_xmit called from ULD Tx handler
562 */
563int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
564{
565 struct xfrm_state *x = xfrm_input_state(skb);
566 struct ipsec_sa_entry *sa_entry;
567 u64 *pos, *end, *before, *sgl;
568 int qidx, left, credits;
569 unsigned int flits = 0, ndesc, kctx_len;
570 struct adapter *adap;
571 struct sge_eth_txq *q;
572 struct port_info *pi;
573 dma_addr_t addr[MAX_SKB_FRAGS + 1];
574 bool immediate = false;
575
576 if (!x->xso.offload_handle)
577 return NETDEV_TX_BUSY;
578
579 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
580 kctx_len = sa_entry->kctx_len;
581
582 if (skb->sp->len != 1) {
583out_free: dev_kfree_skb_any(skb);
584 return NETDEV_TX_OK;
585 }
586
587 pi = netdev_priv(dev);
588 adap = pi->adapter;
589 qidx = skb->queue_mapping;
590 q = &adap->sge.ethtxq[qidx + pi->first_qset];
591
592 cxgb4_reclaim_completed_tx(adap, &q->q, true);
593
594 flits = calc_tx_sec_flits(skb, sa_entry->kctx_len);
595 ndesc = flits_to_desc(flits);
596 credits = txq_avail(&q->q) - ndesc;
597
598 if (unlikely(credits < 0)) {
599 eth_txq_stop(q);
600 dev_err(adap->pdev_dev,
601 "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
602 dev->name, qidx, credits, ndesc, txq_avail(&q->q),
603 flits);
604 return NETDEV_TX_BUSY;
605 }
606
607 if (is_eth_imm(skb, kctx_len))
608 immediate = true;
609
610 if (!immediate &&
611 unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
612 q->mapping_err++;
613 goto out_free;
614 }
615
616 pos = (u64 *)&q->q.desc[q->q.pidx];
617 before = (u64 *)pos;
618 end = (u64 *)pos + flits;
619 /* Setup IPSec CPL */
620 pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos,
621 credits, sa_entry);
622 if (before > (u64 *)pos) {
623 left = (u8 *)end - (u8 *)q->q.stat;
624 end = (void *)q->q.desc + left;
625 }
626 if (pos == (u64 *)q->q.stat) {
627 left = (u8 *)end - (u8 *)q->q.stat;
628 end = (void *)q->q.desc + left;
629 pos = (void *)q->q.desc;
630 }
631
632 sgl = (void *)pos;
633 if (immediate) {
634 cxgb4_inline_tx_skb(skb, &q->q, sgl);
635 dev_consume_skb_any(skb);
636 } else {
637 int last_desc;
638
639 cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
640 0, addr);
641 skb_orphan(skb);
642
643 last_desc = q->q.pidx + ndesc - 1;
644 if (last_desc >= q->q.size)
645 last_desc -= q->q.size;
646 q->q.sdesc[last_desc].skb = skb;
647 q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
648 }
649 txq_advance(&q->q, ndesc);
650
651 cxgb4_ring_tx_db(adap, &q->q, ndesc);
652 return NETDEV_TX_OK;
653}