Loading...
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <linux/gfp.h>
7#include <linux/mlx5/qp.h>
8#include <linux/mlx5/driver.h>
9#include "wr.h"
10#include "umr.h"
11
12static const u32 mlx5_ib_opcode[] = {
13 [IB_WR_SEND] = MLX5_OPCODE_SEND,
14 [IB_WR_LSO] = MLX5_OPCODE_LSO,
15 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
16 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
17 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
18 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
19 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
20 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
21 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
22 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
23 [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
24 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
25 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
26 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
27};
28
29int mlx5r_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
30{
31 struct mlx5_ib_cq *cq;
32 unsigned int cur;
33
34 cur = wq->head - wq->tail;
35 if (likely(cur + nreq < wq->max_post))
36 return 0;
37
38 cq = to_mcq(ib_cq);
39 spin_lock(&cq->lock);
40 cur = wq->head - wq->tail;
41 spin_unlock(&cq->lock);
42
43 return cur + nreq >= wq->max_post;
44}
45
46static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
47 u64 remote_addr, u32 rkey)
48{
49 rseg->raddr = cpu_to_be64(remote_addr);
50 rseg->rkey = cpu_to_be32(rkey);
51 rseg->reserved = 0;
52}
53
54static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
55 void **seg, int *size, void **cur_edge)
56{
57 struct mlx5_wqe_eth_seg *eseg = *seg;
58
59 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
60
61 if (wr->send_flags & IB_SEND_IP_CSUM)
62 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
63 MLX5_ETH_WQE_L4_CSUM;
64
65 if (wr->opcode == IB_WR_LSO) {
66 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
67 size_t left, copysz;
68 void *pdata = ud_wr->header;
69 size_t stride;
70
71 left = ud_wr->hlen;
72 eseg->mss = cpu_to_be16(ud_wr->mss);
73 eseg->inline_hdr.sz = cpu_to_be16(left);
74
75 /* mlx5r_memcpy_send_wqe should get a 16B align address. Hence,
76 * we first copy up to the current edge and then, if needed,
77 * continue to mlx5r_memcpy_send_wqe.
78 */
79 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
80 left);
81 memcpy(eseg->inline_hdr.data, pdata, copysz);
82 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
83 sizeof(eseg->inline_hdr.start) + copysz, 16);
84 *size += stride / 16;
85 *seg += stride;
86
87 if (copysz < left) {
88 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
89 left -= copysz;
90 pdata += copysz;
91 mlx5r_memcpy_send_wqe(&qp->sq, cur_edge, seg, size,
92 pdata, left);
93 }
94
95 return;
96 }
97
98 *seg += sizeof(struct mlx5_wqe_eth_seg);
99 *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
100}
101
102static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
103 const struct ib_send_wr *wr)
104{
105 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
106 dseg->av.dqp_dct =
107 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
108 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
109}
110
111static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
112{
113 dseg->byte_count = cpu_to_be32(sg->length);
114 dseg->lkey = cpu_to_be32(sg->lkey);
115 dseg->addr = cpu_to_be64(sg->addr);
116}
117
118static __be64 frwr_mkey_mask(bool atomic)
119{
120 u64 result;
121
122 result = MLX5_MKEY_MASK_LEN |
123 MLX5_MKEY_MASK_PAGE_SIZE |
124 MLX5_MKEY_MASK_START_ADDR |
125 MLX5_MKEY_MASK_EN_RINVAL |
126 MLX5_MKEY_MASK_KEY |
127 MLX5_MKEY_MASK_LR |
128 MLX5_MKEY_MASK_LW |
129 MLX5_MKEY_MASK_RR |
130 MLX5_MKEY_MASK_RW |
131 MLX5_MKEY_MASK_SMALL_FENCE |
132 MLX5_MKEY_MASK_FREE;
133
134 if (atomic)
135 result |= MLX5_MKEY_MASK_A;
136
137 return cpu_to_be64(result);
138}
139
140static __be64 sig_mkey_mask(void)
141{
142 u64 result;
143
144 result = MLX5_MKEY_MASK_LEN |
145 MLX5_MKEY_MASK_PAGE_SIZE |
146 MLX5_MKEY_MASK_START_ADDR |
147 MLX5_MKEY_MASK_EN_SIGERR |
148 MLX5_MKEY_MASK_EN_RINVAL |
149 MLX5_MKEY_MASK_KEY |
150 MLX5_MKEY_MASK_LR |
151 MLX5_MKEY_MASK_LW |
152 MLX5_MKEY_MASK_RR |
153 MLX5_MKEY_MASK_RW |
154 MLX5_MKEY_MASK_SMALL_FENCE |
155 MLX5_MKEY_MASK_FREE |
156 MLX5_MKEY_MASK_BSF_EN;
157
158 return cpu_to_be64(result);
159}
160
161static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
162 struct mlx5_ib_mr *mr, u8 flags, bool atomic)
163{
164 int size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
165
166 memset(umr, 0, sizeof(*umr));
167
168 umr->flags = flags;
169 umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));
170 umr->mkey_mask = frwr_mkey_mask(atomic);
171}
172
173static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
174{
175 memset(umr, 0, sizeof(*umr));
176 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
177 umr->flags = MLX5_UMR_INLINE;
178}
179
180static u8 get_umr_flags(int acc)
181{
182 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
183 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
184 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
185 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
186 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
187}
188
189static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
190 struct mlx5_ib_mr *mr,
191 u32 key, int access)
192{
193 int ndescs = ALIGN(mr->mmkey.ndescs + mr->meta_ndescs, 8) >> 1;
194
195 memset(seg, 0, sizeof(*seg));
196
197 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
198 seg->log2_page_size = ilog2(mr->ibmr.page_size);
199 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
200 /* KLMs take twice the size of MTTs */
201 ndescs *= 2;
202
203 seg->flags = get_umr_flags(access) | mr->access_mode;
204 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
205 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
206 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
207 seg->len = cpu_to_be64(mr->ibmr.length);
208 seg->xlt_oct_size = cpu_to_be32(ndescs);
209}
210
211static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
212{
213 memset(seg, 0, sizeof(*seg));
214 seg->status = MLX5_MKEY_STATUS_FREE;
215}
216
217static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
218 struct mlx5_ib_mr *mr,
219 struct mlx5_ib_pd *pd)
220{
221 int bcount = mr->desc_size * (mr->mmkey.ndescs + mr->meta_ndescs);
222
223 dseg->addr = cpu_to_be64(mr->desc_map);
224 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
225 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
226}
227
228static __be32 send_ieth(const struct ib_send_wr *wr)
229{
230 switch (wr->opcode) {
231 case IB_WR_SEND_WITH_IMM:
232 case IB_WR_RDMA_WRITE_WITH_IMM:
233 return wr->ex.imm_data;
234
235 case IB_WR_SEND_WITH_INV:
236 return cpu_to_be32(wr->ex.invalidate_rkey);
237
238 default:
239 return 0;
240 }
241}
242
243static u8 calc_sig(void *wqe, int size)
244{
245 u8 *p = wqe;
246 u8 res = 0;
247 int i;
248
249 for (i = 0; i < size; i++)
250 res ^= p[i];
251
252 return ~res;
253}
254
255static u8 wq_sig(void *wqe)
256{
257 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
258}
259
260static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
261 void **wqe, int *wqe_sz, void **cur_edge)
262{
263 struct mlx5_wqe_inline_seg *seg;
264 size_t offset;
265 int inl = 0;
266 int i;
267
268 seg = *wqe;
269 *wqe += sizeof(*seg);
270 offset = sizeof(*seg);
271
272 for (i = 0; i < wr->num_sge; i++) {
273 size_t len = wr->sg_list[i].length;
274 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
275
276 inl += len;
277
278 if (unlikely(inl > qp->max_inline_data))
279 return -ENOMEM;
280
281 while (likely(len)) {
282 size_t leftlen;
283 size_t copysz;
284
285 handle_post_send_edge(&qp->sq, wqe,
286 *wqe_sz + (offset >> 4),
287 cur_edge);
288
289 leftlen = *cur_edge - *wqe;
290 copysz = min_t(size_t, leftlen, len);
291
292 memcpy(*wqe, addr, copysz);
293 len -= copysz;
294 addr += copysz;
295 *wqe += copysz;
296 offset += copysz;
297 }
298 }
299
300 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
301
302 *wqe_sz += ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
303
304 return 0;
305}
306
307static u16 prot_field_size(enum ib_signature_type type)
308{
309 switch (type) {
310 case IB_SIG_TYPE_T10_DIF:
311 return MLX5_DIF_SIZE;
312 default:
313 return 0;
314 }
315}
316
317static u8 bs_selector(int block_size)
318{
319 switch (block_size) {
320 case 512: return 0x1;
321 case 520: return 0x2;
322 case 4096: return 0x3;
323 case 4160: return 0x4;
324 case 1073741824: return 0x5;
325 default: return 0;
326 }
327}
328
329static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
330 struct mlx5_bsf_inl *inl)
331{
332 /* Valid inline section and allow BSF refresh */
333 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
334 MLX5_BSF_REFRESH_DIF);
335 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
336 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
337 /* repeating block */
338 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
339 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
340 MLX5_DIF_CRC : MLX5_DIF_IPCS;
341
342 if (domain->sig.dif.ref_remap)
343 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
344
345 if (domain->sig.dif.app_escape) {
346 if (domain->sig.dif.ref_escape)
347 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
348 else
349 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
350 }
351
352 inl->dif_app_bitmask_check =
353 cpu_to_be16(domain->sig.dif.apptag_check_mask);
354}
355
356static int mlx5_set_bsf(struct ib_mr *sig_mr,
357 struct ib_sig_attrs *sig_attrs,
358 struct mlx5_bsf *bsf, u32 data_size)
359{
360 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
361 struct mlx5_bsf_basic *basic = &bsf->basic;
362 struct ib_sig_domain *mem = &sig_attrs->mem;
363 struct ib_sig_domain *wire = &sig_attrs->wire;
364
365 memset(bsf, 0, sizeof(*bsf));
366
367 /* Basic + Extended + Inline */
368 basic->bsf_size_sbs = 1 << 7;
369 /* Input domain check byte mask */
370 basic->check_byte_mask = sig_attrs->check_mask;
371 basic->raw_data_size = cpu_to_be32(data_size);
372
373 /* Memory domain */
374 switch (sig_attrs->mem.sig_type) {
375 case IB_SIG_TYPE_NONE:
376 break;
377 case IB_SIG_TYPE_T10_DIF:
378 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
379 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
380 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
381 break;
382 default:
383 return -EINVAL;
384 }
385
386 /* Wire domain */
387 switch (sig_attrs->wire.sig_type) {
388 case IB_SIG_TYPE_NONE:
389 break;
390 case IB_SIG_TYPE_T10_DIF:
391 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
392 mem->sig_type == wire->sig_type) {
393 /* Same block structure */
394 basic->bsf_size_sbs |= 1 << 4;
395 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
396 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
397 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
398 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
399 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
400 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
401 } else
402 basic->wire.bs_selector =
403 bs_selector(wire->sig.dif.pi_interval);
404
405 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
406 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
407 break;
408 default:
409 return -EINVAL;
410 }
411
412 return 0;
413}
414
415
416static int set_sig_data_segment(const struct ib_send_wr *send_wr,
417 struct ib_mr *sig_mr,
418 struct ib_sig_attrs *sig_attrs,
419 struct mlx5_ib_qp *qp, void **seg, int *size,
420 void **cur_edge)
421{
422 struct mlx5_bsf *bsf;
423 u32 data_len;
424 u32 data_key;
425 u64 data_va;
426 u32 prot_len = 0;
427 u32 prot_key = 0;
428 u64 prot_va = 0;
429 bool prot = false;
430 int ret;
431 int wqe_size;
432 struct mlx5_ib_mr *mr = to_mmr(sig_mr);
433 struct mlx5_ib_mr *pi_mr = mr->pi_mr;
434
435 data_len = pi_mr->data_length;
436 data_key = pi_mr->ibmr.lkey;
437 data_va = pi_mr->data_iova;
438 if (pi_mr->meta_ndescs) {
439 prot_len = pi_mr->meta_length;
440 prot_key = pi_mr->ibmr.lkey;
441 prot_va = pi_mr->pi_iova;
442 prot = true;
443 }
444
445 if (!prot || (data_key == prot_key && data_va == prot_va &&
446 data_len == prot_len)) {
447 /**
448 * Source domain doesn't contain signature information
449 * or data and protection are interleaved in memory.
450 * So need construct:
451 * ------------------
452 * | data_klm |
453 * ------------------
454 * | BSF |
455 * ------------------
456 **/
457 struct mlx5_klm *data_klm = *seg;
458
459 data_klm->bcount = cpu_to_be32(data_len);
460 data_klm->key = cpu_to_be32(data_key);
461 data_klm->va = cpu_to_be64(data_va);
462 wqe_size = ALIGN(sizeof(*data_klm), 64);
463 } else {
464 /**
465 * Source domain contains signature information
466 * So need construct a strided block format:
467 * ---------------------------
468 * | stride_block_ctrl |
469 * ---------------------------
470 * | data_klm |
471 * ---------------------------
472 * | prot_klm |
473 * ---------------------------
474 * | BSF |
475 * ---------------------------
476 **/
477 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
478 struct mlx5_stride_block_entry *data_sentry;
479 struct mlx5_stride_block_entry *prot_sentry;
480 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
481 int prot_size;
482
483 sblock_ctrl = *seg;
484 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
485 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
486
487 prot_size = prot_field_size(sig_attrs->mem.sig_type);
488 if (!prot_size) {
489 pr_err("Bad block size given: %u\n", block_size);
490 return -EINVAL;
491 }
492 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
493 prot_size);
494 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
495 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
496 sblock_ctrl->num_entries = cpu_to_be16(2);
497
498 data_sentry->bcount = cpu_to_be16(block_size);
499 data_sentry->key = cpu_to_be32(data_key);
500 data_sentry->va = cpu_to_be64(data_va);
501 data_sentry->stride = cpu_to_be16(block_size);
502
503 prot_sentry->bcount = cpu_to_be16(prot_size);
504 prot_sentry->key = cpu_to_be32(prot_key);
505 prot_sentry->va = cpu_to_be64(prot_va);
506 prot_sentry->stride = cpu_to_be16(prot_size);
507
508 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
509 sizeof(*prot_sentry), 64);
510 }
511
512 *seg += wqe_size;
513 *size += wqe_size / 16;
514 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
515
516 bsf = *seg;
517 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
518 if (ret)
519 return -EINVAL;
520
521 *seg += sizeof(*bsf);
522 *size += sizeof(*bsf) / 16;
523 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
524
525 return 0;
526}
527
528static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
529 struct ib_mr *sig_mr, int access_flags,
530 u32 size, u32 length, u32 pdn)
531{
532 u32 sig_key = sig_mr->rkey;
533 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
534
535 memset(seg, 0, sizeof(*seg));
536
537 seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
538 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
539 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
540 MLX5_MKEY_BSF_EN | pdn);
541 seg->len = cpu_to_be64(length);
542 seg->xlt_oct_size = cpu_to_be32(mlx5r_umr_get_xlt_octo(size));
543 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
544}
545
546static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
547 u32 size)
548{
549 memset(umr, 0, sizeof(*umr));
550
551 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
552 umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));
553 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
554 umr->mkey_mask = sig_mkey_mask();
555}
556
557static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
558 struct mlx5_ib_qp *qp, void **seg, int *size,
559 void **cur_edge)
560{
561 const struct ib_reg_wr *wr = reg_wr(send_wr);
562 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
563 struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
564 struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
565 u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
566 u32 xlt_size;
567 int region_len, ret;
568
569 if (unlikely(send_wr->num_sge != 0) ||
570 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
571 unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
572 unlikely(!sig_mr->sig->sig_status_checked))
573 return -EINVAL;
574
575 /* length of the protected region, data + protection */
576 region_len = pi_mr->ibmr.length;
577
578 /**
579 * KLM octoword size - if protection was provided
580 * then we use strided block format (3 octowords),
581 * else we use single KLM (1 octoword)
582 **/
583 if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
584 xlt_size = 0x30;
585 else
586 xlt_size = sizeof(struct mlx5_klm);
587
588 set_sig_umr_segment(*seg, xlt_size);
589 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
590 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
591 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
592
593 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
594 pdn);
595 *seg += sizeof(struct mlx5_mkey_seg);
596 *size += sizeof(struct mlx5_mkey_seg) / 16;
597 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
598
599 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
600 cur_edge);
601 if (ret)
602 return ret;
603
604 sig_mr->sig->sig_status_checked = false;
605 return 0;
606}
607
608static int set_psv_wr(struct ib_sig_domain *domain,
609 u32 psv_idx, void **seg, int *size)
610{
611 struct mlx5_seg_set_psv *psv_seg = *seg;
612
613 memset(psv_seg, 0, sizeof(*psv_seg));
614 psv_seg->psv_num = cpu_to_be32(psv_idx);
615 switch (domain->sig_type) {
616 case IB_SIG_TYPE_NONE:
617 break;
618 case IB_SIG_TYPE_T10_DIF:
619 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
620 domain->sig.dif.app_tag);
621 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
622 break;
623 default:
624 pr_err("Bad signature type (%d) is given.\n",
625 domain->sig_type);
626 return -EINVAL;
627 }
628
629 *seg += sizeof(*psv_seg);
630 *size += sizeof(*psv_seg) / 16;
631
632 return 0;
633}
634
635static int set_reg_wr(struct mlx5_ib_qp *qp,
636 const struct ib_reg_wr *wr,
637 void **seg, int *size, void **cur_edge,
638 bool check_not_free)
639{
640 struct mlx5_ib_mr *mr = to_mmr(wr->mr);
641 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
642 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
643 int mr_list_size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
644 bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
645 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
646 u8 flags = 0;
647
648 /* Matches access in mlx5_set_umr_free_mkey().
649 * Relaxed Ordering is set implicitly in mlx5_set_umr_free_mkey() and
650 * kernel ULPs are not aware of it, so we don't set it here.
651 */
652 if (!mlx5r_umr_can_reconfig(dev, 0, wr->access)) {
653 mlx5_ib_warn(
654 to_mdev(qp->ibqp.device),
655 "Fast update for MR access flags is not possible\n");
656 return -EINVAL;
657 }
658
659 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
660 mlx5_ib_warn(to_mdev(qp->ibqp.device),
661 "Invalid IB_SEND_INLINE send flag\n");
662 return -EINVAL;
663 }
664
665 if (check_not_free)
666 flags |= MLX5_UMR_CHECK_NOT_FREE;
667 if (umr_inline)
668 flags |= MLX5_UMR_INLINE;
669
670 set_reg_umr_seg(*seg, mr, flags, atomic);
671 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
672 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
673 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
674
675 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
676 *seg += sizeof(struct mlx5_mkey_seg);
677 *size += sizeof(struct mlx5_mkey_seg) / 16;
678 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
679
680 if (umr_inline) {
681 mlx5r_memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
682 mr_list_size);
683 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
684 } else {
685 set_reg_data_seg(*seg, mr, pd);
686 *seg += sizeof(struct mlx5_wqe_data_seg);
687 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
688 }
689 return 0;
690}
691
692static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
693 void **cur_edge)
694{
695 set_linv_umr_seg(*seg);
696 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
697 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
698 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
699 set_linv_mkey_seg(*seg);
700 *seg += sizeof(struct mlx5_mkey_seg);
701 *size += sizeof(struct mlx5_mkey_seg) / 16;
702 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
703}
704
705static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
706{
707 __be32 *p = NULL;
708 int i, j;
709
710 pr_debug("dump WQE index %u:\n", idx);
711 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
712 if ((i & 0xf) == 0) {
713 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
714 pr_debug("WQBB at %p:\n", (void *)p);
715 j = 0;
716 idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
717 }
718 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
719 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
720 be32_to_cpu(p[j + 3]));
721 }
722}
723
724int mlx5r_begin_wqe(struct mlx5_ib_qp *qp, void **seg,
725 struct mlx5_wqe_ctrl_seg **ctrl, unsigned int *idx,
726 int *size, void **cur_edge, int nreq, __be32 general_id,
727 bool send_signaled, bool solicited)
728{
729 if (unlikely(mlx5r_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
730 return -ENOMEM;
731
732 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
733 *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
734 *ctrl = *seg;
735 *(uint32_t *)(*seg + 8) = 0;
736 (*ctrl)->general_id = general_id;
737 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
738 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
739 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
740
741 *seg += sizeof(**ctrl);
742 *size = sizeof(**ctrl) / 16;
743 *cur_edge = qp->sq.cur_edge;
744
745 return 0;
746}
747
748static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
749 struct mlx5_wqe_ctrl_seg **ctrl,
750 const struct ib_send_wr *wr, unsigned int *idx, int *size,
751 void **cur_edge, int nreq)
752{
753 return mlx5r_begin_wqe(qp, seg, ctrl, idx, size, cur_edge, nreq,
754 send_ieth(wr), wr->send_flags & IB_SEND_SIGNALED,
755 wr->send_flags & IB_SEND_SOLICITED);
756}
757
758void mlx5r_finish_wqe(struct mlx5_ib_qp *qp, struct mlx5_wqe_ctrl_seg *ctrl,
759 void *seg, u8 size, void *cur_edge, unsigned int idx,
760 u64 wr_id, int nreq, u8 fence, u32 mlx5_opcode)
761{
762 u8 opmod = 0;
763
764 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
765 mlx5_opcode | ((u32)opmod << 24));
766 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
767 ctrl->fm_ce_se |= fence;
768 if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
769 ctrl->signature = wq_sig(ctrl);
770
771 qp->sq.wrid[idx] = wr_id;
772 qp->sq.w_list[idx].opcode = mlx5_opcode;
773 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
774 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
775 qp->sq.w_list[idx].next = qp->sq.cur_post;
776
777 /* We save the edge which was possibly updated during the WQE
778 * construction, into SQ's cache.
779 */
780 seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
781 qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
782 get_sq_edge(&qp->sq, qp->sq.cur_post &
783 (qp->sq.wqe_cnt - 1)) :
784 cur_edge;
785}
786
787static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
788{
789 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
790 *seg += sizeof(struct mlx5_wqe_raddr_seg);
791 *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
792}
793
794static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
795 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
796 int *size, void **cur_edge, unsigned int idx)
797{
798 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
799 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
800 set_linv_wr(qp, seg, size, cur_edge);
801}
802
803static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
804 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
805 void **cur_edge, unsigned int idx)
806{
807 qp->sq.wr_data[idx] = IB_WR_REG_MR;
808 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
809 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
810}
811
812static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
813 const struct ib_send_wr *wr,
814 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
815 void **cur_edge, unsigned int *idx, int nreq,
816 struct ib_sig_domain *domain, u32 psv_index,
817 u8 next_fence)
818{
819 int err;
820
821 /*
822 * SET_PSV WQEs are not signaled and solicited on error.
823 */
824 err = mlx5r_begin_wqe(qp, seg, ctrl, idx, size, cur_edge, nreq,
825 send_ieth(wr), false, true);
826 if (unlikely(err)) {
827 mlx5_ib_warn(dev, "\n");
828 err = -ENOMEM;
829 goto out;
830 }
831 err = set_psv_wr(domain, psv_index, seg, size);
832 if (unlikely(err)) {
833 mlx5_ib_warn(dev, "\n");
834 goto out;
835 }
836 mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
837 nreq, next_fence, MLX5_OPCODE_SET_PSV);
838
839out:
840 return err;
841}
842
843static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
844 struct mlx5_ib_qp *qp,
845 const struct ib_send_wr *wr,
846 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
847 int *size, void **cur_edge,
848 unsigned int *idx, int nreq, u8 fence,
849 u8 next_fence)
850{
851 struct mlx5_ib_mr *mr;
852 struct mlx5_ib_mr *pi_mr;
853 struct mlx5_ib_mr pa_pi_mr;
854 struct ib_sig_attrs *sig_attrs;
855 struct ib_reg_wr reg_pi_wr;
856 int err;
857
858 qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
859
860 mr = to_mmr(reg_wr(wr)->mr);
861 pi_mr = mr->pi_mr;
862
863 if (pi_mr) {
864 memset(®_pi_wr, 0,
865 sizeof(struct ib_reg_wr));
866
867 reg_pi_wr.mr = &pi_mr->ibmr;
868 reg_pi_wr.access = reg_wr(wr)->access;
869 reg_pi_wr.key = pi_mr->ibmr.rkey;
870
871 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
872 /* UMR for data + prot registration */
873 err = set_reg_wr(qp, ®_pi_wr, seg, size, cur_edge, false);
874 if (unlikely(err))
875 goto out;
876
877 mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx,
878 wr->wr_id, nreq, fence, MLX5_OPCODE_UMR);
879
880 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
881 if (unlikely(err)) {
882 mlx5_ib_warn(dev, "\n");
883 err = -ENOMEM;
884 goto out;
885 }
886 } else {
887 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
888 /* No UMR, use local_dma_lkey */
889 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
890 pa_pi_mr.mmkey.ndescs = mr->mmkey.ndescs;
891 pa_pi_mr.data_length = mr->data_length;
892 pa_pi_mr.data_iova = mr->data_iova;
893 if (mr->meta_ndescs) {
894 pa_pi_mr.meta_ndescs = mr->meta_ndescs;
895 pa_pi_mr.meta_length = mr->meta_length;
896 pa_pi_mr.pi_iova = mr->pi_iova;
897 }
898
899 pa_pi_mr.ibmr.length = mr->ibmr.length;
900 mr->pi_mr = &pa_pi_mr;
901 }
902 (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
903 /* UMR for sig MR */
904 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
905 if (unlikely(err)) {
906 mlx5_ib_warn(dev, "\n");
907 goto out;
908 }
909 mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
910 nreq, fence, MLX5_OPCODE_UMR);
911
912 sig_attrs = mr->ibmr.sig_attrs;
913 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
914 &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
915 next_fence);
916 if (unlikely(err))
917 goto out;
918
919 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
920 &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
921 next_fence);
922 if (unlikely(err))
923 goto out;
924
925 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
926
927out:
928 return err;
929}
930
931static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
932 const struct ib_send_wr *wr,
933 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
934 void **cur_edge, unsigned int *idx, int nreq, u8 fence,
935 u8 next_fence, int *num_sge)
936{
937 int err = 0;
938
939 switch (wr->opcode) {
940 case IB_WR_RDMA_READ:
941 case IB_WR_RDMA_WRITE:
942 case IB_WR_RDMA_WRITE_WITH_IMM:
943 handle_rdma_op(wr, seg, size);
944 break;
945
946 case IB_WR_ATOMIC_CMP_AND_SWP:
947 case IB_WR_ATOMIC_FETCH_AND_ADD:
948 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
949 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
950 err = -EOPNOTSUPP;
951 goto out;
952
953 case IB_WR_LOCAL_INV:
954 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
955 *num_sge = 0;
956 break;
957
958 case IB_WR_REG_MR:
959 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
960 if (unlikely(err))
961 goto out;
962 *num_sge = 0;
963 break;
964
965 case IB_WR_REG_MR_INTEGRITY:
966 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
967 cur_edge, idx, nreq, fence,
968 next_fence);
969 if (unlikely(err))
970 goto out;
971 *num_sge = 0;
972 break;
973
974 default:
975 break;
976 }
977
978out:
979 return err;
980}
981
982static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
983{
984 switch (wr->opcode) {
985 case IB_WR_RDMA_WRITE:
986 case IB_WR_RDMA_WRITE_WITH_IMM:
987 handle_rdma_op(wr, seg, size);
988 break;
989 default:
990 break;
991 }
992}
993
994static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
995 const struct ib_send_wr *wr, void **seg,
996 int *size, void **cur_edge)
997{
998 set_datagram_seg(*seg, wr);
999 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1000 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1001 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1002}
1003
1004static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1005 void **seg, int *size, void **cur_edge)
1006{
1007 set_datagram_seg(*seg, wr);
1008 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1009 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1010 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1011
1012 /* handle qp that supports ud offload */
1013 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1014 struct mlx5_wqe_eth_pad *pad;
1015
1016 pad = *seg;
1017 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1018 *seg += sizeof(struct mlx5_wqe_eth_pad);
1019 *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1020 set_eth_seg(wr, qp, seg, size, cur_edge);
1021 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1022 }
1023}
1024
1025void mlx5r_ring_db(struct mlx5_ib_qp *qp, unsigned int nreq,
1026 struct mlx5_wqe_ctrl_seg *ctrl)
1027{
1028 struct mlx5_bf *bf = &qp->bf;
1029
1030 qp->sq.head += nreq;
1031
1032 /* Make sure that descriptors are written before
1033 * updating doorbell record and ringing the doorbell
1034 */
1035 wmb();
1036
1037 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1038
1039 /* Make sure doorbell record is visible to the HCA before
1040 * we hit doorbell.
1041 */
1042 wmb();
1043
1044 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1045 /* Make sure doorbells don't leak out of SQ spinlock
1046 * and reach the HCA out of order.
1047 */
1048 bf->offset ^= bf->buf_size;
1049}
1050
1051int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1052 const struct ib_send_wr **bad_wr, bool drain)
1053{
1054 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
1055 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1056 struct mlx5_core_dev *mdev = dev->mdev;
1057 struct mlx5_ib_qp *qp = to_mqp(ibqp);
1058 struct mlx5_wqe_xrc_seg *xrc;
1059 void *cur_edge;
1060 int size;
1061 unsigned long flags;
1062 unsigned int idx;
1063 int err = 0;
1064 int num_sge;
1065 void *seg;
1066 int nreq;
1067 int i;
1068 u8 next_fence = 0;
1069 u8 fence;
1070
1071 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1072 !drain)) {
1073 *bad_wr = wr;
1074 return -EIO;
1075 }
1076
1077 if (qp->type == IB_QPT_GSI)
1078 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1079
1080 spin_lock_irqsave(&qp->sq.lock, flags);
1081
1082 for (nreq = 0; wr; nreq++, wr = wr->next) {
1083 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1084 mlx5_ib_warn(dev, "\n");
1085 err = -EINVAL;
1086 *bad_wr = wr;
1087 goto out;
1088 }
1089
1090 num_sge = wr->num_sge;
1091 if (unlikely(num_sge > qp->sq.max_gs)) {
1092 mlx5_ib_warn(dev, "\n");
1093 err = -EINVAL;
1094 *bad_wr = wr;
1095 goto out;
1096 }
1097
1098 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1099 nreq);
1100 if (err) {
1101 mlx5_ib_warn(dev, "\n");
1102 err = -ENOMEM;
1103 *bad_wr = wr;
1104 goto out;
1105 }
1106
1107 if (wr->opcode == IB_WR_REG_MR ||
1108 wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1109 fence = dev->umr_fence;
1110 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1111 } else {
1112 if (wr->send_flags & IB_SEND_FENCE) {
1113 if (qp->next_fence)
1114 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1115 else
1116 fence = MLX5_FENCE_MODE_FENCE;
1117 } else {
1118 fence = qp->next_fence;
1119 }
1120 }
1121
1122 switch (qp->type) {
1123 case IB_QPT_XRC_INI:
1124 xrc = seg;
1125 seg += sizeof(*xrc);
1126 size += sizeof(*xrc) / 16;
1127 fallthrough;
1128 case IB_QPT_RC:
1129 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1130 &cur_edge, &idx, nreq, fence,
1131 next_fence, &num_sge);
1132 if (unlikely(err)) {
1133 *bad_wr = wr;
1134 goto out;
1135 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1136 goto skip_psv;
1137 }
1138 break;
1139
1140 case IB_QPT_UC:
1141 handle_qpt_uc(wr, &seg, &size);
1142 break;
1143 case IB_QPT_SMI:
1144 if (unlikely(!dev->port_caps[qp->port - 1].has_smi)) {
1145 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1146 err = -EPERM;
1147 *bad_wr = wr;
1148 goto out;
1149 }
1150 fallthrough;
1151 case MLX5_IB_QPT_HW_GSI:
1152 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1153 break;
1154 case IB_QPT_UD:
1155 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1156 break;
1157
1158 default:
1159 break;
1160 }
1161
1162 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1163 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1164 if (unlikely(err)) {
1165 mlx5_ib_warn(dev, "\n");
1166 *bad_wr = wr;
1167 goto out;
1168 }
1169 } else {
1170 for (i = 0; i < num_sge; i++) {
1171 handle_post_send_edge(&qp->sq, &seg, size,
1172 &cur_edge);
1173 if (unlikely(!wr->sg_list[i].length))
1174 continue;
1175
1176 set_data_ptr_seg(
1177 (struct mlx5_wqe_data_seg *)seg,
1178 wr->sg_list + i);
1179 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1180 seg += sizeof(struct mlx5_wqe_data_seg);
1181 }
1182 }
1183
1184 qp->next_fence = next_fence;
1185 mlx5r_finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id,
1186 nreq, fence, mlx5_ib_opcode[wr->opcode]);
1187skip_psv:
1188 if (0)
1189 dump_wqe(qp, idx, size);
1190 }
1191
1192out:
1193 if (likely(nreq))
1194 mlx5r_ring_db(qp, nreq, ctrl);
1195
1196 spin_unlock_irqrestore(&qp->sq.lock, flags);
1197
1198 return err;
1199}
1200
1201static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1202{
1203 sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1204}
1205
1206int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1207 const struct ib_recv_wr **bad_wr, bool drain)
1208{
1209 struct mlx5_ib_qp *qp = to_mqp(ibqp);
1210 struct mlx5_wqe_data_seg *scat;
1211 struct mlx5_rwqe_sig *sig;
1212 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1213 struct mlx5_core_dev *mdev = dev->mdev;
1214 unsigned long flags;
1215 int err = 0;
1216 int nreq;
1217 int ind;
1218 int i;
1219
1220 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1221 !drain)) {
1222 *bad_wr = wr;
1223 return -EIO;
1224 }
1225
1226 if (qp->type == IB_QPT_GSI)
1227 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1228
1229 spin_lock_irqsave(&qp->rq.lock, flags);
1230
1231 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1232
1233 for (nreq = 0; wr; nreq++, wr = wr->next) {
1234 if (mlx5r_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1235 err = -ENOMEM;
1236 *bad_wr = wr;
1237 goto out;
1238 }
1239
1240 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1241 err = -EINVAL;
1242 *bad_wr = wr;
1243 goto out;
1244 }
1245
1246 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1247 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1248 scat++;
1249
1250 for (i = 0; i < wr->num_sge; i++)
1251 set_data_ptr_seg(scat + i, wr->sg_list + i);
1252
1253 if (i < qp->rq.max_gs) {
1254 scat[i].byte_count = 0;
1255 scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey;
1256 scat[i].addr = 0;
1257 }
1258
1259 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1260 sig = (struct mlx5_rwqe_sig *)scat;
1261 set_sig_seg(sig, qp->rq.max_gs);
1262 }
1263
1264 qp->rq.wrid[ind] = wr->wr_id;
1265
1266 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1267 }
1268
1269out:
1270 if (likely(nreq)) {
1271 qp->rq.head += nreq;
1272
1273 /* Make sure that descriptors are written before
1274 * doorbell record.
1275 */
1276 wmb();
1277
1278 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1279 }
1280
1281 spin_unlock_irqrestore(&qp->rq.lock, flags);
1282
1283 return err;
1284}
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <linux/gfp.h>
7#include <linux/mlx5/qp.h>
8#include <linux/mlx5/driver.h>
9#include "wr.h"
10
11static const u32 mlx5_ib_opcode[] = {
12 [IB_WR_SEND] = MLX5_OPCODE_SEND,
13 [IB_WR_LSO] = MLX5_OPCODE_LSO,
14 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
15 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
16 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
17 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
18 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
19 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
20 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
21 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
22 [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
23 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
24 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
25 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
26};
27
28/* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
29 * next nearby edge and get new address translation for current WQE position.
30 * @sq - SQ buffer.
31 * @seg: Current WQE position (16B aligned).
32 * @wqe_sz: Total current WQE size [16B].
33 * @cur_edge: Updated current edge.
34 */
35static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
36 u32 wqe_sz, void **cur_edge)
37{
38 u32 idx;
39
40 if (likely(*seg != *cur_edge))
41 return;
42
43 idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
44 *cur_edge = get_sq_edge(sq, idx);
45
46 *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
47}
48
49/* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's
50 * pointers. At the end @seg is aligned to 16B regardless the copied size.
51 * @sq - SQ buffer.
52 * @cur_edge: Updated current edge.
53 * @seg: Current WQE position (16B aligned).
54 * @wqe_sz: Total current WQE size [16B].
55 * @src: Pointer to copy from.
56 * @n: Number of bytes to copy.
57 */
58static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
59 void **seg, u32 *wqe_sz, const void *src,
60 size_t n)
61{
62 while (likely(n)) {
63 size_t leftlen = *cur_edge - *seg;
64 size_t copysz = min_t(size_t, leftlen, n);
65 size_t stride;
66
67 memcpy(*seg, src, copysz);
68
69 n -= copysz;
70 src += copysz;
71 stride = !n ? ALIGN(copysz, 16) : copysz;
72 *seg += stride;
73 *wqe_sz += stride >> 4;
74 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
75 }
76}
77
78static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq,
79 struct ib_cq *ib_cq)
80{
81 struct mlx5_ib_cq *cq;
82 unsigned int cur;
83
84 cur = wq->head - wq->tail;
85 if (likely(cur + nreq < wq->max_post))
86 return 0;
87
88 cq = to_mcq(ib_cq);
89 spin_lock(&cq->lock);
90 cur = wq->head - wq->tail;
91 spin_unlock(&cq->lock);
92
93 return cur + nreq >= wq->max_post;
94}
95
96static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
97 u64 remote_addr, u32 rkey)
98{
99 rseg->raddr = cpu_to_be64(remote_addr);
100 rseg->rkey = cpu_to_be32(rkey);
101 rseg->reserved = 0;
102}
103
104static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
105 void **seg, int *size, void **cur_edge)
106{
107 struct mlx5_wqe_eth_seg *eseg = *seg;
108
109 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
110
111 if (wr->send_flags & IB_SEND_IP_CSUM)
112 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
113 MLX5_ETH_WQE_L4_CSUM;
114
115 if (wr->opcode == IB_WR_LSO) {
116 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
117 size_t left, copysz;
118 void *pdata = ud_wr->header;
119 size_t stride;
120
121 left = ud_wr->hlen;
122 eseg->mss = cpu_to_be16(ud_wr->mss);
123 eseg->inline_hdr.sz = cpu_to_be16(left);
124
125 /* memcpy_send_wqe should get a 16B align address. Hence, we
126 * first copy up to the current edge and then, if needed,
127 * continue to memcpy_send_wqe.
128 */
129 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
130 left);
131 memcpy(eseg->inline_hdr.start, pdata, copysz);
132 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
133 sizeof(eseg->inline_hdr.start) + copysz, 16);
134 *size += stride / 16;
135 *seg += stride;
136
137 if (copysz < left) {
138 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
139 left -= copysz;
140 pdata += copysz;
141 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
142 left);
143 }
144
145 return;
146 }
147
148 *seg += sizeof(struct mlx5_wqe_eth_seg);
149 *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
150}
151
152static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
153 const struct ib_send_wr *wr)
154{
155 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
156 dseg->av.dqp_dct =
157 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
158 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
159}
160
161static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
162{
163 dseg->byte_count = cpu_to_be32(sg->length);
164 dseg->lkey = cpu_to_be32(sg->lkey);
165 dseg->addr = cpu_to_be64(sg->addr);
166}
167
168static u64 get_xlt_octo(u64 bytes)
169{
170 return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
171 MLX5_IB_UMR_OCTOWORD;
172}
173
174static __be64 frwr_mkey_mask(bool atomic)
175{
176 u64 result;
177
178 result = MLX5_MKEY_MASK_LEN |
179 MLX5_MKEY_MASK_PAGE_SIZE |
180 MLX5_MKEY_MASK_START_ADDR |
181 MLX5_MKEY_MASK_EN_RINVAL |
182 MLX5_MKEY_MASK_KEY |
183 MLX5_MKEY_MASK_LR |
184 MLX5_MKEY_MASK_LW |
185 MLX5_MKEY_MASK_RR |
186 MLX5_MKEY_MASK_RW |
187 MLX5_MKEY_MASK_SMALL_FENCE |
188 MLX5_MKEY_MASK_FREE;
189
190 if (atomic)
191 result |= MLX5_MKEY_MASK_A;
192
193 return cpu_to_be64(result);
194}
195
196static __be64 sig_mkey_mask(void)
197{
198 u64 result;
199
200 result = MLX5_MKEY_MASK_LEN |
201 MLX5_MKEY_MASK_PAGE_SIZE |
202 MLX5_MKEY_MASK_START_ADDR |
203 MLX5_MKEY_MASK_EN_SIGERR |
204 MLX5_MKEY_MASK_EN_RINVAL |
205 MLX5_MKEY_MASK_KEY |
206 MLX5_MKEY_MASK_LR |
207 MLX5_MKEY_MASK_LW |
208 MLX5_MKEY_MASK_RR |
209 MLX5_MKEY_MASK_RW |
210 MLX5_MKEY_MASK_SMALL_FENCE |
211 MLX5_MKEY_MASK_FREE |
212 MLX5_MKEY_MASK_BSF_EN;
213
214 return cpu_to_be64(result);
215}
216
217static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
218 struct mlx5_ib_mr *mr, u8 flags, bool atomic)
219{
220 int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
221
222 memset(umr, 0, sizeof(*umr));
223
224 umr->flags = flags;
225 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
226 umr->mkey_mask = frwr_mkey_mask(atomic);
227}
228
229static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
230{
231 memset(umr, 0, sizeof(*umr));
232 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
233 umr->flags = MLX5_UMR_INLINE;
234}
235
236static __be64 get_umr_enable_mr_mask(void)
237{
238 u64 result;
239
240 result = MLX5_MKEY_MASK_KEY |
241 MLX5_MKEY_MASK_FREE;
242
243 return cpu_to_be64(result);
244}
245
246static __be64 get_umr_disable_mr_mask(void)
247{
248 u64 result;
249
250 result = MLX5_MKEY_MASK_FREE;
251
252 return cpu_to_be64(result);
253}
254
255static __be64 get_umr_update_translation_mask(void)
256{
257 u64 result;
258
259 result = MLX5_MKEY_MASK_LEN |
260 MLX5_MKEY_MASK_PAGE_SIZE |
261 MLX5_MKEY_MASK_START_ADDR;
262
263 return cpu_to_be64(result);
264}
265
266static __be64 get_umr_update_access_mask(int atomic,
267 int relaxed_ordering_write,
268 int relaxed_ordering_read)
269{
270 u64 result;
271
272 result = MLX5_MKEY_MASK_LR |
273 MLX5_MKEY_MASK_LW |
274 MLX5_MKEY_MASK_RR |
275 MLX5_MKEY_MASK_RW;
276
277 if (atomic)
278 result |= MLX5_MKEY_MASK_A;
279
280 if (relaxed_ordering_write)
281 result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;
282
283 if (relaxed_ordering_read)
284 result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;
285
286 return cpu_to_be64(result);
287}
288
289static __be64 get_umr_update_pd_mask(void)
290{
291 u64 result;
292
293 result = MLX5_MKEY_MASK_PD;
294
295 return cpu_to_be64(result);
296}
297
298static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
299{
300 if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&
301 MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
302 return -EPERM;
303
304 if (mask & MLX5_MKEY_MASK_A &&
305 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
306 return -EPERM;
307
308 if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&
309 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
310 return -EPERM;
311
312 if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&
313 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
314 return -EPERM;
315
316 return 0;
317}
318
319static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
320 struct mlx5_wqe_umr_ctrl_seg *umr,
321 const struct ib_send_wr *wr)
322{
323 const struct mlx5_umr_wr *umrwr = umr_wr(wr);
324
325 memset(umr, 0, sizeof(*umr));
326
327 if (!umrwr->ignore_free_state) {
328 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
329 /* fail if free */
330 umr->flags = MLX5_UMR_CHECK_FREE;
331 else
332 /* fail if not free */
333 umr->flags = MLX5_UMR_CHECK_NOT_FREE;
334 }
335
336 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
337 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
338 u64 offset = get_xlt_octo(umrwr->offset);
339
340 umr->xlt_offset = cpu_to_be16(offset & 0xffff);
341 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
342 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
343 }
344 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
345 umr->mkey_mask |= get_umr_update_translation_mask();
346 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
347 umr->mkey_mask |= get_umr_update_access_mask(
348 !!(MLX5_CAP_GEN(dev->mdev, atomic)),
349 !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)),
350 !!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)));
351 umr->mkey_mask |= get_umr_update_pd_mask();
352 }
353 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
354 umr->mkey_mask |= get_umr_enable_mr_mask();
355 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
356 umr->mkey_mask |= get_umr_disable_mr_mask();
357
358 if (!wr->num_sge)
359 umr->flags |= MLX5_UMR_INLINE;
360
361 return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
362}
363
364static u8 get_umr_flags(int acc)
365{
366 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
367 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
368 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
369 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
370 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
371}
372
373static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
374 struct mlx5_ib_mr *mr,
375 u32 key, int access)
376{
377 int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
378
379 memset(seg, 0, sizeof(*seg));
380
381 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
382 seg->log2_page_size = ilog2(mr->ibmr.page_size);
383 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
384 /* KLMs take twice the size of MTTs */
385 ndescs *= 2;
386
387 seg->flags = get_umr_flags(access) | mr->access_mode;
388 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
389 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
390 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
391 seg->len = cpu_to_be64(mr->ibmr.length);
392 seg->xlt_oct_size = cpu_to_be32(ndescs);
393}
394
395static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
396{
397 memset(seg, 0, sizeof(*seg));
398 seg->status = MLX5_MKEY_STATUS_FREE;
399}
400
401static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,
402 const struct ib_send_wr *wr)
403{
404 const struct mlx5_umr_wr *umrwr = umr_wr(wr);
405
406 memset(seg, 0, sizeof(*seg));
407 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
408 MLX5_SET(mkc, seg, free, 1);
409
410 MLX5_SET(mkc, seg, a,
411 !!(umrwr->access_flags & IB_ACCESS_REMOTE_ATOMIC));
412 MLX5_SET(mkc, seg, rw,
413 !!(umrwr->access_flags & IB_ACCESS_REMOTE_WRITE));
414 MLX5_SET(mkc, seg, rr, !!(umrwr->access_flags & IB_ACCESS_REMOTE_READ));
415 MLX5_SET(mkc, seg, lw, !!(umrwr->access_flags & IB_ACCESS_LOCAL_WRITE));
416 MLX5_SET(mkc, seg, lr, 1);
417 MLX5_SET(mkc, seg, relaxed_ordering_write,
418 !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
419 MLX5_SET(mkc, seg, relaxed_ordering_read,
420 !!(umrwr->access_flags & IB_ACCESS_RELAXED_ORDERING));
421
422 if (umrwr->pd)
423 MLX5_SET(mkc, seg, pd, to_mpd(umrwr->pd)->pdn);
424 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
425 !umrwr->length)
426 MLX5_SET(mkc, seg, length64, 1);
427
428 MLX5_SET64(mkc, seg, start_addr, umrwr->virt_addr);
429 MLX5_SET64(mkc, seg, len, umrwr->length);
430 MLX5_SET(mkc, seg, log_page_size, umrwr->page_shift);
431 MLX5_SET(mkc, seg, qpn, 0xffffff);
432 MLX5_SET(mkc, seg, mkey_7_0, mlx5_mkey_variant(umrwr->mkey));
433}
434
435static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
436 struct mlx5_ib_mr *mr,
437 struct mlx5_ib_pd *pd)
438{
439 int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
440
441 dseg->addr = cpu_to_be64(mr->desc_map);
442 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
443 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
444}
445
446static __be32 send_ieth(const struct ib_send_wr *wr)
447{
448 switch (wr->opcode) {
449 case IB_WR_SEND_WITH_IMM:
450 case IB_WR_RDMA_WRITE_WITH_IMM:
451 return wr->ex.imm_data;
452
453 case IB_WR_SEND_WITH_INV:
454 return cpu_to_be32(wr->ex.invalidate_rkey);
455
456 default:
457 return 0;
458 }
459}
460
461static u8 calc_sig(void *wqe, int size)
462{
463 u8 *p = wqe;
464 u8 res = 0;
465 int i;
466
467 for (i = 0; i < size; i++)
468 res ^= p[i];
469
470 return ~res;
471}
472
473static u8 wq_sig(void *wqe)
474{
475 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
476}
477
478static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
479 void **wqe, int *wqe_sz, void **cur_edge)
480{
481 struct mlx5_wqe_inline_seg *seg;
482 size_t offset;
483 int inl = 0;
484 int i;
485
486 seg = *wqe;
487 *wqe += sizeof(*seg);
488 offset = sizeof(*seg);
489
490 for (i = 0; i < wr->num_sge; i++) {
491 size_t len = wr->sg_list[i].length;
492 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
493
494 inl += len;
495
496 if (unlikely(inl > qp->max_inline_data))
497 return -ENOMEM;
498
499 while (likely(len)) {
500 size_t leftlen;
501 size_t copysz;
502
503 handle_post_send_edge(&qp->sq, wqe,
504 *wqe_sz + (offset >> 4),
505 cur_edge);
506
507 leftlen = *cur_edge - *wqe;
508 copysz = min_t(size_t, leftlen, len);
509
510 memcpy(*wqe, addr, copysz);
511 len -= copysz;
512 addr += copysz;
513 *wqe += copysz;
514 offset += copysz;
515 }
516 }
517
518 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
519
520 *wqe_sz += ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
521
522 return 0;
523}
524
525static u16 prot_field_size(enum ib_signature_type type)
526{
527 switch (type) {
528 case IB_SIG_TYPE_T10_DIF:
529 return MLX5_DIF_SIZE;
530 default:
531 return 0;
532 }
533}
534
535static u8 bs_selector(int block_size)
536{
537 switch (block_size) {
538 case 512: return 0x1;
539 case 520: return 0x2;
540 case 4096: return 0x3;
541 case 4160: return 0x4;
542 case 1073741824: return 0x5;
543 default: return 0;
544 }
545}
546
547static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
548 struct mlx5_bsf_inl *inl)
549{
550 /* Valid inline section and allow BSF refresh */
551 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
552 MLX5_BSF_REFRESH_DIF);
553 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
554 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
555 /* repeating block */
556 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
557 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
558 MLX5_DIF_CRC : MLX5_DIF_IPCS;
559
560 if (domain->sig.dif.ref_remap)
561 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
562
563 if (domain->sig.dif.app_escape) {
564 if (domain->sig.dif.ref_escape)
565 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
566 else
567 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
568 }
569
570 inl->dif_app_bitmask_check =
571 cpu_to_be16(domain->sig.dif.apptag_check_mask);
572}
573
574static int mlx5_set_bsf(struct ib_mr *sig_mr,
575 struct ib_sig_attrs *sig_attrs,
576 struct mlx5_bsf *bsf, u32 data_size)
577{
578 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
579 struct mlx5_bsf_basic *basic = &bsf->basic;
580 struct ib_sig_domain *mem = &sig_attrs->mem;
581 struct ib_sig_domain *wire = &sig_attrs->wire;
582
583 memset(bsf, 0, sizeof(*bsf));
584
585 /* Basic + Extended + Inline */
586 basic->bsf_size_sbs = 1 << 7;
587 /* Input domain check byte mask */
588 basic->check_byte_mask = sig_attrs->check_mask;
589 basic->raw_data_size = cpu_to_be32(data_size);
590
591 /* Memory domain */
592 switch (sig_attrs->mem.sig_type) {
593 case IB_SIG_TYPE_NONE:
594 break;
595 case IB_SIG_TYPE_T10_DIF:
596 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
597 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
598 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
599 break;
600 default:
601 return -EINVAL;
602 }
603
604 /* Wire domain */
605 switch (sig_attrs->wire.sig_type) {
606 case IB_SIG_TYPE_NONE:
607 break;
608 case IB_SIG_TYPE_T10_DIF:
609 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
610 mem->sig_type == wire->sig_type) {
611 /* Same block structure */
612 basic->bsf_size_sbs |= 1 << 4;
613 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
614 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
615 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
616 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
617 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
618 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
619 } else
620 basic->wire.bs_selector =
621 bs_selector(wire->sig.dif.pi_interval);
622
623 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
624 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
625 break;
626 default:
627 return -EINVAL;
628 }
629
630 return 0;
631}
632
633
634static int set_sig_data_segment(const struct ib_send_wr *send_wr,
635 struct ib_mr *sig_mr,
636 struct ib_sig_attrs *sig_attrs,
637 struct mlx5_ib_qp *qp, void **seg, int *size,
638 void **cur_edge)
639{
640 struct mlx5_bsf *bsf;
641 u32 data_len;
642 u32 data_key;
643 u64 data_va;
644 u32 prot_len = 0;
645 u32 prot_key = 0;
646 u64 prot_va = 0;
647 bool prot = false;
648 int ret;
649 int wqe_size;
650 struct mlx5_ib_mr *mr = to_mmr(sig_mr);
651 struct mlx5_ib_mr *pi_mr = mr->pi_mr;
652
653 data_len = pi_mr->data_length;
654 data_key = pi_mr->ibmr.lkey;
655 data_va = pi_mr->data_iova;
656 if (pi_mr->meta_ndescs) {
657 prot_len = pi_mr->meta_length;
658 prot_key = pi_mr->ibmr.lkey;
659 prot_va = pi_mr->pi_iova;
660 prot = true;
661 }
662
663 if (!prot || (data_key == prot_key && data_va == prot_va &&
664 data_len == prot_len)) {
665 /**
666 * Source domain doesn't contain signature information
667 * or data and protection are interleaved in memory.
668 * So need construct:
669 * ------------------
670 * | data_klm |
671 * ------------------
672 * | BSF |
673 * ------------------
674 **/
675 struct mlx5_klm *data_klm = *seg;
676
677 data_klm->bcount = cpu_to_be32(data_len);
678 data_klm->key = cpu_to_be32(data_key);
679 data_klm->va = cpu_to_be64(data_va);
680 wqe_size = ALIGN(sizeof(*data_klm), 64);
681 } else {
682 /**
683 * Source domain contains signature information
684 * So need construct a strided block format:
685 * ---------------------------
686 * | stride_block_ctrl |
687 * ---------------------------
688 * | data_klm |
689 * ---------------------------
690 * | prot_klm |
691 * ---------------------------
692 * | BSF |
693 * ---------------------------
694 **/
695 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
696 struct mlx5_stride_block_entry *data_sentry;
697 struct mlx5_stride_block_entry *prot_sentry;
698 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
699 int prot_size;
700
701 sblock_ctrl = *seg;
702 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
703 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
704
705 prot_size = prot_field_size(sig_attrs->mem.sig_type);
706 if (!prot_size) {
707 pr_err("Bad block size given: %u\n", block_size);
708 return -EINVAL;
709 }
710 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
711 prot_size);
712 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
713 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
714 sblock_ctrl->num_entries = cpu_to_be16(2);
715
716 data_sentry->bcount = cpu_to_be16(block_size);
717 data_sentry->key = cpu_to_be32(data_key);
718 data_sentry->va = cpu_to_be64(data_va);
719 data_sentry->stride = cpu_to_be16(block_size);
720
721 prot_sentry->bcount = cpu_to_be16(prot_size);
722 prot_sentry->key = cpu_to_be32(prot_key);
723 prot_sentry->va = cpu_to_be64(prot_va);
724 prot_sentry->stride = cpu_to_be16(prot_size);
725
726 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
727 sizeof(*prot_sentry), 64);
728 }
729
730 *seg += wqe_size;
731 *size += wqe_size / 16;
732 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
733
734 bsf = *seg;
735 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
736 if (ret)
737 return -EINVAL;
738
739 *seg += sizeof(*bsf);
740 *size += sizeof(*bsf) / 16;
741 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
742
743 return 0;
744}
745
746static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
747 struct ib_mr *sig_mr, int access_flags,
748 u32 size, u32 length, u32 pdn)
749{
750 u32 sig_key = sig_mr->rkey;
751 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
752
753 memset(seg, 0, sizeof(*seg));
754
755 seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
756 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
757 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
758 MLX5_MKEY_BSF_EN | pdn);
759 seg->len = cpu_to_be64(length);
760 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
761 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
762}
763
764static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
765 u32 size)
766{
767 memset(umr, 0, sizeof(*umr));
768
769 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
770 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
771 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
772 umr->mkey_mask = sig_mkey_mask();
773}
774
775static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
776 struct mlx5_ib_qp *qp, void **seg, int *size,
777 void **cur_edge)
778{
779 const struct ib_reg_wr *wr = reg_wr(send_wr);
780 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
781 struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
782 struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
783 u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
784 u32 xlt_size;
785 int region_len, ret;
786
787 if (unlikely(send_wr->num_sge != 0) ||
788 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
789 unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
790 unlikely(!sig_mr->sig->sig_status_checked))
791 return -EINVAL;
792
793 /* length of the protected region, data + protection */
794 region_len = pi_mr->ibmr.length;
795
796 /**
797 * KLM octoword size - if protection was provided
798 * then we use strided block format (3 octowords),
799 * else we use single KLM (1 octoword)
800 **/
801 if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
802 xlt_size = 0x30;
803 else
804 xlt_size = sizeof(struct mlx5_klm);
805
806 set_sig_umr_segment(*seg, xlt_size);
807 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
808 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
809 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
810
811 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
812 pdn);
813 *seg += sizeof(struct mlx5_mkey_seg);
814 *size += sizeof(struct mlx5_mkey_seg) / 16;
815 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
816
817 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
818 cur_edge);
819 if (ret)
820 return ret;
821
822 sig_mr->sig->sig_status_checked = false;
823 return 0;
824}
825
826static int set_psv_wr(struct ib_sig_domain *domain,
827 u32 psv_idx, void **seg, int *size)
828{
829 struct mlx5_seg_set_psv *psv_seg = *seg;
830
831 memset(psv_seg, 0, sizeof(*psv_seg));
832 psv_seg->psv_num = cpu_to_be32(psv_idx);
833 switch (domain->sig_type) {
834 case IB_SIG_TYPE_NONE:
835 break;
836 case IB_SIG_TYPE_T10_DIF:
837 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
838 domain->sig.dif.app_tag);
839 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
840 break;
841 default:
842 pr_err("Bad signature type (%d) is given.\n",
843 domain->sig_type);
844 return -EINVAL;
845 }
846
847 *seg += sizeof(*psv_seg);
848 *size += sizeof(*psv_seg) / 16;
849
850 return 0;
851}
852
853static int set_reg_wr(struct mlx5_ib_qp *qp,
854 const struct ib_reg_wr *wr,
855 void **seg, int *size, void **cur_edge,
856 bool check_not_free)
857{
858 struct mlx5_ib_mr *mr = to_mmr(wr->mr);
859 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
860 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
861 int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
862 bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
863 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
864 u8 flags = 0;
865
866 if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) {
867 mlx5_ib_warn(to_mdev(qp->ibqp.device),
868 "Fast update of %s for MR is disabled\n",
869 (MLX5_CAP_GEN(dev->mdev,
870 umr_modify_entity_size_disabled)) ?
871 "entity size" :
872 "atomic access");
873 return -EINVAL;
874 }
875
876 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
877 mlx5_ib_warn(to_mdev(qp->ibqp.device),
878 "Invalid IB_SEND_INLINE send flag\n");
879 return -EINVAL;
880 }
881
882 if (check_not_free)
883 flags |= MLX5_UMR_CHECK_NOT_FREE;
884 if (umr_inline)
885 flags |= MLX5_UMR_INLINE;
886
887 set_reg_umr_seg(*seg, mr, flags, atomic);
888 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
889 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
890 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
891
892 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
893 *seg += sizeof(struct mlx5_mkey_seg);
894 *size += sizeof(struct mlx5_mkey_seg) / 16;
895 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
896
897 if (umr_inline) {
898 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
899 mr_list_size);
900 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
901 } else {
902 set_reg_data_seg(*seg, mr, pd);
903 *seg += sizeof(struct mlx5_wqe_data_seg);
904 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
905 }
906 return 0;
907}
908
909static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
910 void **cur_edge)
911{
912 set_linv_umr_seg(*seg);
913 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
914 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
915 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
916 set_linv_mkey_seg(*seg);
917 *seg += sizeof(struct mlx5_mkey_seg);
918 *size += sizeof(struct mlx5_mkey_seg) / 16;
919 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
920}
921
922static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
923{
924 __be32 *p = NULL;
925 int i, j;
926
927 pr_debug("dump WQE index %u:\n", idx);
928 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
929 if ((i & 0xf) == 0) {
930 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
931 pr_debug("WQBB at %p:\n", (void *)p);
932 j = 0;
933 idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
934 }
935 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
936 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
937 be32_to_cpu(p[j + 3]));
938 }
939}
940
941static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
942 struct mlx5_wqe_ctrl_seg **ctrl,
943 const struct ib_send_wr *wr, unsigned int *idx,
944 int *size, void **cur_edge, int nreq,
945 bool send_signaled, bool solicited)
946{
947 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
948 return -ENOMEM;
949
950 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
951 *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
952 *ctrl = *seg;
953 *(uint32_t *)(*seg + 8) = 0;
954 (*ctrl)->imm = send_ieth(wr);
955 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
956 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
957 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
958
959 *seg += sizeof(**ctrl);
960 *size = sizeof(**ctrl) / 16;
961 *cur_edge = qp->sq.cur_edge;
962
963 return 0;
964}
965
966static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
967 struct mlx5_wqe_ctrl_seg **ctrl,
968 const struct ib_send_wr *wr, unsigned int *idx, int *size,
969 void **cur_edge, int nreq)
970{
971 return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
972 wr->send_flags & IB_SEND_SIGNALED,
973 wr->send_flags & IB_SEND_SOLICITED);
974}
975
976static void finish_wqe(struct mlx5_ib_qp *qp,
977 struct mlx5_wqe_ctrl_seg *ctrl,
978 void *seg, u8 size, void *cur_edge,
979 unsigned int idx, u64 wr_id, int nreq, u8 fence,
980 u32 mlx5_opcode)
981{
982 u8 opmod = 0;
983
984 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
985 mlx5_opcode | ((u32)opmod << 24));
986 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
987 ctrl->fm_ce_se |= fence;
988 if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
989 ctrl->signature = wq_sig(ctrl);
990
991 qp->sq.wrid[idx] = wr_id;
992 qp->sq.w_list[idx].opcode = mlx5_opcode;
993 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
994 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
995 qp->sq.w_list[idx].next = qp->sq.cur_post;
996
997 /* We save the edge which was possibly updated during the WQE
998 * construction, into SQ's cache.
999 */
1000 seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
1001 qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
1002 get_sq_edge(&qp->sq, qp->sq.cur_post &
1003 (qp->sq.wqe_cnt - 1)) :
1004 cur_edge;
1005}
1006
1007static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
1008{
1009 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
1010 *seg += sizeof(struct mlx5_wqe_raddr_seg);
1011 *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
1012}
1013
1014static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1015 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1016 int *size, void **cur_edge, unsigned int idx)
1017{
1018 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
1019 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
1020 set_linv_wr(qp, seg, size, cur_edge);
1021}
1022
1023static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1024 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1025 void **cur_edge, unsigned int idx)
1026{
1027 qp->sq.wr_data[idx] = IB_WR_REG_MR;
1028 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
1029 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
1030}
1031
1032static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1033 const struct ib_send_wr *wr,
1034 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1035 void **cur_edge, unsigned int *idx, int nreq,
1036 struct ib_sig_domain *domain, u32 psv_index,
1037 u8 next_fence)
1038{
1039 int err;
1040
1041 /*
1042 * SET_PSV WQEs are not signaled and solicited on error.
1043 */
1044 err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
1045 false, true);
1046 if (unlikely(err)) {
1047 mlx5_ib_warn(dev, "\n");
1048 err = -ENOMEM;
1049 goto out;
1050 }
1051 err = set_psv_wr(domain, psv_index, seg, size);
1052 if (unlikely(err)) {
1053 mlx5_ib_warn(dev, "\n");
1054 goto out;
1055 }
1056 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1057 next_fence, MLX5_OPCODE_SET_PSV);
1058
1059out:
1060 return err;
1061}
1062
1063static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
1064 struct mlx5_ib_qp *qp,
1065 const struct ib_send_wr *wr,
1066 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1067 int *size, void **cur_edge,
1068 unsigned int *idx, int nreq, u8 fence,
1069 u8 next_fence)
1070{
1071 struct mlx5_ib_mr *mr;
1072 struct mlx5_ib_mr *pi_mr;
1073 struct mlx5_ib_mr pa_pi_mr;
1074 struct ib_sig_attrs *sig_attrs;
1075 struct ib_reg_wr reg_pi_wr;
1076 int err;
1077
1078 qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
1079
1080 mr = to_mmr(reg_wr(wr)->mr);
1081 pi_mr = mr->pi_mr;
1082
1083 if (pi_mr) {
1084 memset(®_pi_wr, 0,
1085 sizeof(struct ib_reg_wr));
1086
1087 reg_pi_wr.mr = &pi_mr->ibmr;
1088 reg_pi_wr.access = reg_wr(wr)->access;
1089 reg_pi_wr.key = pi_mr->ibmr.rkey;
1090
1091 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
1092 /* UMR for data + prot registration */
1093 err = set_reg_wr(qp, ®_pi_wr, seg, size, cur_edge, false);
1094 if (unlikely(err))
1095 goto out;
1096
1097 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
1098 nreq, fence, MLX5_OPCODE_UMR);
1099
1100 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
1101 if (unlikely(err)) {
1102 mlx5_ib_warn(dev, "\n");
1103 err = -ENOMEM;
1104 goto out;
1105 }
1106 } else {
1107 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
1108 /* No UMR, use local_dma_lkey */
1109 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
1110 pa_pi_mr.ndescs = mr->ndescs;
1111 pa_pi_mr.data_length = mr->data_length;
1112 pa_pi_mr.data_iova = mr->data_iova;
1113 if (mr->meta_ndescs) {
1114 pa_pi_mr.meta_ndescs = mr->meta_ndescs;
1115 pa_pi_mr.meta_length = mr->meta_length;
1116 pa_pi_mr.pi_iova = mr->pi_iova;
1117 }
1118
1119 pa_pi_mr.ibmr.length = mr->ibmr.length;
1120 mr->pi_mr = &pa_pi_mr;
1121 }
1122 (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
1123 /* UMR for sig MR */
1124 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
1125 if (unlikely(err)) {
1126 mlx5_ib_warn(dev, "\n");
1127 goto out;
1128 }
1129 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1130 fence, MLX5_OPCODE_UMR);
1131
1132 sig_attrs = mr->ibmr.sig_attrs;
1133 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1134 &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
1135 next_fence);
1136 if (unlikely(err))
1137 goto out;
1138
1139 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1140 &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
1141 next_fence);
1142 if (unlikely(err))
1143 goto out;
1144
1145 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1146
1147out:
1148 return err;
1149}
1150
1151static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1152 const struct ib_send_wr *wr,
1153 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1154 void **cur_edge, unsigned int *idx, int nreq, u8 fence,
1155 u8 next_fence, int *num_sge)
1156{
1157 int err = 0;
1158
1159 switch (wr->opcode) {
1160 case IB_WR_RDMA_READ:
1161 case IB_WR_RDMA_WRITE:
1162 case IB_WR_RDMA_WRITE_WITH_IMM:
1163 handle_rdma_op(wr, seg, size);
1164 break;
1165
1166 case IB_WR_ATOMIC_CMP_AND_SWP:
1167 case IB_WR_ATOMIC_FETCH_AND_ADD:
1168 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
1169 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
1170 err = -EOPNOTSUPP;
1171 goto out;
1172
1173 case IB_WR_LOCAL_INV:
1174 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
1175 *num_sge = 0;
1176 break;
1177
1178 case IB_WR_REG_MR:
1179 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
1180 if (unlikely(err))
1181 goto out;
1182 *num_sge = 0;
1183 break;
1184
1185 case IB_WR_REG_MR_INTEGRITY:
1186 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
1187 cur_edge, idx, nreq, fence,
1188 next_fence);
1189 if (unlikely(err))
1190 goto out;
1191 *num_sge = 0;
1192 break;
1193
1194 default:
1195 break;
1196 }
1197
1198out:
1199 return err;
1200}
1201
1202static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
1203{
1204 switch (wr->opcode) {
1205 case IB_WR_RDMA_WRITE:
1206 case IB_WR_RDMA_WRITE_WITH_IMM:
1207 handle_rdma_op(wr, seg, size);
1208 break;
1209 default:
1210 break;
1211 }
1212}
1213
1214static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
1215 const struct ib_send_wr *wr, void **seg,
1216 int *size, void **cur_edge)
1217{
1218 set_datagram_seg(*seg, wr);
1219 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1220 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1221 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1222}
1223
1224static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1225 void **seg, int *size, void **cur_edge)
1226{
1227 set_datagram_seg(*seg, wr);
1228 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1229 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1230 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1231
1232 /* handle qp that supports ud offload */
1233 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1234 struct mlx5_wqe_eth_pad *pad;
1235
1236 pad = *seg;
1237 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1238 *seg += sizeof(struct mlx5_wqe_eth_pad);
1239 *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1240 set_eth_seg(wr, qp, seg, size, cur_edge);
1241 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1242 }
1243}
1244
1245static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1246 const struct ib_send_wr *wr,
1247 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1248 int *size, void **cur_edge, unsigned int idx)
1249{
1250 int err = 0;
1251
1252 if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) {
1253 err = -EINVAL;
1254 mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode);
1255 goto out;
1256 }
1257
1258 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
1259 (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
1260 err = set_reg_umr_segment(dev, *seg, wr);
1261 if (unlikely(err))
1262 goto out;
1263 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
1264 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
1265 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1266 set_reg_mkey_segment(*seg, wr);
1267 *seg += sizeof(struct mlx5_mkey_seg);
1268 *size += sizeof(struct mlx5_mkey_seg) / 16;
1269 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1270out:
1271 return err;
1272}
1273
1274int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1275 const struct ib_send_wr **bad_wr, bool drain)
1276{
1277 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
1278 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1279 struct mlx5_core_dev *mdev = dev->mdev;
1280 struct mlx5_ib_qp *qp;
1281 struct mlx5_wqe_xrc_seg *xrc;
1282 struct mlx5_bf *bf;
1283 void *cur_edge;
1284 int size;
1285 unsigned long flags;
1286 unsigned int idx;
1287 int err = 0;
1288 int num_sge;
1289 void *seg;
1290 int nreq;
1291 int i;
1292 u8 next_fence = 0;
1293 u8 fence;
1294
1295 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1296 !drain)) {
1297 *bad_wr = wr;
1298 return -EIO;
1299 }
1300
1301 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1302 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1303
1304 qp = to_mqp(ibqp);
1305 bf = &qp->bf;
1306
1307 spin_lock_irqsave(&qp->sq.lock, flags);
1308
1309 for (nreq = 0; wr; nreq++, wr = wr->next) {
1310 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1311 mlx5_ib_warn(dev, "\n");
1312 err = -EINVAL;
1313 *bad_wr = wr;
1314 goto out;
1315 }
1316
1317 num_sge = wr->num_sge;
1318 if (unlikely(num_sge > qp->sq.max_gs)) {
1319 mlx5_ib_warn(dev, "\n");
1320 err = -EINVAL;
1321 *bad_wr = wr;
1322 goto out;
1323 }
1324
1325 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1326 nreq);
1327 if (err) {
1328 mlx5_ib_warn(dev, "\n");
1329 err = -ENOMEM;
1330 *bad_wr = wr;
1331 goto out;
1332 }
1333
1334 if (wr->opcode == IB_WR_REG_MR ||
1335 wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1336 fence = dev->umr_fence;
1337 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1338 } else {
1339 if (wr->send_flags & IB_SEND_FENCE) {
1340 if (qp->next_fence)
1341 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1342 else
1343 fence = MLX5_FENCE_MODE_FENCE;
1344 } else {
1345 fence = qp->next_fence;
1346 }
1347 }
1348
1349 switch (ibqp->qp_type) {
1350 case IB_QPT_XRC_INI:
1351 xrc = seg;
1352 seg += sizeof(*xrc);
1353 size += sizeof(*xrc) / 16;
1354 fallthrough;
1355 case IB_QPT_RC:
1356 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1357 &cur_edge, &idx, nreq, fence,
1358 next_fence, &num_sge);
1359 if (unlikely(err)) {
1360 *bad_wr = wr;
1361 goto out;
1362 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1363 goto skip_psv;
1364 }
1365 break;
1366
1367 case IB_QPT_UC:
1368 handle_qpt_uc(wr, &seg, &size);
1369 break;
1370 case IB_QPT_SMI:
1371 if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
1372 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1373 err = -EPERM;
1374 *bad_wr = wr;
1375 goto out;
1376 }
1377 fallthrough;
1378 case MLX5_IB_QPT_HW_GSI:
1379 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1380 break;
1381 case IB_QPT_UD:
1382 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1383 break;
1384 case MLX5_IB_QPT_REG_UMR:
1385 err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg,
1386 &size, &cur_edge, idx);
1387 if (unlikely(err))
1388 goto out;
1389 break;
1390
1391 default:
1392 break;
1393 }
1394
1395 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1396 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1397 if (unlikely(err)) {
1398 mlx5_ib_warn(dev, "\n");
1399 *bad_wr = wr;
1400 goto out;
1401 }
1402 } else {
1403 for (i = 0; i < num_sge; i++) {
1404 handle_post_send_edge(&qp->sq, &seg, size,
1405 &cur_edge);
1406 if (unlikely(!wr->sg_list[i].length))
1407 continue;
1408
1409 set_data_ptr_seg(
1410 (struct mlx5_wqe_data_seg *)seg,
1411 wr->sg_list + i);
1412 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1413 seg += sizeof(struct mlx5_wqe_data_seg);
1414 }
1415 }
1416
1417 qp->next_fence = next_fence;
1418 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,
1419 fence, mlx5_ib_opcode[wr->opcode]);
1420skip_psv:
1421 if (0)
1422 dump_wqe(qp, idx, size);
1423 }
1424
1425out:
1426 if (likely(nreq)) {
1427 qp->sq.head += nreq;
1428
1429 /* Make sure that descriptors are written before
1430 * updating doorbell record and ringing the doorbell
1431 */
1432 wmb();
1433
1434 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1435
1436 /* Make sure doorbell record is visible to the HCA before
1437 * we hit doorbell.
1438 */
1439 wmb();
1440
1441 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1442 /* Make sure doorbells don't leak out of SQ spinlock
1443 * and reach the HCA out of order.
1444 */
1445 bf->offset ^= bf->buf_size;
1446 }
1447
1448 spin_unlock_irqrestore(&qp->sq.lock, flags);
1449
1450 return err;
1451}
1452
1453static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1454{
1455 sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1456}
1457
1458int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1459 const struct ib_recv_wr **bad_wr, bool drain)
1460{
1461 struct mlx5_ib_qp *qp = to_mqp(ibqp);
1462 struct mlx5_wqe_data_seg *scat;
1463 struct mlx5_rwqe_sig *sig;
1464 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1465 struct mlx5_core_dev *mdev = dev->mdev;
1466 unsigned long flags;
1467 int err = 0;
1468 int nreq;
1469 int ind;
1470 int i;
1471
1472 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1473 !drain)) {
1474 *bad_wr = wr;
1475 return -EIO;
1476 }
1477
1478 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1479 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1480
1481 spin_lock_irqsave(&qp->rq.lock, flags);
1482
1483 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1484
1485 for (nreq = 0; wr; nreq++, wr = wr->next) {
1486 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1487 err = -ENOMEM;
1488 *bad_wr = wr;
1489 goto out;
1490 }
1491
1492 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1493 err = -EINVAL;
1494 *bad_wr = wr;
1495 goto out;
1496 }
1497
1498 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1499 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1500 scat++;
1501
1502 for (i = 0; i < wr->num_sge; i++)
1503 set_data_ptr_seg(scat + i, wr->sg_list + i);
1504
1505 if (i < qp->rq.max_gs) {
1506 scat[i].byte_count = 0;
1507 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
1508 scat[i].addr = 0;
1509 }
1510
1511 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1512 sig = (struct mlx5_rwqe_sig *)scat;
1513 set_sig_seg(sig, qp->rq.max_gs);
1514 }
1515
1516 qp->rq.wrid[ind] = wr->wr_id;
1517
1518 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1519 }
1520
1521out:
1522 if (likely(nreq)) {
1523 qp->rq.head += nreq;
1524
1525 /* Make sure that descriptors are written before
1526 * doorbell record.
1527 */
1528 wmb();
1529
1530 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1531 }
1532
1533 spin_unlock_irqrestore(&qp->rq.lock, flags);
1534
1535 return err;
1536}