Loading...
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2
3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4/* Kai Shen <kaishen@linux.alibaba.com> */
5/* Copyright (c) 2020-2022, Alibaba Group. */
6
7#ifndef __ERDMA_HW_H__
8#define __ERDMA_HW_H__
9
10#include <linux/kernel.h>
11#include <linux/types.h>
12
13/* PCIe device related definition. */
14#define ERDMA_PCI_WIDTH 64
15#define ERDMA_FUNC_BAR 0
16#define ERDMA_MISX_BAR 2
17
18#define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))
19
20/* MSI-X related. */
21#define ERDMA_NUM_MSIX_VEC 32U
22#define ERDMA_MSIX_VECTOR_CMDQ 0
23
24/* PCIe Bar0 Registers. */
25#define ERDMA_REGS_VERSION_REG 0x0
26#define ERDMA_REGS_DEV_CTRL_REG 0x10
27#define ERDMA_REGS_DEV_ST_REG 0x14
28#define ERDMA_REGS_NETDEV_MAC_L_REG 0x18
29#define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C
30#define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20
31#define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24
32#define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28
33#define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C
34#define ERDMA_REGS_CMDQ_DEPTH_REG 0x30
35#define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34
36#define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38
37#define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C
38#define ERDMA_REGS_AEQ_ADDR_L_REG 0x40
39#define ERDMA_REGS_AEQ_ADDR_H_REG 0x44
40#define ERDMA_REGS_AEQ_DEPTH_REG 0x48
41#define ERDMA_REGS_GRP_NUM_REG 0x4c
42#define ERDMA_REGS_AEQ_DB_REG 0x50
43#define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60
44#define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68
45#define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70
46#define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78
47#define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80
48#define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88
49#define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90
50#define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98
51#define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0
52#define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8
53#define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0
54#define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8
55#define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0
56#define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8
57#define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0
58#define ERDMA_REGS_CEQ_DB_BASE_REG 0x100
59#define ERDMA_CMDQ_SQDB_REG 0x200
60#define ERDMA_CMDQ_CQDB_REG 0x300
61
62/* DEV_CTRL_REG details. */
63#define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001
64#define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002
65
66/* DEV_ST_REG details. */
67#define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U
68#define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U
69
70/* eRDMA PCIe DBs definition. */
71#define ERDMA_BAR_DB_SPACE_BASE 4096
72
73#define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE
74#define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024)
75
76#define ERDMA_BAR_RQDB_SPACE_OFFSET \
77 (ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)
78#define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024)
79
80#define ERDMA_BAR_CQDB_SPACE_OFFSET \
81 (ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)
82
83#define ERDMA_SDB_SHARED_PAGE_INDEX 95
84
85/* Doorbell related. */
86#define ERDMA_DB_SIZE 8
87
88#define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56)
89#define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)
90#define ERDMA_CQDB_ARM_MASK BIT_ULL(31)
91#define ERDMA_CQDB_SOL_MASK BIT_ULL(30)
92#define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)
93#define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)
94
95#define ERDMA_EQDB_ARM_MASK BIT(31)
96#define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0)
97
98#define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
99
100/* Hardware page size definition */
101#define ERDMA_HW_PAGE_SHIFT 12
102#define ERDMA_HW_PAGE_SIZE 4096
103
104/* WQE related. */
105#define EQE_SIZE 16
106#define EQE_SHIFT 4
107#define RQE_SIZE 32
108#define RQE_SHIFT 5
109#define CQE_SIZE 32
110#define CQE_SHIFT 5
111#define SQEBB_SIZE 32
112#define SQEBB_SHIFT 5
113#define SQEBB_MASK (~(SQEBB_SIZE - 1))
114#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
115#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
116
117#define ERDMA_MAX_SQE_SIZE 128
118#define ERDMA_MAX_WQEBB_PER_SQE 4
119
120/* CMDQ related. */
121#define ERDMA_CMDQ_MAX_OUTSTANDING 128
122#define ERDMA_CMDQ_SQE_SIZE 128
123
124/* cmdq sub module definition. */
125enum CMDQ_WQE_SUB_MOD {
126 CMDQ_SUBMOD_RDMA = 0,
127 CMDQ_SUBMOD_COMMON = 1
128};
129
130enum CMDQ_RDMA_OPCODE {
131 CMDQ_OPCODE_QUERY_DEVICE = 0,
132 CMDQ_OPCODE_CREATE_QP = 1,
133 CMDQ_OPCODE_DESTROY_QP = 2,
134 CMDQ_OPCODE_MODIFY_QP = 3,
135 CMDQ_OPCODE_CREATE_CQ = 4,
136 CMDQ_OPCODE_DESTROY_CQ = 5,
137 CMDQ_OPCODE_REFLUSH = 6,
138 CMDQ_OPCODE_REG_MR = 8,
139 CMDQ_OPCODE_DEREG_MR = 9
140};
141
142enum CMDQ_COMMON_OPCODE {
143 CMDQ_OPCODE_CREATE_EQ = 0,
144 CMDQ_OPCODE_DESTROY_EQ = 1,
145 CMDQ_OPCODE_QUERY_FW_INFO = 2,
146 CMDQ_OPCODE_CONF_MTU = 3,
147 CMDQ_OPCODE_GET_STATS = 4,
148 CMDQ_OPCODE_CONF_DEVICE = 5,
149 CMDQ_OPCODE_ALLOC_DB = 8,
150 CMDQ_OPCODE_FREE_DB = 9,
151};
152
153/* cmdq-SQE HDR */
154#define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
155#define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32)
156#define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24)
157#define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16)
158#define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
159
160struct erdma_cmdq_destroy_cq_req {
161 u64 hdr;
162 u32 cqn;
163};
164
165#define ERDMA_EQ_TYPE_AEQ 0
166#define ERDMA_EQ_TYPE_CEQ 1
167
168struct erdma_cmdq_create_eq_req {
169 u64 hdr;
170 u64 qbuf_addr;
171 u8 vector_idx;
172 u8 eqn;
173 u8 depth;
174 u8 qtype;
175 u32 db_dma_addr_l;
176 u32 db_dma_addr_h;
177};
178
179struct erdma_cmdq_destroy_eq_req {
180 u64 hdr;
181 u64 rsvd0;
182 u8 vector_idx;
183 u8 eqn;
184 u8 rsvd1;
185 u8 qtype;
186};
187
188/* config device cfg */
189#define ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK BIT(31)
190#define ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK GENMASK(4, 0)
191
192struct erdma_cmdq_config_device_req {
193 u64 hdr;
194 u32 cfg;
195 u32 rsvd[5];
196};
197
198struct erdma_cmdq_config_mtu_req {
199 u64 hdr;
200 u32 mtu;
201};
202
203/* ext db requests(alloc and free) cfg */
204#define ERDMA_CMD_EXT_DB_CQ_EN_MASK BIT(2)
205#define ERDMA_CMD_EXT_DB_RQ_EN_MASK BIT(1)
206#define ERDMA_CMD_EXT_DB_SQ_EN_MASK BIT(0)
207
208struct erdma_cmdq_ext_db_req {
209 u64 hdr;
210 u32 cfg;
211 u16 rdb_off;
212 u16 sdb_off;
213 u16 rsvd0;
214 u16 cdb_off;
215 u32 rsvd1[3];
216};
217
218/* alloc db response qword 0 definition */
219#define ERDMA_CMD_ALLOC_DB_RESP_RDB_MASK GENMASK_ULL(63, 48)
220#define ERDMA_CMD_ALLOC_DB_RESP_CDB_MASK GENMASK_ULL(47, 32)
221#define ERDMA_CMD_ALLOC_DB_RESP_SDB_MASK GENMASK_ULL(15, 0)
222
223/* create_cq cfg0 */
224#define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24)
225#define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20)
226#define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0)
227
228/* create_cq cfg1 */
229#define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)
230#define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15)
231#define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)
232#define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)
233
234/* create_cq cfg2 */
235#define ERDMA_CMD_CREATE_CQ_DB_CFG_MASK GENMASK(15, 0)
236
237struct erdma_cmdq_create_cq_req {
238 u64 hdr;
239 u32 cfg0;
240 u32 qbuf_addr_l;
241 u32 qbuf_addr_h;
242 u32 cfg1;
243 u64 cq_db_info_addr;
244 u32 first_page_offset;
245 u32 cfg2;
246};
247
248/* regmr/deregmr cfg0 */
249#define ERDMA_CMD_MR_VALID_MASK BIT(31)
250#define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28)
251#define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)
252#define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)
253
254/* regmr cfg1 */
255#define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
256#define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
257#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
258
259/* regmr cfg2 */
260#define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
261#define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24)
262#define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)
263#define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)
264
265struct erdma_cmdq_reg_mr_req {
266 u64 hdr;
267 u32 cfg0;
268 u32 cfg1;
269 u64 start_va;
270 u32 size;
271 u32 cfg2;
272 union {
273 u64 phy_addr[4];
274 struct {
275 u64 rsvd;
276 u32 size_h;
277 u32 mtt_cnt_h;
278 };
279 };
280};
281
282struct erdma_cmdq_dereg_mr_req {
283 u64 hdr;
284 u32 cfg;
285};
286
287/* modify qp cfg */
288#define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24)
289#define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20)
290#define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0)
291
292struct erdma_cmdq_modify_qp_req {
293 u64 hdr;
294 u32 cfg;
295 u32 cookie;
296 __be32 dip;
297 __be32 sip;
298 __be16 sport;
299 __be16 dport;
300 u32 send_nxt;
301 u32 recv_nxt;
302};
303
304/* create qp cfg0 */
305#define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20)
306#define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0)
307
308/* create qp cfg1 */
309#define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20)
310#define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0)
311
312/* create qp cqn_mtt_cfg */
313#define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)
314#define ERDMA_CMD_CREATE_QP_DB_CFG_MASK BIT(25)
315#define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)
316
317/* create qp mtt_cfg */
318#define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)
319#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)
320#define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0)
321
322/* create qp db cfg */
323#define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)
324#define ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK GENMASK(15, 0)
325
326#define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)
327
328struct erdma_cmdq_create_qp_req {
329 u64 hdr;
330 u32 cfg0;
331 u32 cfg1;
332 u32 sq_cqn_mtt_cfg;
333 u32 rq_cqn_mtt_cfg;
334 u64 sq_buf_addr;
335 u64 rq_buf_addr;
336 u32 sq_mtt_cfg;
337 u32 rq_mtt_cfg;
338 u64 sq_db_info_dma_addr;
339 u64 rq_db_info_dma_addr;
340
341 u64 sq_mtt_entry[3];
342 u64 rq_mtt_entry[3];
343
344 u32 db_cfg;
345};
346
347struct erdma_cmdq_destroy_qp_req {
348 u64 hdr;
349 u32 qpn;
350};
351
352struct erdma_cmdq_reflush_req {
353 u64 hdr;
354 u32 qpn;
355 u32 sq_pi;
356 u32 rq_pi;
357};
358
359#define ERDMA_HW_RESP_SIZE 256
360
361struct erdma_cmdq_query_req {
362 u64 hdr;
363 u32 rsvd;
364 u32 index;
365
366 u64 target_addr;
367 u32 target_length;
368};
369
370#define ERDMA_HW_RESP_MAGIC 0x5566
371
372struct erdma_cmdq_query_resp_hdr {
373 u16 magic;
374 u8 ver;
375 u8 length;
376
377 u32 index;
378 u32 rsvd[2];
379};
380
381struct erdma_cmdq_query_stats_resp {
382 struct erdma_cmdq_query_resp_hdr hdr;
383
384 u64 tx_req_cnt;
385 u64 tx_packets_cnt;
386 u64 tx_bytes_cnt;
387 u64 tx_drop_packets_cnt;
388 u64 tx_bps_meter_drop_packets_cnt;
389 u64 tx_pps_meter_drop_packets_cnt;
390 u64 rx_packets_cnt;
391 u64 rx_bytes_cnt;
392 u64 rx_drop_packets_cnt;
393 u64 rx_bps_meter_drop_packets_cnt;
394 u64 rx_pps_meter_drop_packets_cnt;
395};
396
397/* cap qword 0 definition */
398#define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
399#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
400#define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
401#define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
402
403/* cap qword 1 definition */
404#define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)
405#define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28)
406#define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16)
407#define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0)
408
409#define ERDMA_NQP_PER_QBLOCK 1024
410
411enum {
412 ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
413 ERDMA_DEV_CAP_FLAGS_MTT_VA = 1 << 5,
414 ERDMA_DEV_CAP_FLAGS_EXTEND_DB = 1 << 3,
415};
416
417#define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
418
419/* CQE hdr */
420#define ERDMA_CQE_HDR_OWNER_MASK BIT(31)
421#define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)
422#define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)
423#define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)
424
425#define ERDMA_CQE_QTYPE_SQ 0
426#define ERDMA_CQE_QTYPE_RQ 1
427#define ERDMA_CQE_QTYPE_CMDQ 2
428
429struct erdma_cqe {
430 __be32 hdr;
431 __be32 qe_idx;
432 __be32 qpn;
433 union {
434 __le32 imm_data;
435 __be32 inv_rkey;
436 };
437 __be32 size;
438 __be32 rsvd[3];
439};
440
441struct erdma_sge {
442 __aligned_le64 addr;
443 __le32 length;
444 __le32 key;
445};
446
447/* Receive Queue Element */
448struct erdma_rqe {
449 __le16 qe_idx;
450 __le16 rsvd0;
451 __le32 qpn;
452 __le32 rsvd1;
453 __le32 rsvd2;
454 __le64 to;
455 __le32 length;
456 __le32 stag;
457};
458
459/* SQE */
460#define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)
461#define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
462#define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)
463#define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)
464#define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)
465#define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)
466#define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)
467#define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)
468#define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)
469#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
470
471/* REG MR attrs */
472#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
473#define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
474#define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
475
476struct erdma_write_sqe {
477 __le64 hdr;
478 __be32 imm_data;
479 __le32 length;
480
481 __le32 sink_stag;
482 __le32 sink_to_l;
483 __le32 sink_to_h;
484
485 __le32 rsvd;
486
487 struct erdma_sge sgl[];
488};
489
490struct erdma_send_sqe {
491 __le64 hdr;
492 union {
493 __be32 imm_data;
494 __le32 invalid_stag;
495 };
496
497 __le32 length;
498 struct erdma_sge sgl[];
499};
500
501struct erdma_readreq_sqe {
502 __le64 hdr;
503 __le32 invalid_stag;
504 __le32 length;
505 __le32 sink_stag;
506 __le32 sink_to_l;
507 __le32 sink_to_h;
508 __le32 rsvd;
509};
510
511struct erdma_atomic_sqe {
512 __le64 hdr;
513 __le64 rsvd;
514 __le64 fetchadd_swap_data;
515 __le64 cmp_data;
516
517 struct erdma_sge remote;
518 struct erdma_sge sgl;
519};
520
521struct erdma_reg_mr_sqe {
522 __le64 hdr;
523 __le64 addr;
524 __le32 length;
525 __le32 stag;
526 __le32 attrs;
527 __le32 rsvd;
528};
529
530/* EQ related. */
531#define ERDMA_DEFAULT_EQ_DEPTH 4096
532
533/* ceqe */
534#define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)
535#define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32)
536#define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31)
537#define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)
538
539/* aeqe */
540#define ERDMA_AEQE_HDR_O_MASK BIT(31)
541#define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16)
542#define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)
543
544#define ERDMA_AE_TYPE_QP_FATAL_EVENT 0
545#define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1
546#define ERDMA_AE_TYPE_ACC_ERR_EVENT 2
547#define ERDMA_AE_TYPE_CQ_ERR 3
548#define ERDMA_AE_TYPE_OTHER_ERROR 4
549
550struct erdma_aeqe {
551 __le32 hdr;
552 __le32 event_data0;
553 __le32 event_data1;
554 __le32 rsvd;
555};
556
557enum erdma_opcode {
558 ERDMA_OP_WRITE = 0,
559 ERDMA_OP_READ = 1,
560 ERDMA_OP_SEND = 2,
561 ERDMA_OP_SEND_WITH_IMM = 3,
562
563 ERDMA_OP_RECEIVE = 4,
564 ERDMA_OP_RECV_IMM = 5,
565 ERDMA_OP_RECV_INV = 6,
566
567 ERDMA_OP_RSVD0 = 7,
568 ERDMA_OP_RSVD1 = 8,
569 ERDMA_OP_WRITE_WITH_IMM = 9,
570
571 ERDMA_OP_RSVD2 = 10,
572 ERDMA_OP_RSVD3 = 11,
573
574 ERDMA_OP_RSP_SEND_IMM = 12,
575 ERDMA_OP_SEND_WITH_INV = 13,
576
577 ERDMA_OP_REG_MR = 14,
578 ERDMA_OP_LOCAL_INV = 15,
579 ERDMA_OP_READ_WITH_INV = 16,
580 ERDMA_OP_ATOMIC_CAS = 17,
581 ERDMA_OP_ATOMIC_FAA = 18,
582 ERDMA_NUM_OPCODES = 19,
583 ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
584};
585
586enum erdma_wc_status {
587 ERDMA_WC_SUCCESS = 0,
588 ERDMA_WC_GENERAL_ERR = 1,
589 ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
590 ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
591 ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
592 ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
593 ERDMA_WC_RECV_PDID_ERR = 6,
594 ERDMA_WC_RECV_WARRPING_ERR = 7,
595 ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
596 ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
597 ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
598 ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
599 ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
600 ERDMA_WC_SEND_PDID_ERR = 13,
601 ERDMA_WC_SEND_WARRPING_ERR = 14,
602 ERDMA_WC_FLUSH_ERR = 15,
603 ERDMA_WC_RETRY_EXC_ERR = 16,
604 ERDMA_NUM_WC_STATUS
605};
606
607enum erdma_vendor_err {
608 ERDMA_WC_VENDOR_NO_ERR = 0,
609 ERDMA_WC_VENDOR_INVALID_RQE = 1,
610 ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
611 ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
612 ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
613 ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
614 ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
615 ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
616 ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
617 ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
618 ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
619 ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
620 ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
621 ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
622};
623
624#endif
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2
3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4/* Kai Shen <kaishen@linux.alibaba.com> */
5/* Copyright (c) 2020-2022, Alibaba Group. */
6
7#ifndef __ERDMA_HW_H__
8#define __ERDMA_HW_H__
9
10#include <linux/kernel.h>
11#include <linux/types.h>
12
13/* PCIe device related definition. */
14#define PCI_VENDOR_ID_ALIBABA 0x1ded
15
16#define ERDMA_PCI_WIDTH 64
17#define ERDMA_FUNC_BAR 0
18#define ERDMA_MISX_BAR 2
19
20#define ERDMA_BAR_MASK (BIT(ERDMA_FUNC_BAR) | BIT(ERDMA_MISX_BAR))
21
22/* MSI-X related. */
23#define ERDMA_NUM_MSIX_VEC 32U
24#define ERDMA_MSIX_VECTOR_CMDQ 0
25
26/* PCIe Bar0 Registers. */
27#define ERDMA_REGS_VERSION_REG 0x0
28#define ERDMA_REGS_DEV_CTRL_REG 0x10
29#define ERDMA_REGS_DEV_ST_REG 0x14
30#define ERDMA_REGS_NETDEV_MAC_L_REG 0x18
31#define ERDMA_REGS_NETDEV_MAC_H_REG 0x1C
32#define ERDMA_REGS_CMDQ_SQ_ADDR_L_REG 0x20
33#define ERDMA_REGS_CMDQ_SQ_ADDR_H_REG 0x24
34#define ERDMA_REGS_CMDQ_CQ_ADDR_L_REG 0x28
35#define ERDMA_REGS_CMDQ_CQ_ADDR_H_REG 0x2C
36#define ERDMA_REGS_CMDQ_DEPTH_REG 0x30
37#define ERDMA_REGS_CMDQ_EQ_DEPTH_REG 0x34
38#define ERDMA_REGS_CMDQ_EQ_ADDR_L_REG 0x38
39#define ERDMA_REGS_CMDQ_EQ_ADDR_H_REG 0x3C
40#define ERDMA_REGS_AEQ_ADDR_L_REG 0x40
41#define ERDMA_REGS_AEQ_ADDR_H_REG 0x44
42#define ERDMA_REGS_AEQ_DEPTH_REG 0x48
43#define ERDMA_REGS_GRP_NUM_REG 0x4c
44#define ERDMA_REGS_AEQ_DB_REG 0x50
45#define ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG 0x60
46#define ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG 0x68
47#define ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG 0x70
48#define ERDMA_AEQ_DB_HOST_ADDR_REG 0x78
49#define ERDMA_REGS_STATS_TSO_IN_PKTS_REG 0x80
50#define ERDMA_REGS_STATS_TSO_OUT_PKTS_REG 0x88
51#define ERDMA_REGS_STATS_TSO_OUT_BYTES_REG 0x90
52#define ERDMA_REGS_STATS_TX_DROP_PKTS_REG 0x98
53#define ERDMA_REGS_STATS_TX_BPS_METER_DROP_PKTS_REG 0xa0
54#define ERDMA_REGS_STATS_TX_PPS_METER_DROP_PKTS_REG 0xa8
55#define ERDMA_REGS_STATS_RX_PKTS_REG 0xc0
56#define ERDMA_REGS_STATS_RX_BYTES_REG 0xc8
57#define ERDMA_REGS_STATS_RX_DROP_PKTS_REG 0xd0
58#define ERDMA_REGS_STATS_RX_BPS_METER_DROP_PKTS_REG 0xd8
59#define ERDMA_REGS_STATS_RX_PPS_METER_DROP_PKTS_REG 0xe0
60#define ERDMA_REGS_CEQ_DB_BASE_REG 0x100
61#define ERDMA_CMDQ_SQDB_REG 0x200
62#define ERDMA_CMDQ_CQDB_REG 0x300
63
64/* DEV_CTRL_REG details. */
65#define ERDMA_REG_DEV_CTRL_RESET_MASK 0x00000001
66#define ERDMA_REG_DEV_CTRL_INIT_MASK 0x00000002
67
68/* DEV_ST_REG details. */
69#define ERDMA_REG_DEV_ST_RESET_DONE_MASK 0x00000001U
70#define ERDMA_REG_DEV_ST_INIT_DONE_MASK 0x00000002U
71
72/* eRDMA PCIe DBs definition. */
73#define ERDMA_BAR_DB_SPACE_BASE 4096
74
75#define ERDMA_BAR_SQDB_SPACE_OFFSET ERDMA_BAR_DB_SPACE_BASE
76#define ERDMA_BAR_SQDB_SPACE_SIZE (384 * 1024)
77
78#define ERDMA_BAR_RQDB_SPACE_OFFSET \
79 (ERDMA_BAR_SQDB_SPACE_OFFSET + ERDMA_BAR_SQDB_SPACE_SIZE)
80#define ERDMA_BAR_RQDB_SPACE_SIZE (96 * 1024)
81
82#define ERDMA_BAR_CQDB_SPACE_OFFSET \
83 (ERDMA_BAR_RQDB_SPACE_OFFSET + ERDMA_BAR_RQDB_SPACE_SIZE)
84
85/* Doorbell page resources related. */
86/*
87 * Max # of parallelly issued directSQE is 3072 per device,
88 * hardware organizes this into 24 group, per group has 128 credits.
89 */
90#define ERDMA_DWQE_MAX_GRP_CNT 24
91#define ERDMA_DWQE_NUM_PER_GRP 128
92
93#define ERDMA_DWQE_TYPE0_CNT 64
94#define ERDMA_DWQE_TYPE1_CNT 496
95/* type1 DB contains 2 DBs, takes 256Byte. */
96#define ERDMA_DWQE_TYPE1_CNT_PER_PAGE 16
97
98#define ERDMA_SDB_SHARED_PAGE_INDEX 95
99
100/* Doorbell related. */
101#define ERDMA_DB_SIZE 8
102
103#define ERDMA_CQDB_IDX_MASK GENMASK_ULL(63, 56)
104#define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)
105#define ERDMA_CQDB_ARM_MASK BIT_ULL(31)
106#define ERDMA_CQDB_SOL_MASK BIT_ULL(30)
107#define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)
108#define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)
109
110#define ERDMA_EQDB_ARM_MASK BIT(31)
111#define ERDMA_EQDB_CI_MASK GENMASK_ULL(23, 0)
112
113#define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
114
115/* WQE related. */
116#define EQE_SIZE 16
117#define EQE_SHIFT 4
118#define RQE_SIZE 32
119#define RQE_SHIFT 5
120#define CQE_SIZE 32
121#define CQE_SHIFT 5
122#define SQEBB_SIZE 32
123#define SQEBB_SHIFT 5
124#define SQEBB_MASK (~(SQEBB_SIZE - 1))
125#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
126#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
127
128#define ERDMA_MAX_SQE_SIZE 128
129#define ERDMA_MAX_WQEBB_PER_SQE 4
130
131/* CMDQ related. */
132#define ERDMA_CMDQ_MAX_OUTSTANDING 128
133#define ERDMA_CMDQ_SQE_SIZE 64
134
135/* cmdq sub module definition. */
136enum CMDQ_WQE_SUB_MOD {
137 CMDQ_SUBMOD_RDMA = 0,
138 CMDQ_SUBMOD_COMMON = 1
139};
140
141enum CMDQ_RDMA_OPCODE {
142 CMDQ_OPCODE_QUERY_DEVICE = 0,
143 CMDQ_OPCODE_CREATE_QP = 1,
144 CMDQ_OPCODE_DESTROY_QP = 2,
145 CMDQ_OPCODE_MODIFY_QP = 3,
146 CMDQ_OPCODE_CREATE_CQ = 4,
147 CMDQ_OPCODE_DESTROY_CQ = 5,
148 CMDQ_OPCODE_REFLUSH = 6,
149 CMDQ_OPCODE_REG_MR = 8,
150 CMDQ_OPCODE_DEREG_MR = 9
151};
152
153enum CMDQ_COMMON_OPCODE {
154 CMDQ_OPCODE_CREATE_EQ = 0,
155 CMDQ_OPCODE_DESTROY_EQ = 1,
156 CMDQ_OPCODE_QUERY_FW_INFO = 2,
157 CMDQ_OPCODE_CONF_MTU = 3,
158};
159
160/* cmdq-SQE HDR */
161#define ERDMA_CMD_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
162#define ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK GENMASK_ULL(47, 32)
163#define ERDMA_CMD_HDR_SUB_MOD_MASK GENMASK_ULL(25, 24)
164#define ERDMA_CMD_HDR_OPCODE_MASK GENMASK_ULL(23, 16)
165#define ERDMA_CMD_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
166
167struct erdma_cmdq_destroy_cq_req {
168 u64 hdr;
169 u32 cqn;
170};
171
172#define ERDMA_EQ_TYPE_AEQ 0
173#define ERDMA_EQ_TYPE_CEQ 1
174
175struct erdma_cmdq_create_eq_req {
176 u64 hdr;
177 u64 qbuf_addr;
178 u8 vector_idx;
179 u8 eqn;
180 u8 depth;
181 u8 qtype;
182 u32 db_dma_addr_l;
183 u32 db_dma_addr_h;
184};
185
186struct erdma_cmdq_destroy_eq_req {
187 u64 hdr;
188 u64 rsvd0;
189 u8 vector_idx;
190 u8 eqn;
191 u8 rsvd1;
192 u8 qtype;
193};
194
195struct erdma_cmdq_config_mtu_req {
196 u64 hdr;
197 u32 mtu;
198};
199
200/* create_cq cfg0 */
201#define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24)
202#define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20)
203#define ERDMA_CMD_CREATE_CQ_CQN_MASK GENMASK(19, 0)
204
205/* create_cq cfg1 */
206#define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16)
207#define ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK BIT(15)
208#define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)
209
210struct erdma_cmdq_create_cq_req {
211 u64 hdr;
212 u32 cfg0;
213 u32 qbuf_addr_l;
214 u32 qbuf_addr_h;
215 u32 cfg1;
216 u64 cq_db_info_addr;
217 u32 first_page_offset;
218};
219
220/* regmr/deregmr cfg0 */
221#define ERDMA_CMD_MR_VALID_MASK BIT(31)
222#define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)
223#define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)
224
225/* regmr cfg1 */
226#define ERDMA_CMD_REGMR_PD_MASK GENMASK(31, 12)
227#define ERDMA_CMD_REGMR_TYPE_MASK GENMASK(7, 6)
228#define ERDMA_CMD_REGMR_RIGHT_MASK GENMASK(5, 1)
229
230/* regmr cfg2 */
231#define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27)
232#define ERDMA_CMD_REGMR_MTT_TYPE_MASK GENMASK(21, 20)
233#define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)
234
235struct erdma_cmdq_reg_mr_req {
236 u64 hdr;
237 u32 cfg0;
238 u32 cfg1;
239 u64 start_va;
240 u32 size;
241 u32 cfg2;
242 u64 phy_addr[4];
243};
244
245struct erdma_cmdq_dereg_mr_req {
246 u64 hdr;
247 u32 cfg;
248};
249
250/* modify qp cfg */
251#define ERDMA_CMD_MODIFY_QP_STATE_MASK GENMASK(31, 24)
252#define ERDMA_CMD_MODIFY_QP_CC_MASK GENMASK(23, 20)
253#define ERDMA_CMD_MODIFY_QP_QPN_MASK GENMASK(19, 0)
254
255struct erdma_cmdq_modify_qp_req {
256 u64 hdr;
257 u32 cfg;
258 u32 cookie;
259 __be32 dip;
260 __be32 sip;
261 __be16 sport;
262 __be16 dport;
263 u32 send_nxt;
264 u32 recv_nxt;
265};
266
267/* create qp cfg0 */
268#define ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK GENMASK(31, 20)
269#define ERDMA_CMD_CREATE_QP_QPN_MASK GENMASK(19, 0)
270
271/* create qp cfg1 */
272#define ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK GENMASK(31, 20)
273#define ERDMA_CMD_CREATE_QP_PD_MASK GENMASK(19, 0)
274
275/* create qp cqn_mtt_cfg */
276#define ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK GENMASK(31, 28)
277#define ERDMA_CMD_CREATE_QP_CQN_MASK GENMASK(23, 0)
278
279/* create qp mtt_cfg */
280#define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)
281#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1)
282#define ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK BIT(0)
283
284#define ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK GENMASK_ULL(31, 0)
285
286struct erdma_cmdq_create_qp_req {
287 u64 hdr;
288 u32 cfg0;
289 u32 cfg1;
290 u32 sq_cqn_mtt_cfg;
291 u32 rq_cqn_mtt_cfg;
292 u64 sq_buf_addr;
293 u64 rq_buf_addr;
294 u32 sq_mtt_cfg;
295 u32 rq_mtt_cfg;
296 u64 sq_db_info_dma_addr;
297 u64 rq_db_info_dma_addr;
298};
299
300struct erdma_cmdq_destroy_qp_req {
301 u64 hdr;
302 u32 qpn;
303};
304
305struct erdma_cmdq_reflush_req {
306 u64 hdr;
307 u32 qpn;
308 u32 sq_pi;
309 u32 rq_pi;
310};
311
312/* cap qword 0 definition */
313#define ERDMA_CMD_DEV_CAP_MAX_CQE_MASK GENMASK_ULL(47, 40)
314#define ERDMA_CMD_DEV_CAP_FLAGS_MASK GENMASK_ULL(31, 24)
315#define ERDMA_CMD_DEV_CAP_MAX_RECV_WR_MASK GENMASK_ULL(23, 16)
316#define ERDMA_CMD_DEV_CAP_MAX_MR_SIZE_MASK GENMASK_ULL(7, 0)
317
318/* cap qword 1 definition */
319#define ERDMA_CMD_DEV_CAP_DMA_LOCAL_KEY_MASK GENMASK_ULL(63, 32)
320#define ERDMA_CMD_DEV_CAP_DEFAULT_CC_MASK GENMASK_ULL(31, 28)
321#define ERDMA_CMD_DEV_CAP_QBLOCK_MASK GENMASK_ULL(27, 16)
322#define ERDMA_CMD_DEV_CAP_MAX_MW_MASK GENMASK_ULL(7, 0)
323
324#define ERDMA_NQP_PER_QBLOCK 1024
325
326enum {
327 ERDMA_DEV_CAP_FLAGS_ATOMIC = 1 << 7,
328};
329
330#define ERDMA_CMD_INFO0_FW_VER_MASK GENMASK_ULL(31, 0)
331
332/* CQE hdr */
333#define ERDMA_CQE_HDR_OWNER_MASK BIT(31)
334#define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)
335#define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)
336#define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)
337
338#define ERDMA_CQE_QTYPE_SQ 0
339#define ERDMA_CQE_QTYPE_RQ 1
340#define ERDMA_CQE_QTYPE_CMDQ 2
341
342struct erdma_cqe {
343 __be32 hdr;
344 __be32 qe_idx;
345 __be32 qpn;
346 union {
347 __le32 imm_data;
348 __be32 inv_rkey;
349 };
350 __be32 size;
351 __be32 rsvd[3];
352};
353
354struct erdma_sge {
355 __aligned_le64 addr;
356 __le32 length;
357 __le32 key;
358};
359
360/* Receive Queue Element */
361struct erdma_rqe {
362 __le16 qe_idx;
363 __le16 rsvd0;
364 __le32 qpn;
365 __le32 rsvd1;
366 __le32 rsvd2;
367 __le64 to;
368 __le32 length;
369 __le32 stag;
370};
371
372/* SQE */
373#define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)
374#define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
375#define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)
376#define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)
377#define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)
378#define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)
379#define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)
380#define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)
381#define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)
382#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
383
384/* REG MR attrs */
385#define ERDMA_SQE_MR_ACCESS_MASK GENMASK(5, 1)
386#define ERDMA_SQE_MR_MTT_TYPE_MASK GENMASK(7, 6)
387#define ERDMA_SQE_MR_MTT_CNT_MASK GENMASK(31, 12)
388
389struct erdma_write_sqe {
390 __le64 hdr;
391 __be32 imm_data;
392 __le32 length;
393
394 __le32 sink_stag;
395 __le32 sink_to_l;
396 __le32 sink_to_h;
397
398 __le32 rsvd;
399
400 struct erdma_sge sgl[0];
401};
402
403struct erdma_send_sqe {
404 __le64 hdr;
405 union {
406 __be32 imm_data;
407 __le32 invalid_stag;
408 };
409
410 __le32 length;
411 struct erdma_sge sgl[0];
412};
413
414struct erdma_readreq_sqe {
415 __le64 hdr;
416 __le32 invalid_stag;
417 __le32 length;
418 __le32 sink_stag;
419 __le32 sink_to_l;
420 __le32 sink_to_h;
421 __le32 rsvd;
422};
423
424struct erdma_atomic_sqe {
425 __le64 hdr;
426 __le64 rsvd;
427 __le64 fetchadd_swap_data;
428 __le64 cmp_data;
429
430 struct erdma_sge remote;
431 struct erdma_sge sgl;
432};
433
434struct erdma_reg_mr_sqe {
435 __le64 hdr;
436 __le64 addr;
437 __le32 length;
438 __le32 stag;
439 __le32 attrs;
440 __le32 rsvd;
441};
442
443/* EQ related. */
444#define ERDMA_DEFAULT_EQ_DEPTH 256
445
446/* ceqe */
447#define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)
448#define ERDMA_CEQE_HDR_PI_MASK GENMASK_ULL(55, 32)
449#define ERDMA_CEQE_HDR_O_MASK BIT_ULL(31)
450#define ERDMA_CEQE_HDR_CQN_MASK GENMASK_ULL(19, 0)
451
452/* aeqe */
453#define ERDMA_AEQE_HDR_O_MASK BIT(31)
454#define ERDMA_AEQE_HDR_TYPE_MASK GENMASK(23, 16)
455#define ERDMA_AEQE_HDR_SUBTYPE_MASK GENMASK(7, 0)
456
457#define ERDMA_AE_TYPE_QP_FATAL_EVENT 0
458#define ERDMA_AE_TYPE_QP_ERQ_ERR_EVENT 1
459#define ERDMA_AE_TYPE_ACC_ERR_EVENT 2
460#define ERDMA_AE_TYPE_CQ_ERR 3
461#define ERDMA_AE_TYPE_OTHER_ERROR 4
462
463struct erdma_aeqe {
464 __le32 hdr;
465 __le32 event_data0;
466 __le32 event_data1;
467 __le32 rsvd;
468};
469
470enum erdma_opcode {
471 ERDMA_OP_WRITE = 0,
472 ERDMA_OP_READ = 1,
473 ERDMA_OP_SEND = 2,
474 ERDMA_OP_SEND_WITH_IMM = 3,
475
476 ERDMA_OP_RECEIVE = 4,
477 ERDMA_OP_RECV_IMM = 5,
478 ERDMA_OP_RECV_INV = 6,
479
480 ERDMA_OP_RSVD0 = 7,
481 ERDMA_OP_RSVD1 = 8,
482 ERDMA_OP_WRITE_WITH_IMM = 9,
483
484 ERDMA_OP_RSVD2 = 10,
485 ERDMA_OP_RSVD3 = 11,
486
487 ERDMA_OP_RSP_SEND_IMM = 12,
488 ERDMA_OP_SEND_WITH_INV = 13,
489
490 ERDMA_OP_REG_MR = 14,
491 ERDMA_OP_LOCAL_INV = 15,
492 ERDMA_OP_READ_WITH_INV = 16,
493 ERDMA_OP_ATOMIC_CAS = 17,
494 ERDMA_OP_ATOMIC_FAD = 18,
495 ERDMA_NUM_OPCODES = 19,
496 ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
497};
498
499enum erdma_wc_status {
500 ERDMA_WC_SUCCESS = 0,
501 ERDMA_WC_GENERAL_ERR = 1,
502 ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
503 ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
504 ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
505 ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
506 ERDMA_WC_RECV_PDID_ERR = 6,
507 ERDMA_WC_RECV_WARRPING_ERR = 7,
508 ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
509 ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
510 ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
511 ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
512 ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
513 ERDMA_WC_SEND_PDID_ERR = 13,
514 ERDMA_WC_SEND_WARRPING_ERR = 14,
515 ERDMA_WC_FLUSH_ERR = 15,
516 ERDMA_WC_RETRY_EXC_ERR = 16,
517 ERDMA_NUM_WC_STATUS
518};
519
520enum erdma_vendor_err {
521 ERDMA_WC_VENDOR_NO_ERR = 0,
522 ERDMA_WC_VENDOR_INVALID_RQE = 1,
523 ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
524 ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
525 ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
526 ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
527 ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
528 ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
529 ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
530 ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
531 ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
532 ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
533 ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
534 ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
535};
536
537#endif