Loading...
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
2/*
3 * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
4 */
5
6#ifndef _EFA_IO_H_
7#define _EFA_IO_H_
8
9#define EFA_IO_TX_DESC_NUM_BUFS 2
10#define EFA_IO_TX_DESC_NUM_RDMA_BUFS 1
11#define EFA_IO_TX_DESC_INLINE_MAX_SIZE 32
12#define EFA_IO_TX_DESC_IMM_DATA_SIZE 4
13
14enum efa_io_queue_type {
15 /* send queue (of a QP) */
16 EFA_IO_SEND_QUEUE = 1,
17 /* recv queue (of a QP) */
18 EFA_IO_RECV_QUEUE = 2,
19};
20
21enum efa_io_send_op_type {
22 /* send message */
23 EFA_IO_SEND = 0,
24 /* RDMA read */
25 EFA_IO_RDMA_READ = 1,
26 /* RDMA write */
27 EFA_IO_RDMA_WRITE = 2,
28};
29
30enum efa_io_comp_status {
31 /* Successful completion */
32 EFA_IO_COMP_STATUS_OK = 0,
33 /* Flushed during QP destroy */
34 EFA_IO_COMP_STATUS_FLUSHED = 1,
35 /* Internal QP error */
36 EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,
37 /* Bad operation type */
38 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_OP_TYPE = 3,
39 /* Bad AH */
40 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH = 4,
41 /* LKEY not registered or does not match IOVA */
42 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,
43 /* Message too long */
44 EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH = 6,
45 /* Destination ENI is down or does not run EFA */
46 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,
47 /* Connection was reset by remote side */
48 EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT = 8,
49 /* Bad dest QP number (QP does not exist or is in error state) */
50 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,
51 /* Destination resource not ready (no WQEs posted on RQ) */
52 EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR = 10,
53 /* Receiver SGL too short */
54 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH = 11,
55 /* Unexpected status returned by responder */
56 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS = 12,
57 /* Unresponsive remote - detected locally */
58 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,
59};
60
61struct efa_io_tx_meta_desc {
62 /* Verbs-generated Request ID */
63 u16 req_id;
64
65 /*
66 * control flags
67 * 3:0 : op_type - enum efa_io_send_op_type
68 * 4 : has_imm - immediate_data field carries valid
69 * data.
70 * 5 : inline_msg - inline mode - inline message data
71 * follows this descriptor (no buffer descriptors).
72 * Note that it is different from immediate data
73 * 6 : meta_extension - Extended metadata. MBZ
74 * 7 : meta_desc - Indicates metadata descriptor.
75 * Must be set.
76 */
77 u8 ctrl1;
78
79 /*
80 * control flags
81 * 0 : phase
82 * 1 : reserved25 - MBZ
83 * 2 : first - Indicates first descriptor in
84 * transaction. Must be set.
85 * 3 : last - Indicates last descriptor in
86 * transaction. Must be set.
87 * 4 : comp_req - Indicates whether completion should
88 * be posted, after packet is transmitted. Valid only
89 * for the first descriptor
90 * 7:5 : reserved29 - MBZ
91 */
92 u8 ctrl2;
93
94 u16 dest_qp_num;
95
96 /*
97 * If inline_msg bit is set, length of inline message in bytes,
98 * otherwise length of SGL (number of buffers).
99 */
100 u16 length;
101
102 /*
103 * immediate data: if has_imm is set, then this field is included
104 * within Tx message and reported in remote Rx completion.
105 */
106 u32 immediate_data;
107
108 u16 ah;
109
110 u16 reserved;
111
112 /* Queue key */
113 u32 qkey;
114
115 u8 reserved2[12];
116};
117
118/*
119 * Tx queue buffer descriptor, for any transport type. Preceded by metadata
120 * descriptor.
121 */
122struct efa_io_tx_buf_desc {
123 /* length in bytes */
124 u32 length;
125
126 /*
127 * 23:0 : lkey - local memory translation key
128 * 31:24 : reserved - MBZ
129 */
130 u32 lkey;
131
132 /* Buffer address bits[31:0] */
133 u32 buf_addr_lo;
134
135 /* Buffer address bits[63:32] */
136 u32 buf_addr_hi;
137};
138
139struct efa_io_remote_mem_addr {
140 /* length in bytes */
141 u32 length;
142
143 /* remote memory translation key */
144 u32 rkey;
145
146 /* Buffer address bits[31:0] */
147 u32 buf_addr_lo;
148
149 /* Buffer address bits[63:32] */
150 u32 buf_addr_hi;
151};
152
153struct efa_io_rdma_req {
154 /* Remote memory address */
155 struct efa_io_remote_mem_addr remote_mem;
156
157 /* Local memory address */
158 struct efa_io_tx_buf_desc local_mem[1];
159};
160
161/*
162 * Tx WQE, composed of tx meta descriptors followed by either tx buffer
163 * descriptors or inline data
164 */
165struct efa_io_tx_wqe {
166 /* TX meta */
167 struct efa_io_tx_meta_desc meta;
168
169 union {
170 /* Send buffer descriptors */
171 struct efa_io_tx_buf_desc sgl[2];
172
173 u8 inline_data[32];
174
175 /* RDMA local and remote memory addresses */
176 struct efa_io_rdma_req rdma_req;
177 } data;
178};
179
180/*
181 * Rx buffer descriptor; RX WQE is composed of one or more RX buffer
182 * descriptors.
183 */
184struct efa_io_rx_desc {
185 /* Buffer address bits[31:0] */
186 u32 buf_addr_lo;
187
188 /* Buffer Pointer[63:32] */
189 u32 buf_addr_hi;
190
191 /* Verbs-generated request id. */
192 u16 req_id;
193
194 /* Length in bytes. */
195 u16 length;
196
197 /*
198 * LKey and control flags
199 * 23:0 : lkey
200 * 29:24 : reserved - MBZ
201 * 30 : first - Indicates first descriptor in WQE
202 * 31 : last - Indicates last descriptor in WQE
203 */
204 u32 lkey_ctrl;
205};
206
207/* Common IO completion descriptor */
208struct efa_io_cdesc_common {
209 /*
210 * verbs-generated request ID, as provided in the completed tx or rx
211 * descriptor.
212 */
213 u16 req_id;
214
215 u8 status;
216
217 /*
218 * flags
219 * 0 : phase - Phase bit
220 * 2:1 : q_type - enum efa_io_queue_type: send/recv
221 * 3 : has_imm - indicates that immediate data is
222 * present - for RX completions only
223 * 6:4 : op_type - enum efa_io_send_op_type
224 * 7 : reserved31 - MBZ
225 */
226 u8 flags;
227
228 /* local QP number */
229 u16 qp_num;
230};
231
232/* Tx completion descriptor */
233struct efa_io_tx_cdesc {
234 /* Common completion info */
235 struct efa_io_cdesc_common common;
236
237 /* MBZ */
238 u16 reserved16;
239};
240
241/* Rx Completion Descriptor */
242struct efa_io_rx_cdesc {
243 /* Common completion info */
244 struct efa_io_cdesc_common common;
245
246 /* Transferred length bits[15:0] */
247 u16 length;
248
249 /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
250 u16 ah;
251
252 u16 src_qp_num;
253
254 /* Immediate data */
255 u32 imm;
256};
257
258/* Rx Completion Descriptor RDMA write info */
259struct efa_io_rx_cdesc_rdma_write {
260 /* Transferred length bits[31:16] */
261 u16 length_hi;
262};
263
264/* Extended Rx Completion Descriptor */
265struct efa_io_rx_cdesc_ex {
266 /* Base RX completion info */
267 struct efa_io_rx_cdesc base;
268
269 union {
270 struct efa_io_rx_cdesc_rdma_write rdma_write;
271
272 /*
273 * Valid only in case of unknown AH (0xFFFF) and CQ
274 * set_src_addr is enabled.
275 */
276 u8 src_addr[16];
277 } u;
278};
279
280/* tx_meta_desc */
281#define EFA_IO_TX_META_DESC_OP_TYPE_MASK GENMASK(3, 0)
282#define EFA_IO_TX_META_DESC_HAS_IMM_MASK BIT(4)
283#define EFA_IO_TX_META_DESC_INLINE_MSG_MASK BIT(5)
284#define EFA_IO_TX_META_DESC_META_EXTENSION_MASK BIT(6)
285#define EFA_IO_TX_META_DESC_META_DESC_MASK BIT(7)
286#define EFA_IO_TX_META_DESC_PHASE_MASK BIT(0)
287#define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2)
288#define EFA_IO_TX_META_DESC_LAST_MASK BIT(3)
289#define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4)
290
291/* tx_buf_desc */
292#define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0)
293
294/* rx_desc */
295#define EFA_IO_RX_DESC_LKEY_MASK GENMASK(23, 0)
296#define EFA_IO_RX_DESC_FIRST_MASK BIT(30)
297#define EFA_IO_RX_DESC_LAST_MASK BIT(31)
298
299/* cdesc_common */
300#define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0)
301#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1)
302#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3)
303#define EFA_IO_CDESC_COMMON_OP_TYPE_MASK GENMASK(6, 4)
304
305#endif /* _EFA_IO_H_ */
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
2/*
3 * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
4 */
5
6#ifndef _EFA_IO_H_
7#define _EFA_IO_H_
8
9#define EFA_IO_TX_DESC_NUM_BUFS 2
10#define EFA_IO_TX_DESC_NUM_RDMA_BUFS 1
11#define EFA_IO_TX_DESC_INLINE_MAX_SIZE 32
12#define EFA_IO_TX_DESC_IMM_DATA_SIZE 4
13
14enum efa_io_queue_type {
15 /* send queue (of a QP) */
16 EFA_IO_SEND_QUEUE = 1,
17 /* recv queue (of a QP) */
18 EFA_IO_RECV_QUEUE = 2,
19};
20
21enum efa_io_send_op_type {
22 /* send message */
23 EFA_IO_SEND = 0,
24 /* RDMA read */
25 EFA_IO_RDMA_READ = 1,
26 /* RDMA write */
27 EFA_IO_RDMA_WRITE = 2,
28};
29
30enum efa_io_comp_status {
31 /* Successful completion */
32 EFA_IO_COMP_STATUS_OK = 0,
33 /* Flushed during QP destroy */
34 EFA_IO_COMP_STATUS_FLUSHED = 1,
35 /* Internal QP error */
36 EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,
37 /* Bad operation type */
38 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_OP_TYPE = 3,
39 /* Bad AH */
40 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH = 4,
41 /* LKEY not registered or does not match IOVA */
42 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,
43 /* Message too long */
44 EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH = 6,
45 /* Destination ENI is down or does not run EFA */
46 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,
47 /* Connection was reset by remote side */
48 EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT = 8,
49 /* Bad dest QP number (QP does not exist or is in error state) */
50 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,
51 /* Destination resource not ready (no WQEs posted on RQ) */
52 EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR = 10,
53 /* Receiver SGL too short */
54 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH = 11,
55 /* Unexpected status returned by responder */
56 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS = 12,
57 /* Unresponsive remote - detected locally */
58 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,
59};
60
61struct efa_io_tx_meta_desc {
62 /* Verbs-generated Request ID */
63 u16 req_id;
64
65 /*
66 * control flags
67 * 3:0 : op_type - enum efa_io_send_op_type
68 * 4 : has_imm - immediate_data field carries valid
69 * data.
70 * 5 : inline_msg - inline mode - inline message data
71 * follows this descriptor (no buffer descriptors).
72 * Note that it is different from immediate data
73 * 6 : meta_extension - Extended metadata. MBZ
74 * 7 : meta_desc - Indicates metadata descriptor.
75 * Must be set.
76 */
77 u8 ctrl1;
78
79 /*
80 * control flags
81 * 0 : phase
82 * 1 : reserved25 - MBZ
83 * 2 : first - Indicates first descriptor in
84 * transaction. Must be set.
85 * 3 : last - Indicates last descriptor in
86 * transaction. Must be set.
87 * 4 : comp_req - Indicates whether completion should
88 * be posted, after packet is transmitted. Valid only
89 * for the first descriptor
90 * 7:5 : reserved29 - MBZ
91 */
92 u8 ctrl2;
93
94 u16 dest_qp_num;
95
96 /*
97 * If inline_msg bit is set, length of inline message in bytes,
98 * otherwise length of SGL (number of buffers).
99 */
100 u16 length;
101
102 /*
103 * immediate data: if has_imm is set, then this field is included
104 * within Tx message and reported in remote Rx completion.
105 */
106 u32 immediate_data;
107
108 u16 ah;
109
110 u16 reserved;
111
112 /* Queue key */
113 u32 qkey;
114
115 u8 reserved2[12];
116};
117
118/*
119 * Tx queue buffer descriptor, for any transport type. Preceded by metadata
120 * descriptor.
121 */
122struct efa_io_tx_buf_desc {
123 /* length in bytes */
124 u32 length;
125
126 /*
127 * 23:0 : lkey - local memory translation key
128 * 31:24 : reserved - MBZ
129 */
130 u32 lkey;
131
132 /* Buffer address bits[31:0] */
133 u32 buf_addr_lo;
134
135 /* Buffer address bits[63:32] */
136 u32 buf_addr_hi;
137};
138
139struct efa_io_remote_mem_addr {
140 /* length in bytes */
141 u32 length;
142
143 /* remote memory translation key */
144 u32 rkey;
145
146 /* Buffer address bits[31:0] */
147 u32 buf_addr_lo;
148
149 /* Buffer address bits[63:32] */
150 u32 buf_addr_hi;
151};
152
153struct efa_io_rdma_req {
154 /* Remote memory address */
155 struct efa_io_remote_mem_addr remote_mem;
156
157 /* Local memory address */
158 struct efa_io_tx_buf_desc local_mem[1];
159};
160
161/*
162 * Tx WQE, composed of tx meta descriptors followed by either tx buffer
163 * descriptors or inline data
164 */
165struct efa_io_tx_wqe {
166 /* TX meta */
167 struct efa_io_tx_meta_desc meta;
168
169 union {
170 /* Send buffer descriptors */
171 struct efa_io_tx_buf_desc sgl[2];
172
173 u8 inline_data[32];
174
175 /* RDMA local and remote memory addresses */
176 struct efa_io_rdma_req rdma_req;
177 } data;
178};
179
180/*
181 * Rx buffer descriptor; RX WQE is composed of one or more RX buffer
182 * descriptors.
183 */
184struct efa_io_rx_desc {
185 /* Buffer address bits[31:0] */
186 u32 buf_addr_lo;
187
188 /* Buffer Pointer[63:32] */
189 u32 buf_addr_hi;
190
191 /* Verbs-generated request id. */
192 u16 req_id;
193
194 /* Length in bytes. */
195 u16 length;
196
197 /*
198 * LKey and control flags
199 * 23:0 : lkey
200 * 29:24 : reserved - MBZ
201 * 30 : first - Indicates first descriptor in WQE
202 * 31 : last - Indicates last descriptor in WQE
203 */
204 u32 lkey_ctrl;
205};
206
207/* Common IO completion descriptor */
208struct efa_io_cdesc_common {
209 /*
210 * verbs-generated request ID, as provided in the completed tx or rx
211 * descriptor.
212 */
213 u16 req_id;
214
215 u8 status;
216
217 /*
218 * flags
219 * 0 : phase - Phase bit
220 * 2:1 : q_type - enum efa_io_queue_type: send/recv
221 * 3 : has_imm - indicates that immediate data is
222 * present - for RX completions only
223 * 6:4 : op_type - enum efa_io_send_op_type
224 * 7 : reserved31 - MBZ
225 */
226 u8 flags;
227
228 /* local QP number */
229 u16 qp_num;
230};
231
232/* Tx completion descriptor */
233struct efa_io_tx_cdesc {
234 /* Common completion info */
235 struct efa_io_cdesc_common common;
236
237 /* MBZ */
238 u16 reserved16;
239};
240
241/* Rx Completion Descriptor */
242struct efa_io_rx_cdesc {
243 /* Common completion info */
244 struct efa_io_cdesc_common common;
245
246 /* Transferred length bits[15:0] */
247 u16 length;
248
249 /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
250 u16 ah;
251
252 u16 src_qp_num;
253
254 /* Immediate data */
255 u32 imm;
256};
257
258/* Rx Completion Descriptor RDMA write info */
259struct efa_io_rx_cdesc_rdma_write {
260 /* Transferred length bits[31:16] */
261 u16 length_hi;
262};
263
264/* Extended Rx Completion Descriptor */
265struct efa_io_rx_cdesc_ex {
266 /* Base RX completion info */
267 struct efa_io_rx_cdesc base;
268
269 union {
270 struct efa_io_rx_cdesc_rdma_write rdma_write;
271
272 /*
273 * Valid only in case of unknown AH (0xFFFF) and CQ
274 * set_src_addr is enabled.
275 */
276 u8 src_addr[16];
277 } u;
278};
279
280/* tx_meta_desc */
281#define EFA_IO_TX_META_DESC_OP_TYPE_MASK GENMASK(3, 0)
282#define EFA_IO_TX_META_DESC_HAS_IMM_MASK BIT(4)
283#define EFA_IO_TX_META_DESC_INLINE_MSG_MASK BIT(5)
284#define EFA_IO_TX_META_DESC_META_EXTENSION_MASK BIT(6)
285#define EFA_IO_TX_META_DESC_META_DESC_MASK BIT(7)
286#define EFA_IO_TX_META_DESC_PHASE_MASK BIT(0)
287#define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2)
288#define EFA_IO_TX_META_DESC_LAST_MASK BIT(3)
289#define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4)
290
291/* tx_buf_desc */
292#define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0)
293
294/* rx_desc */
295#define EFA_IO_RX_DESC_LKEY_MASK GENMASK(23, 0)
296#define EFA_IO_RX_DESC_FIRST_MASK BIT(30)
297#define EFA_IO_RX_DESC_LAST_MASK BIT(31)
298
299/* cdesc_common */
300#define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0)
301#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1)
302#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3)
303#define EFA_IO_CDESC_COMMON_OP_TYPE_MASK GENMASK(6, 4)
304
305#endif /* _EFA_IO_H_ */