Loading...
Note: File does not exist in v3.1.
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Connection Data Control (CDC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12#ifndef SMC_CDC_H
13#define SMC_CDC_H
14
15#include <linux/kernel.h> /* max_t */
16#include <linux/atomic.h>
17#include <linux/in.h>
18#include <linux/compiler.h>
19
20#include "smc.h"
21#include "smc_core.h"
22#include "smc_wr.h"
23
24#define SMC_CDC_MSG_TYPE 0xFE
25
26/* in network byte order */
27union smc_cdc_cursor { /* SMC cursor */
28 struct {
29 __be16 reserved;
30 __be16 wrap;
31 __be32 count;
32 };
33#ifdef KERNEL_HAS_ATOMIC64
34 atomic64_t acurs; /* for atomic processing */
35#else
36 u64 acurs; /* for atomic processing */
37#endif
38} __aligned(8);
39
40/* in network byte order */
41struct smc_cdc_msg {
42 struct smc_wr_rx_hdr common; /* .type = 0xFE */
43 u8 len; /* 44 */
44 __be16 seqno;
45 __be32 token;
46 union smc_cdc_cursor prod;
47 union smc_cdc_cursor cons; /* piggy backed "ack" */
48 struct smc_cdc_producer_flags prod_flags;
49 struct smc_cdc_conn_state_flags conn_state_flags;
50 u8 reserved[18];
51};
52
53/* SMC-D cursor format */
54union smcd_cdc_cursor {
55 struct {
56 u16 wrap;
57 u32 count;
58 struct smc_cdc_producer_flags prod_flags;
59 struct smc_cdc_conn_state_flags conn_state_flags;
60 } __packed;
61#ifdef KERNEL_HAS_ATOMIC64
62 atomic64_t acurs; /* for atomic processing */
63#else
64 u64 acurs; /* for atomic processing */
65#endif
66} __aligned(8);
67
68/* CDC message for SMC-D */
69struct smcd_cdc_msg {
70 struct smc_wr_rx_hdr common; /* Type = 0xFE */
71 u8 res1[7];
72 union smcd_cdc_cursor prod;
73 union smcd_cdc_cursor cons;
74 u8 res3[8];
75} __aligned(8);
76
77static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
78{
79 return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
80 conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
81}
82
83static inline bool smc_cdc_rxed_any_close_or_senddone(
84 struct smc_connection *conn)
85{
86 return smc_cdc_rxed_any_close(conn) ||
87 conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
88}
89
90static inline void smc_curs_add(int size, union smc_host_cursor *curs,
91 int value)
92{
93 curs->count += value;
94 if (curs->count >= size) {
95 curs->wrap++;
96 curs->count -= size;
97 }
98}
99
100/* Copy cursor src into tgt */
101static inline void smc_curs_copy(union smc_host_cursor *tgt,
102 union smc_host_cursor *src,
103 struct smc_connection *conn)
104{
105#ifndef KERNEL_HAS_ATOMIC64
106 unsigned long flags;
107
108 spin_lock_irqsave(&conn->acurs_lock, flags);
109 tgt->acurs = src->acurs;
110 spin_unlock_irqrestore(&conn->acurs_lock, flags);
111#else
112 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
113#endif
114}
115
116static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
117 union smc_cdc_cursor *src,
118 struct smc_connection *conn)
119{
120#ifndef KERNEL_HAS_ATOMIC64
121 unsigned long flags;
122
123 spin_lock_irqsave(&conn->acurs_lock, flags);
124 tgt->acurs = src->acurs;
125 spin_unlock_irqrestore(&conn->acurs_lock, flags);
126#else
127 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
128#endif
129}
130
131static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
132 union smcd_cdc_cursor *src,
133 struct smc_connection *conn)
134{
135#ifndef KERNEL_HAS_ATOMIC64
136 unsigned long flags;
137
138 spin_lock_irqsave(&conn->acurs_lock, flags);
139 tgt->acurs = src->acurs;
140 spin_unlock_irqrestore(&conn->acurs_lock, flags);
141#else
142 atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
143#endif
144}
145
146/* calculate cursor difference between old and new, where old <= new and
147 * difference cannot exceed size
148 */
149static inline int smc_curs_diff(unsigned int size,
150 union smc_host_cursor *old,
151 union smc_host_cursor *new)
152{
153 if (old->wrap != new->wrap)
154 return max_t(int, 0,
155 ((size - old->count) + new->count));
156
157 return max_t(int, 0, (new->count - old->count));
158}
159
160/* calculate cursor difference between old and new - returns negative
161 * value in case old > new
162 */
163static inline int smc_curs_comp(unsigned int size,
164 union smc_host_cursor *old,
165 union smc_host_cursor *new)
166{
167 if (old->wrap > new->wrap ||
168 (old->wrap == new->wrap && old->count > new->count))
169 return -smc_curs_diff(size, new, old);
170 return smc_curs_diff(size, old, new);
171}
172
173/* calculate cursor difference between old and new, where old <= new and
174 * difference may exceed size
175 */
176static inline int smc_curs_diff_large(unsigned int size,
177 union smc_host_cursor *old,
178 union smc_host_cursor *new)
179{
180 if (old->wrap < new->wrap)
181 return min_t(int,
182 (size - old->count) + new->count +
183 (new->wrap - old->wrap - 1) * size,
184 size);
185
186 if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
187 return min_t(int,
188 (size - old->count) + new->count +
189 (new->wrap + 0xffff - old->wrap) * size,
190 size);
191
192 return max_t(int, 0, (new->count - old->count));
193}
194
195static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
196 union smc_host_cursor *local,
197 union smc_host_cursor *save,
198 struct smc_connection *conn)
199{
200 smc_curs_copy(save, local, conn);
201 peer->count = htonl(save->count);
202 peer->wrap = htons(save->wrap);
203 /* peer->reserved = htons(0); must be ensured by caller */
204}
205
206static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
207 struct smc_connection *conn,
208 union smc_host_cursor *save)
209{
210 struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
211
212 peer->common.type = local->common.type;
213 peer->len = local->len;
214 peer->seqno = htons(local->seqno);
215 peer->token = htonl(local->token);
216 smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
217 smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
218 peer->prod_flags = local->prod_flags;
219 peer->conn_state_flags = local->conn_state_flags;
220}
221
222static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
223 union smc_cdc_cursor *peer,
224 struct smc_connection *conn)
225{
226 union smc_host_cursor temp, old;
227 union smc_cdc_cursor net;
228
229 smc_curs_copy(&old, local, conn);
230 smc_curs_copy_net(&net, peer, conn);
231 temp.count = ntohl(net.count);
232 temp.wrap = ntohs(net.wrap);
233 if ((old.wrap > temp.wrap) && temp.wrap)
234 return;
235 if ((old.wrap == temp.wrap) &&
236 (old.count > temp.count))
237 return;
238 smc_curs_copy(local, &temp, conn);
239}
240
241static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
242 struct smc_cdc_msg *peer,
243 struct smc_connection *conn)
244{
245 local->common.type = peer->common.type;
246 local->len = peer->len;
247 local->seqno = ntohs(peer->seqno);
248 local->token = ntohl(peer->token);
249 smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
250 smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
251 local->prod_flags = peer->prod_flags;
252 local->conn_state_flags = peer->conn_state_flags;
253}
254
255static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
256 struct smcd_cdc_msg *peer,
257 struct smc_connection *conn)
258{
259 union smc_host_cursor temp;
260
261 temp.wrap = peer->prod.wrap;
262 temp.count = peer->prod.count;
263 smc_curs_copy(&local->prod, &temp, conn);
264
265 temp.wrap = peer->cons.wrap;
266 temp.count = peer->cons.count;
267 smc_curs_copy(&local->cons, &temp, conn);
268 local->prod_flags = peer->cons.prod_flags;
269 local->conn_state_flags = peer->cons.conn_state_flags;
270}
271
272static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
273 struct smc_cdc_msg *peer,
274 struct smc_connection *conn)
275{
276 if (conn->lgr->is_smcd)
277 smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
278 else
279 smcr_cdc_msg_to_host(local, peer, conn);
280}
281
282struct smc_cdc_tx_pend {
283 struct smc_connection *conn; /* socket connection */
284 union smc_host_cursor cursor; /* tx sndbuf cursor sent */
285 union smc_host_cursor p_cursor; /* rx RMBE cursor produced */
286 u16 ctrl_seq; /* conn. tx sequence # */
287};
288
289int smc_cdc_get_free_slot(struct smc_connection *conn,
290 struct smc_link *link,
291 struct smc_wr_buf **wr_buf,
292 struct smc_rdma_wr **wr_rdma_buf,
293 struct smc_cdc_tx_pend **pend);
294void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
295int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
296 struct smc_cdc_tx_pend *pend);
297int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
298int smcd_cdc_msg_send(struct smc_connection *conn);
299int smcr_cdc_msg_send_validation(struct smc_connection *conn,
300 struct smc_cdc_tx_pend *pend,
301 struct smc_wr_buf *wr_buf);
302int smc_cdc_init(void) __init;
303void smcd_cdc_rx_init(struct smc_connection *conn);
304
305#endif /* SMC_CDC_H */