Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Interface for implementing AF_XDP zero-copy support in drivers.
3 * Copyright(c) 2020 Intel Corporation.
4 */
5
6#ifndef _LINUX_XDP_SOCK_DRV_H
7#define _LINUX_XDP_SOCK_DRV_H
8
9#include <net/xdp_sock.h>
10#include <net/xsk_buff_pool.h>
11
12#ifdef CONFIG_XDP_SOCKETS
13
14void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
15bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
16u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, u32 max);
17void xsk_tx_release(struct xsk_buff_pool *pool);
18struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
19 u16 queue_id);
20void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool);
21void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool);
22void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
23void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
24bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
25
26static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
27{
28 return XDP_PACKET_HEADROOM + pool->headroom;
29}
30
31static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
32{
33 return pool->chunk_size;
34}
35
36static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
37{
38 return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool);
39}
40
41static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
42 struct xdp_rxq_info *rxq)
43{
44 xp_set_rxq_info(pool, rxq);
45}
46
47static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
48 unsigned long attrs)
49{
50 xp_dma_unmap(pool, attrs);
51}
52
53static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
54 struct device *dev, unsigned long attrs)
55{
56 struct xdp_umem *umem = pool->umem;
57
58 return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs);
59}
60
61static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
62{
63 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
64
65 return xp_get_dma(xskb);
66}
67
68static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
69{
70 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
71
72 return xp_get_frame_dma(xskb);
73}
74
75static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
76{
77 return xp_alloc(pool);
78}
79
80static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
81{
82 return xp_can_alloc(pool, count);
83}
84
85static inline void xsk_buff_free(struct xdp_buff *xdp)
86{
87 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
88
89 xp_free(xskb);
90}
91
92static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
93 u64 addr)
94{
95 return xp_raw_get_dma(pool, addr);
96}
97
98static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
99{
100 return xp_raw_get_data(pool, addr);
101}
102
103static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
104{
105 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
106
107 if (!pool->dma_need_sync)
108 return;
109
110 xp_dma_sync_for_cpu(xskb);
111}
112
113static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
114 dma_addr_t dma,
115 size_t size)
116{
117 xp_dma_sync_for_device(pool, dma, size);
118}
119
120#else
121
122static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
123{
124}
125
126static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
127 struct xdp_desc *desc)
128{
129 return false;
130}
131
132static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc,
133 u32 max)
134{
135 return 0;
136}
137
138static inline void xsk_tx_release(struct xsk_buff_pool *pool)
139{
140}
141
142static inline struct xsk_buff_pool *
143xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id)
144{
145 return NULL;
146}
147
148static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
149{
150}
151
152static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
153{
154}
155
156static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
157{
158}
159
160static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
161{
162}
163
164static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
165{
166 return false;
167}
168
169static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
170{
171 return 0;
172}
173
174static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
175{
176 return 0;
177}
178
179static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
180{
181 return 0;
182}
183
184static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
185 struct xdp_rxq_info *rxq)
186{
187}
188
189static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
190 unsigned long attrs)
191{
192}
193
194static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
195 struct device *dev, unsigned long attrs)
196{
197 return 0;
198}
199
200static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
201{
202 return 0;
203}
204
205static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
206{
207 return 0;
208}
209
210static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
211{
212 return NULL;
213}
214
215static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
216{
217 return false;
218}
219
220static inline void xsk_buff_free(struct xdp_buff *xdp)
221{
222}
223
224static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
225 u64 addr)
226{
227 return 0;
228}
229
230static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
231{
232 return NULL;
233}
234
235static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
236{
237}
238
239static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
240 dma_addr_t dma,
241 size_t size)
242{
243}
244
245#endif /* CONFIG_XDP_SOCKETS */
246
247#endif /* _LINUX_XDP_SOCK_DRV_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Interface for implementing AF_XDP zero-copy support in drivers.
3 * Copyright(c) 2020 Intel Corporation.
4 */
5
6#ifndef _LINUX_XDP_SOCK_DRV_H
7#define _LINUX_XDP_SOCK_DRV_H
8
9#include <net/xdp_sock.h>
10#include <net/xsk_buff_pool.h>
11
12#define XDP_UMEM_MIN_CHUNK_SHIFT 11
13#define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT)
14
15#ifdef CONFIG_XDP_SOCKETS
16
17void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
18bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
19u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
20void xsk_tx_release(struct xsk_buff_pool *pool);
21struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
22 u16 queue_id);
23void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool);
24void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool);
25void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
26void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
27bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
28
29static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
30{
31 return XDP_PACKET_HEADROOM + pool->headroom;
32}
33
34static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
35{
36 return pool->chunk_size;
37}
38
39static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
40{
41 return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool);
42}
43
44static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
45 struct xdp_rxq_info *rxq)
46{
47 xp_set_rxq_info(pool, rxq);
48}
49
50static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool)
51{
52#ifdef CONFIG_NET_RX_BUSY_POLL
53 return pool->heads[0].xdp.rxq->napi_id;
54#else
55 return 0;
56#endif
57}
58
59static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
60 unsigned long attrs)
61{
62 xp_dma_unmap(pool, attrs);
63}
64
65static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
66 struct device *dev, unsigned long attrs)
67{
68 struct xdp_umem *umem = pool->umem;
69
70 return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs);
71}
72
73static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
74{
75 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
76
77 return xp_get_dma(xskb);
78}
79
80static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
81{
82 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
83
84 return xp_get_frame_dma(xskb);
85}
86
87static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
88{
89 return xp_alloc(pool);
90}
91
92/* Returns as many entries as possible up to max. 0 <= N <= max. */
93static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
94{
95 return xp_alloc_batch(pool, xdp, max);
96}
97
98static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
99{
100 return xp_can_alloc(pool, count);
101}
102
103static inline void xsk_buff_free(struct xdp_buff *xdp)
104{
105 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
106
107 xp_free(xskb);
108}
109
110static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
111{
112 xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
113 xdp->data_meta = xdp->data;
114 xdp->data_end = xdp->data + size;
115}
116
117static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
118 u64 addr)
119{
120 return xp_raw_get_dma(pool, addr);
121}
122
123static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
124{
125 return xp_raw_get_data(pool, addr);
126}
127
128static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
129{
130 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
131
132 if (!pool->dma_need_sync)
133 return;
134
135 xp_dma_sync_for_cpu(xskb);
136}
137
138static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
139 dma_addr_t dma,
140 size_t size)
141{
142 xp_dma_sync_for_device(pool, dma, size);
143}
144
145#else
146
147static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
148{
149}
150
151static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
152 struct xdp_desc *desc)
153{
154 return false;
155}
156
157static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
158{
159 return 0;
160}
161
162static inline void xsk_tx_release(struct xsk_buff_pool *pool)
163{
164}
165
166static inline struct xsk_buff_pool *
167xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id)
168{
169 return NULL;
170}
171
172static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
173{
174}
175
176static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
177{
178}
179
180static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
181{
182}
183
184static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
185{
186}
187
188static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
189{
190 return false;
191}
192
193static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
194{
195 return 0;
196}
197
198static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
199{
200 return 0;
201}
202
203static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
204{
205 return 0;
206}
207
208static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
209 struct xdp_rxq_info *rxq)
210{
211}
212
213static inline unsigned int xsk_pool_get_napi_id(struct xsk_buff_pool *pool)
214{
215 return 0;
216}
217
218static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
219 unsigned long attrs)
220{
221}
222
223static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
224 struct device *dev, unsigned long attrs)
225{
226 return 0;
227}
228
229static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
230{
231 return 0;
232}
233
234static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
235{
236 return 0;
237}
238
239static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
240{
241 return NULL;
242}
243
244static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
245{
246 return 0;
247}
248
249static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
250{
251 return false;
252}
253
254static inline void xsk_buff_free(struct xdp_buff *xdp)
255{
256}
257
258static inline void xsk_buff_discard(struct xdp_buff *xdp)
259{
260}
261
262static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
263{
264}
265
266static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
267 u64 addr)
268{
269 return 0;
270}
271
272static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
273{
274 return NULL;
275}
276
277static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, struct xsk_buff_pool *pool)
278{
279}
280
281static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
282 dma_addr_t dma,
283 size_t size)
284{
285}
286
287#endif /* CONFIG_XDP_SOCKETS */
288
289#endif /* _LINUX_XDP_SOCK_DRV_H */