Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation.
4 */
5
6#ifndef _LINUX_XDP_SOCK_H
7#define _LINUX_XDP_SOCK_H
8
9#include <linux/bpf.h>
10#include <linux/workqueue.h>
11#include <linux/if_xdp.h>
12#include <linux/mutex.h>
13#include <linux/spinlock.h>
14#include <linux/mm.h>
15#include <net/sock.h>
16
17#define XDP_UMEM_SG_FLAG (1 << 1)
18
19struct net_device;
20struct xsk_queue;
21struct xdp_buff;
22
23struct xdp_umem {
24 void *addrs;
25 u64 size;
26 u32 headroom;
27 u32 chunk_size;
28 u32 chunks;
29 u32 npgs;
30 struct user_struct *user;
31 refcount_t users;
32 u8 flags;
33 u8 tx_metadata_len;
34 bool zc;
35 struct page **pgs;
36 int id;
37 struct list_head xsk_dma_list;
38 struct work_struct work;
39};
40
41struct xsk_map {
42 struct bpf_map map;
43 spinlock_t lock; /* Synchronize map updates */
44 atomic_t count;
45 struct xdp_sock __rcu *xsk_map[];
46};
47
48struct xdp_sock {
49 /* struct sock must be the first member of struct xdp_sock */
50 struct sock sk;
51 struct xsk_queue *rx ____cacheline_aligned_in_smp;
52 struct net_device *dev;
53 struct xdp_umem *umem;
54 struct list_head flush_node;
55 struct xsk_buff_pool *pool;
56 u16 queue_id;
57 bool zc;
58 bool sg;
59 enum {
60 XSK_READY = 0,
61 XSK_BOUND,
62 XSK_UNBOUND,
63 } state;
64
65 struct xsk_queue *tx ____cacheline_aligned_in_smp;
66 struct list_head tx_list;
67 /* record the number of tx descriptors sent by this xsk and
68 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
69 * to be given to other xsks for sending tx descriptors, thereby
70 * preventing other XSKs from being starved.
71 */
72 u32 tx_budget_spent;
73
74 /* Protects generic receive. */
75 spinlock_t rx_lock;
76
77 /* Statistics */
78 u64 rx_dropped;
79 u64 rx_queue_full;
80
81 /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
82 * packet, the partially built skb is saved here so that packet building can resume in next
83 * call of __xsk_generic_xmit().
84 */
85 struct sk_buff *skb;
86
87 struct list_head map_list;
88 /* Protects map_list */
89 spinlock_t map_list_lock;
90 /* Protects multiple processes in the control path */
91 struct mutex mutex;
92 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
93 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
94};
95
96/*
97 * AF_XDP TX metadata hooks for network devices.
98 * The following hooks can be defined; unless noted otherwise, they are
99 * optional and can be filled with a null pointer.
100 *
101 * void (*tmo_request_timestamp)(void *priv)
102 * Called when AF_XDP frame requested egress timestamp.
103 *
104 * u64 (*tmo_fill_timestamp)(void *priv)
105 * Called when AF_XDP frame, that had requested egress timestamp,
106 * received a completion. The hook needs to return the actual HW timestamp.
107 *
108 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
109 * Called when AF_XDP frame requested HW checksum offload. csum_start
110 * indicates position where checksumming should start.
111 * csum_offset indicates position where checksum should be stored.
112 *
113 */
114struct xsk_tx_metadata_ops {
115 void (*tmo_request_timestamp)(void *priv);
116 u64 (*tmo_fill_timestamp)(void *priv);
117 void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
118};
119
120#ifdef CONFIG_XDP_SOCKETS
121
122int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
123int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
124void __xsk_map_flush(struct list_head *flush_list);
125
126/**
127 * xsk_tx_metadata_to_compl - Save enough relevant metadata information
128 * to perform tx completion in the future.
129 * @meta: pointer to AF_XDP metadata area
130 * @compl: pointer to output struct xsk_tx_metadata_to_compl
131 *
132 * This function should be called by the networking device when
133 * it prepares AF_XDP egress packet. The value of @compl should be stored
134 * and passed to xsk_tx_metadata_complete upon TX completion.
135 */
136static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
137 struct xsk_tx_metadata_compl *compl)
138{
139 if (!meta)
140 return;
141
142 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
143 compl->tx_timestamp = &meta->completion.tx_timestamp;
144 else
145 compl->tx_timestamp = NULL;
146}
147
148/**
149 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
150 * and call appropriate xsk_tx_metadata_ops operation.
151 * @meta: pointer to AF_XDP metadata area
152 * @ops: pointer to struct xsk_tx_metadata_ops
153 * @priv: pointer to driver-private aread
154 *
155 * This function should be called by the networking device when
156 * it prepares AF_XDP egress packet.
157 */
158static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
159 const struct xsk_tx_metadata_ops *ops,
160 void *priv)
161{
162 if (!meta)
163 return;
164
165 if (ops->tmo_request_timestamp)
166 if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
167 ops->tmo_request_timestamp(priv);
168
169 if (ops->tmo_request_checksum)
170 if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
171 ops->tmo_request_checksum(meta->request.csum_start,
172 meta->request.csum_offset, priv);
173}
174
175/**
176 * xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
177 * and call appropriate xsk_tx_metadata_ops operation.
178 * @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
179 * @ops: pointer to struct xsk_tx_metadata_ops
180 * @priv: pointer to driver-private aread
181 *
182 * This function should be called by the networking device upon
183 * AF_XDP egress completion.
184 */
185static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
186 const struct xsk_tx_metadata_ops *ops,
187 void *priv)
188{
189 if (!compl)
190 return;
191 if (!compl->tx_timestamp)
192 return;
193
194 *compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
195}
196
197#else
198
199static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
200{
201 return -ENOTSUPP;
202}
203
204static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
205{
206 return -EOPNOTSUPP;
207}
208
209static inline void __xsk_map_flush(struct list_head *flush_list)
210{
211}
212
213static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
214 struct xsk_tx_metadata_compl *compl)
215{
216}
217
218static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
219 const struct xsk_tx_metadata_ops *ops,
220 void *priv)
221{
222}
223
224static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
225 const struct xsk_tx_metadata_ops *ops,
226 void *priv)
227{
228}
229
230#endif /* CONFIG_XDP_SOCKETS */
231#endif /* _LINUX_XDP_SOCK_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2/* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation.
4 */
5
6#ifndef _LINUX_XDP_SOCK_H
7#define _LINUX_XDP_SOCK_H
8
9#include <linux/workqueue.h>
10#include <linux/if_xdp.h>
11#include <linux/mutex.h>
12#include <linux/spinlock.h>
13#include <linux/mm.h>
14#include <net/sock.h>
15
16struct net_device;
17struct xsk_queue;
18struct xdp_buff;
19
20struct xdp_umem {
21 struct xsk_queue *fq;
22 struct xsk_queue *cq;
23 struct xsk_buff_pool *pool;
24 u64 size;
25 u32 headroom;
26 u32 chunk_size;
27 struct user_struct *user;
28 refcount_t users;
29 struct work_struct work;
30 struct page **pgs;
31 u32 npgs;
32 u16 queue_id;
33 u8 need_wakeup;
34 u8 flags;
35 int id;
36 struct net_device *dev;
37 bool zc;
38 spinlock_t xsk_tx_list_lock;
39 struct list_head xsk_tx_list;
40};
41
42struct xsk_map {
43 struct bpf_map map;
44 spinlock_t lock; /* Synchronize map updates */
45 struct xdp_sock *xsk_map[];
46};
47
48struct xdp_sock {
49 /* struct sock must be the first member of struct xdp_sock */
50 struct sock sk;
51 struct xsk_queue *rx;
52 struct net_device *dev;
53 struct xdp_umem *umem;
54 struct list_head flush_node;
55 u16 queue_id;
56 bool zc;
57 enum {
58 XSK_READY = 0,
59 XSK_BOUND,
60 XSK_UNBOUND,
61 } state;
62 /* Protects multiple processes in the control path */
63 struct mutex mutex;
64 struct xsk_queue *tx ____cacheline_aligned_in_smp;
65 struct list_head list;
66 /* Mutual exclusion of NAPI TX thread and sendmsg error paths
67 * in the SKB destructor callback.
68 */
69 spinlock_t tx_completion_lock;
70 /* Protects generic receive. */
71 spinlock_t rx_lock;
72
73 /* Statistics */
74 u64 rx_dropped;
75 u64 rx_queue_full;
76
77 struct list_head map_list;
78 /* Protects map_list */
79 spinlock_t map_list_lock;
80};
81
82#ifdef CONFIG_XDP_SOCKETS
83
84int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
85int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
86void __xsk_map_flush(void);
87
88static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
89 u32 key)
90{
91 struct xsk_map *m = container_of(map, struct xsk_map, map);
92 struct xdp_sock *xs;
93
94 if (key >= map->max_entries)
95 return NULL;
96
97 xs = READ_ONCE(m->xsk_map[key]);
98 return xs;
99}
100
101#else
102
103static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
104{
105 return -ENOTSUPP;
106}
107
108static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
109{
110 return -EOPNOTSUPP;
111}
112
113static inline void __xsk_map_flush(void)
114{
115}
116
117static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map,
118 u32 key)
119{
120 return NULL;
121}
122
123#endif /* CONFIG_XDP_SOCKETS */
124
125#endif /* _LINUX_XDP_SOCK_H */