1/* SPDX-License-Identifier: GPL-2.0 */
  2/* AF_XDP internal functions
  3 * Copyright(c) 2018 Intel Corporation.
  4 */
  5
  6#ifndef _LINUX_XDP_SOCK_H
  7#define _LINUX_XDP_SOCK_H
  8
  9#include <linux/bpf.h>
 10#include <linux/workqueue.h>
 11#include <linux/if_xdp.h>
 12#include <linux/mutex.h>
 13#include <linux/spinlock.h>
 14#include <linux/mm.h>
 15#include <net/sock.h>
 16
 17#define XDP_UMEM_SG_FLAG (1 << 1)
 18
 19struct net_device;
 20struct xsk_queue;
 21struct xdp_buff;
 22
 23struct xdp_umem {
 24	void *addrs;
 25	u64 size;
 26	u32 headroom;
 27	u32 chunk_size;
 28	u32 chunks;
 29	u32 npgs;
 30	struct user_struct *user;
 31	refcount_t users;
 32	u8 flags;
 33	u8 tx_metadata_len;
 34	bool zc;
 35	struct page **pgs;
 36	int id;
 37	struct list_head xsk_dma_list;
 38	struct work_struct work;
 39};
 40
 41struct xsk_map {
 42	struct bpf_map map;
 43	spinlock_t lock; /* Synchronize map updates */
 44	atomic_t count;
 45	struct xdp_sock __rcu *xsk_map[];
 46};
 47
 48struct xdp_sock {
 49	/* struct sock must be the first member of struct xdp_sock */
 50	struct sock sk;
 51	struct xsk_queue *rx ____cacheline_aligned_in_smp;
 52	struct net_device *dev;
 53	struct xdp_umem *umem;
 54	struct list_head flush_node;
 55	struct xsk_buff_pool *pool;
 56	u16 queue_id;
 57	bool zc;
 58	bool sg;
 59	enum {
 60		XSK_READY = 0,
 61		XSK_BOUND,
 62		XSK_UNBOUND,
 63	} state;
 64
 65	struct xsk_queue *tx ____cacheline_aligned_in_smp;
 66	struct list_head tx_list;
 67	/* record the number of tx descriptors sent by this xsk and
 68	 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
 69	 * to be given to other xsks for sending tx descriptors, thereby
 70	 * preventing other XSKs from being starved.
 71	 */
 72	u32 tx_budget_spent;
 73
 74	/* Protects generic receive. */
 75	spinlock_t rx_lock;
 76
 77	/* Statistics */
 78	u64 rx_dropped;
 79	u64 rx_queue_full;
 80
 81	/* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
 82	 * packet, the partially built skb is saved here so that packet building can resume in next
 83	 * call of __xsk_generic_xmit().
 84	 */
 85	struct sk_buff *skb;
 86
 87	struct list_head map_list;
 88	/* Protects map_list */
 89	spinlock_t map_list_lock;
 90	/* Protects multiple processes in the control path */
 91	struct mutex mutex;
 92	struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
 93	struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
 94};
 95
 96/*
 97 * AF_XDP TX metadata hooks for network devices.
 98 * The following hooks can be defined; unless noted otherwise, they are
 99 * optional and can be filled with a null pointer.
100 *
101 * void (*tmo_request_timestamp)(void *priv)
102 *     Called when AF_XDP frame requested egress timestamp.
103 *
104 * u64 (*tmo_fill_timestamp)(void *priv)
105 *     Called when AF_XDP frame, that had requested egress timestamp,
106 *     received a completion. The hook needs to return the actual HW timestamp.
107 *
108 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
109 *     Called when AF_XDP frame requested HW checksum offload. csum_start
110 *     indicates position where checksumming should start.
111 *     csum_offset indicates position where checksum should be stored.
112 *
113 */
114struct xsk_tx_metadata_ops {
115	void	(*tmo_request_timestamp)(void *priv);
116	u64	(*tmo_fill_timestamp)(void *priv);
117	void	(*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
118};
119
120#ifdef CONFIG_XDP_SOCKETS
121
122int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
123int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
124void __xsk_map_flush(void);
125
126/**
127 *  xsk_tx_metadata_to_compl - Save enough relevant metadata information
128 *  to perform tx completion in the future.
129 *  @meta: pointer to AF_XDP metadata area
130 *  @compl: pointer to output struct xsk_tx_metadata_to_compl
131 *
132 *  This function should be called by the networking device when
133 *  it prepares AF_XDP egress packet. The value of @compl should be stored
134 *  and passed to xsk_tx_metadata_complete upon TX completion.
135 */
136static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
137					    struct xsk_tx_metadata_compl *compl)
138{
139	if (!meta)
140		return;
141
142	if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
143		compl->tx_timestamp = &meta->completion.tx_timestamp;
144	else
145		compl->tx_timestamp = NULL;
146}
147
148/**
149 *  xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
150 *  and call appropriate xsk_tx_metadata_ops operation.
151 *  @meta: pointer to AF_XDP metadata area
152 *  @ops: pointer to struct xsk_tx_metadata_ops
153 *  @priv: pointer to driver-private aread
154 *
155 *  This function should be called by the networking device when
156 *  it prepares AF_XDP egress packet.
157 */
158static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
159					   const struct xsk_tx_metadata_ops *ops,
160					   void *priv)
161{
162	if (!meta)
163		return;
164
165	if (ops->tmo_request_timestamp)
166		if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
167			ops->tmo_request_timestamp(priv);
168
169	if (ops->tmo_request_checksum)
170		if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
171			ops->tmo_request_checksum(meta->request.csum_start,
172						  meta->request.csum_offset, priv);
173}
174
175/**
176 *  xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
177 *  and call appropriate xsk_tx_metadata_ops operation.
178 *  @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
179 *  @ops: pointer to struct xsk_tx_metadata_ops
180 *  @priv: pointer to driver-private aread
181 *
182 *  This function should be called by the networking device upon
183 *  AF_XDP egress completion.
184 */
185static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
186					    const struct xsk_tx_metadata_ops *ops,
187					    void *priv)
188{
189	if (!compl)
190		return;
191	if (!compl->tx_timestamp)
192		return;
193
194	*compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
195}
196
197#else
198
199static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
200{
201	return -ENOTSUPP;
202}
203
204static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
205{
206	return -EOPNOTSUPP;
207}
208
209static inline void __xsk_map_flush(void)
210{
211}
212
213static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
214					    struct xsk_tx_metadata_compl *compl)
215{
216}
217
218static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
219					   const struct xsk_tx_metadata_ops *ops,
220					   void *priv)
221{
222}
223
224static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
225					    const struct xsk_tx_metadata_ops *ops,
226					    void *priv)
227{
228}
229
230#endif /* CONFIG_XDP_SOCKETS */
231
232#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
233bool xsk_map_check_flush(void);
234#else
235static inline bool xsk_map_check_flush(void)
236{
237	return false;
238}
239#endif
240
241#endif /* _LINUX_XDP_SOCK_H */