Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __PACKET_INTERNAL_H__
3#define __PACKET_INTERNAL_H__
4
5#include <linux/refcount.h>
6
7struct packet_mclist {
8 struct packet_mclist *next;
9 int ifindex;
10 int count;
11 unsigned short type;
12 unsigned short alen;
13 unsigned char addr[MAX_ADDR_LEN];
14};
15
16/* kbdq - kernel block descriptor queue */
17struct tpacket_kbdq_core {
18 struct pgv *pkbdq;
19 unsigned int feature_req_word;
20 unsigned int hdrlen;
21 unsigned char reset_pending_on_curr_blk;
22 unsigned char delete_blk_timer;
23 unsigned short kactive_blk_num;
24 unsigned short blk_sizeof_priv;
25
26 /* last_kactive_blk_num:
27 * trick to see if user-space has caught up
28 * in order to avoid refreshing timer when every single pkt arrives.
29 */
30 unsigned short last_kactive_blk_num;
31
32 char *pkblk_start;
33 char *pkblk_end;
34 int kblk_size;
35 unsigned int max_frame_len;
36 unsigned int knum_blocks;
37 uint64_t knxt_seq_num;
38 char *prev;
39 char *nxt_offset;
40 struct sk_buff *skb;
41
42 rwlock_t blk_fill_in_prog_lock;
43
44 /* Default is set to 8ms */
45#define DEFAULT_PRB_RETIRE_TOV (8)
46
47 unsigned short retire_blk_tov;
48 unsigned short version;
49 unsigned long tov_in_jiffies;
50
51 /* timer to retire an outstanding block */
52 struct timer_list retire_blk_timer;
53};
54
55struct pgv {
56 char *buffer;
57};
58
59struct packet_ring_buffer {
60 struct pgv *pg_vec;
61
62 unsigned int head;
63 unsigned int frames_per_block;
64 unsigned int frame_size;
65 unsigned int frame_max;
66
67 unsigned int pg_vec_order;
68 unsigned int pg_vec_pages;
69 unsigned int pg_vec_len;
70
71 unsigned int __percpu *pending_refcnt;
72
73 union {
74 unsigned long *rx_owner_map;
75 struct tpacket_kbdq_core prb_bdqc;
76 };
77};
78
79extern struct mutex fanout_mutex;
80#define PACKET_FANOUT_MAX (1 << 16)
81
82struct packet_fanout {
83 possible_net_t net;
84 unsigned int num_members;
85 u32 max_num_members;
86 u16 id;
87 u8 type;
88 u8 flags;
89 union {
90 atomic_t rr_cur;
91 struct bpf_prog __rcu *bpf_prog;
92 };
93 struct list_head list;
94 spinlock_t lock;
95 refcount_t sk_ref;
96 struct packet_type prot_hook ____cacheline_aligned_in_smp;
97 struct sock __rcu *arr[];
98};
99
100struct packet_rollover {
101 int sock;
102 atomic_long_t num;
103 atomic_long_t num_huge;
104 atomic_long_t num_failed;
105#define ROLLOVER_HLEN (L1_CACHE_BYTES / sizeof(u32))
106 u32 history[ROLLOVER_HLEN] ____cacheline_aligned;
107} ____cacheline_aligned_in_smp;
108
109struct packet_sock {
110 /* struct sock has to be the first member of packet_sock */
111 struct sock sk;
112 struct packet_fanout *fanout;
113 union tpacket_stats_u stats;
114 struct packet_ring_buffer rx_ring;
115 struct packet_ring_buffer tx_ring;
116 int copy_thresh;
117 spinlock_t bind_lock;
118 struct mutex pg_vec_lock;
119 unsigned int running; /* bind_lock must be held */
120 unsigned int auxdata:1, /* writer must hold sock lock */
121 origdev:1,
122 has_vnet_hdr:1,
123 tp_loss:1,
124 tp_tx_has_off:1;
125 int pressure;
126 int ifindex; /* bound device */
127 __be16 num;
128 struct packet_rollover *rollover;
129 struct packet_mclist *mclist;
130 atomic_t mapped;
131 enum tpacket_versions tp_version;
132 unsigned int tp_hdrlen;
133 unsigned int tp_reserve;
134 unsigned int tp_tstamp;
135 struct completion skb_completion;
136 struct net_device __rcu *cached_dev;
137 int (*xmit)(struct sk_buff *skb);
138 struct packet_type prot_hook ____cacheline_aligned_in_smp;
139 atomic_t tp_drops ____cacheline_aligned_in_smp;
140};
141
142static inline struct packet_sock *pkt_sk(struct sock *sk)
143{
144 return (struct packet_sock *)sk;
145}
146
147#endif
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __PACKET_INTERNAL_H__
3#define __PACKET_INTERNAL_H__
4
5#include <linux/refcount.h>
6
7struct packet_mclist {
8 struct packet_mclist *next;
9 int ifindex;
10 int count;
11 unsigned short type;
12 unsigned short alen;
13 unsigned char addr[MAX_ADDR_LEN];
14};
15
16/* kbdq - kernel block descriptor queue */
17struct tpacket_kbdq_core {
18 struct pgv *pkbdq;
19 unsigned int feature_req_word;
20 unsigned int hdrlen;
21 unsigned char reset_pending_on_curr_blk;
22 unsigned char delete_blk_timer;
23 unsigned short kactive_blk_num;
24 unsigned short blk_sizeof_priv;
25
26 /* last_kactive_blk_num:
27 * trick to see if user-space has caught up
28 * in order to avoid refreshing timer when every single pkt arrives.
29 */
30 unsigned short last_kactive_blk_num;
31
32 char *pkblk_start;
33 char *pkblk_end;
34 int kblk_size;
35 unsigned int max_frame_len;
36 unsigned int knum_blocks;
37 uint64_t knxt_seq_num;
38 char *prev;
39 char *nxt_offset;
40 struct sk_buff *skb;
41
42 rwlock_t blk_fill_in_prog_lock;
43
44 /* Default is set to 8ms */
45#define DEFAULT_PRB_RETIRE_TOV (8)
46
47 unsigned short retire_blk_tov;
48 unsigned short version;
49 unsigned long tov_in_jiffies;
50
51 /* timer to retire an outstanding block */
52 struct timer_list retire_blk_timer;
53};
54
55struct pgv {
56 char *buffer;
57};
58
59struct packet_ring_buffer {
60 struct pgv *pg_vec;
61
62 unsigned int head;
63 unsigned int frames_per_block;
64 unsigned int frame_size;
65 unsigned int frame_max;
66
67 unsigned int pg_vec_order;
68 unsigned int pg_vec_pages;
69 unsigned int pg_vec_len;
70
71 unsigned int __percpu *pending_refcnt;
72
73 union {
74 unsigned long *rx_owner_map;
75 struct tpacket_kbdq_core prb_bdqc;
76 };
77};
78
79extern struct mutex fanout_mutex;
80#define PACKET_FANOUT_MAX (1 << 16)
81
82struct packet_fanout {
83 possible_net_t net;
84 unsigned int num_members;
85 u32 max_num_members;
86 u16 id;
87 u8 type;
88 u8 flags;
89 union {
90 atomic_t rr_cur;
91 struct bpf_prog __rcu *bpf_prog;
92 };
93 struct list_head list;
94 spinlock_t lock;
95 refcount_t sk_ref;
96 struct packet_type prot_hook ____cacheline_aligned_in_smp;
97 struct sock __rcu *arr[] __counted_by(max_num_members);
98};
99
100struct packet_rollover {
101 int sock;
102 atomic_long_t num;
103 atomic_long_t num_huge;
104 atomic_long_t num_failed;
105#define ROLLOVER_HLEN (L1_CACHE_BYTES / sizeof(u32))
106 u32 history[ROLLOVER_HLEN] ____cacheline_aligned;
107} ____cacheline_aligned_in_smp;
108
109struct packet_sock {
110 /* struct sock has to be the first member of packet_sock */
111 struct sock sk;
112 struct packet_fanout *fanout;
113 union tpacket_stats_u stats;
114 struct packet_ring_buffer rx_ring;
115 struct packet_ring_buffer tx_ring;
116 int copy_thresh;
117 spinlock_t bind_lock;
118 struct mutex pg_vec_lock;
119 unsigned long flags;
120 int ifindex; /* bound device */
121 u8 vnet_hdr_sz;
122 __be16 num;
123 struct packet_rollover *rollover;
124 struct packet_mclist *mclist;
125 atomic_long_t mapped;
126 enum tpacket_versions tp_version;
127 unsigned int tp_hdrlen;
128 unsigned int tp_reserve;
129 unsigned int tp_tstamp;
130 struct completion skb_completion;
131 struct net_device __rcu *cached_dev;
132 struct packet_type prot_hook ____cacheline_aligned_in_smp;
133 atomic_t tp_drops ____cacheline_aligned_in_smp;
134};
135
136#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk)
137
138enum packet_sock_flags {
139 PACKET_SOCK_ORIGDEV,
140 PACKET_SOCK_AUXDATA,
141 PACKET_SOCK_TX_HAS_OFF,
142 PACKET_SOCK_TP_LOSS,
143 PACKET_SOCK_RUNNING,
144 PACKET_SOCK_PRESSURE,
145 PACKET_SOCK_QDISC_BYPASS,
146};
147
148static inline void packet_sock_flag_set(struct packet_sock *po,
149 enum packet_sock_flags flag,
150 bool val)
151{
152 if (val)
153 set_bit(flag, &po->flags);
154 else
155 clear_bit(flag, &po->flags);
156}
157
158static inline bool packet_sock_flag(const struct packet_sock *po,
159 enum packet_sock_flags flag)
160{
161 return test_bit(flag, &po->flags);
162}
163
164#endif