Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Kernel Connection Multiplexor
4 *
5 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
6 */
7
8#ifndef __NET_KCM_H_
9#define __NET_KCM_H_
10
11#include <linux/skbuff.h>
12#include <net/sock.h>
13#include <net/strparser.h>
14#include <uapi/linux/kcm.h>
15
16extern unsigned int kcm_net_id;
17
18#define KCM_STATS_ADD(stat, count) ((stat) += (count))
19#define KCM_STATS_INCR(stat) ((stat)++)
20
21struct kcm_psock_stats {
22 unsigned long long tx_msgs;
23 unsigned long long tx_bytes;
24 unsigned long long reserved;
25 unsigned long long unreserved;
26 unsigned int tx_aborts;
27};
28
29struct kcm_mux_stats {
30 unsigned long long rx_msgs;
31 unsigned long long rx_bytes;
32 unsigned long long tx_msgs;
33 unsigned long long tx_bytes;
34 unsigned int rx_ready_drops;
35 unsigned int tx_retries;
36 unsigned int psock_attach;
37 unsigned int psock_unattach_rsvd;
38 unsigned int psock_unattach;
39};
40
41struct kcm_stats {
42 unsigned long long rx_msgs;
43 unsigned long long rx_bytes;
44 unsigned long long tx_msgs;
45 unsigned long long tx_bytes;
46};
47
48struct kcm_tx_msg {
49 unsigned int sent;
50 unsigned int frag_offset;
51 unsigned int msg_flags;
52 bool started_tx;
53 struct sk_buff *frag_skb;
54 struct sk_buff *last_skb;
55};
56
57/* Socket structure for KCM client sockets */
58struct kcm_sock {
59 struct sock sk;
60 struct kcm_mux *mux;
61 struct list_head kcm_sock_list;
62 int index;
63 u32 done : 1;
64 struct work_struct done_work;
65
66 struct kcm_stats stats;
67
68 /* Transmit */
69 struct kcm_psock *tx_psock;
70 struct work_struct tx_work;
71 struct list_head wait_psock_list;
72 struct sk_buff *seq_skb;
73 struct mutex tx_mutex;
74 u32 tx_stopped : 1;
75
76 /* Don't use bit fields here, these are set under different locks */
77 bool tx_wait;
78 bool tx_wait_more;
79
80 /* Receive */
81 struct kcm_psock *rx_psock;
82 struct list_head wait_rx_list; /* KCMs waiting for receiving */
83 bool rx_wait;
84 u32 rx_disabled : 1;
85};
86
87struct bpf_prog;
88
89/* Structure for an attached lower socket */
90struct kcm_psock {
91 struct sock *sk;
92 struct strparser strp;
93 struct kcm_mux *mux;
94 int index;
95
96 u32 tx_stopped : 1;
97 u32 done : 1;
98 u32 unattaching : 1;
99
100 void (*save_state_change)(struct sock *sk);
101 void (*save_data_ready)(struct sock *sk);
102 void (*save_write_space)(struct sock *sk);
103
104 struct list_head psock_list;
105
106 struct kcm_psock_stats stats;
107
108 /* Receive */
109 struct list_head psock_ready_list;
110 struct bpf_prog *bpf_prog;
111 struct kcm_sock *rx_kcm;
112 unsigned long long saved_rx_bytes;
113 unsigned long long saved_rx_msgs;
114 struct sk_buff *ready_rx_msg;
115
116 /* Transmit */
117 struct kcm_sock *tx_kcm;
118 struct list_head psock_avail_list;
119 unsigned long long saved_tx_bytes;
120 unsigned long long saved_tx_msgs;
121};
122
123/* Per net MUX list */
124struct kcm_net {
125 struct mutex mutex;
126 struct kcm_psock_stats aggregate_psock_stats;
127 struct kcm_mux_stats aggregate_mux_stats;
128 struct strp_aggr_stats aggregate_strp_stats;
129 struct list_head mux_list;
130 int count;
131};
132
133/* Structure for a MUX */
134struct kcm_mux {
135 struct list_head kcm_mux_list;
136 struct rcu_head rcu;
137 struct kcm_net *knet;
138
139 struct list_head kcm_socks; /* All KCM sockets on MUX */
140 int kcm_socks_cnt; /* Total KCM socket count for MUX */
141 struct list_head psocks; /* List of all psocks on MUX */
142 int psocks_cnt; /* Total attached sockets */
143
144 struct kcm_mux_stats stats;
145 struct kcm_psock_stats aggregate_psock_stats;
146 struct strp_aggr_stats aggregate_strp_stats;
147
148 /* Receive */
149 spinlock_t rx_lock ____cacheline_aligned_in_smp;
150 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
151 struct list_head psocks_ready; /* List of psocks with a msg ready */
152 struct sk_buff_head rx_hold_queue;
153
154 /* Transmit */
155 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */
156 struct list_head psocks_avail; /* List of available psocks */
157 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
158};
159
160#ifdef CONFIG_PROC_FS
161int kcm_proc_init(void);
162void kcm_proc_exit(void);
163#else
164static inline int kcm_proc_init(void) { return 0; }
165static inline void kcm_proc_exit(void) { }
166#endif
167
168static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
169 struct kcm_psock_stats *agg_stats)
170{
171 /* Save psock statistics in the mux when psock is being unattached. */
172
173#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
174 SAVE_PSOCK_STATS(tx_msgs);
175 SAVE_PSOCK_STATS(tx_bytes);
176 SAVE_PSOCK_STATS(reserved);
177 SAVE_PSOCK_STATS(unreserved);
178 SAVE_PSOCK_STATS(tx_aborts);
179#undef SAVE_PSOCK_STATS
180}
181
182static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
183 struct kcm_mux_stats *agg_stats)
184{
185 /* Save psock statistics in the mux when psock is being unattached. */
186
187#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
188 SAVE_MUX_STATS(rx_msgs);
189 SAVE_MUX_STATS(rx_bytes);
190 SAVE_MUX_STATS(tx_msgs);
191 SAVE_MUX_STATS(tx_bytes);
192 SAVE_MUX_STATS(rx_ready_drops);
193 SAVE_MUX_STATS(psock_attach);
194 SAVE_MUX_STATS(psock_unattach_rsvd);
195 SAVE_MUX_STATS(psock_unattach);
196#undef SAVE_MUX_STATS
197}
198
199#endif /* __NET_KCM_H_ */
1/*
2 * Kernel Connection Multiplexor
3 *
4 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2
8 * as published by the Free Software Foundation.
9 */
10
11#ifndef __NET_KCM_H_
12#define __NET_KCM_H_
13
14#include <linux/skbuff.h>
15#include <net/sock.h>
16#include <net/strparser.h>
17#include <uapi/linux/kcm.h>
18
19extern unsigned int kcm_net_id;
20
21#define KCM_STATS_ADD(stat, count) ((stat) += (count))
22#define KCM_STATS_INCR(stat) ((stat)++)
23
24struct kcm_psock_stats {
25 unsigned long long tx_msgs;
26 unsigned long long tx_bytes;
27 unsigned long long reserved;
28 unsigned long long unreserved;
29 unsigned int tx_aborts;
30};
31
32struct kcm_mux_stats {
33 unsigned long long rx_msgs;
34 unsigned long long rx_bytes;
35 unsigned long long tx_msgs;
36 unsigned long long tx_bytes;
37 unsigned int rx_ready_drops;
38 unsigned int tx_retries;
39 unsigned int psock_attach;
40 unsigned int psock_unattach_rsvd;
41 unsigned int psock_unattach;
42};
43
44struct kcm_stats {
45 unsigned long long rx_msgs;
46 unsigned long long rx_bytes;
47 unsigned long long tx_msgs;
48 unsigned long long tx_bytes;
49};
50
51struct kcm_tx_msg {
52 unsigned int sent;
53 unsigned int fragidx;
54 unsigned int frag_offset;
55 unsigned int msg_flags;
56 struct sk_buff *frag_skb;
57 struct sk_buff *last_skb;
58};
59
60/* Socket structure for KCM client sockets */
61struct kcm_sock {
62 struct sock sk;
63 struct kcm_mux *mux;
64 struct list_head kcm_sock_list;
65 int index;
66 u32 done : 1;
67 struct work_struct done_work;
68
69 struct kcm_stats stats;
70
71 /* Transmit */
72 struct kcm_psock *tx_psock;
73 struct work_struct tx_work;
74 struct list_head wait_psock_list;
75 struct sk_buff *seq_skb;
76 u32 tx_stopped : 1;
77
78 /* Don't use bit fields here, these are set under different locks */
79 bool tx_wait;
80 bool tx_wait_more;
81
82 /* Receive */
83 struct kcm_psock *rx_psock;
84 struct list_head wait_rx_list; /* KCMs waiting for receiving */
85 bool rx_wait;
86 u32 rx_disabled : 1;
87};
88
89struct bpf_prog;
90
91/* Structure for an attached lower socket */
92struct kcm_psock {
93 struct sock *sk;
94 struct strparser strp;
95 struct kcm_mux *mux;
96 int index;
97
98 u32 tx_stopped : 1;
99 u32 done : 1;
100 u32 unattaching : 1;
101
102 void (*save_state_change)(struct sock *sk);
103 void (*save_data_ready)(struct sock *sk);
104 void (*save_write_space)(struct sock *sk);
105
106 struct list_head psock_list;
107
108 struct kcm_psock_stats stats;
109
110 /* Receive */
111 struct list_head psock_ready_list;
112 struct bpf_prog *bpf_prog;
113 struct kcm_sock *rx_kcm;
114 unsigned long long saved_rx_bytes;
115 unsigned long long saved_rx_msgs;
116 struct sk_buff *ready_rx_msg;
117
118 /* Transmit */
119 struct kcm_sock *tx_kcm;
120 struct list_head psock_avail_list;
121 unsigned long long saved_tx_bytes;
122 unsigned long long saved_tx_msgs;
123};
124
125/* Per net MUX list */
126struct kcm_net {
127 struct mutex mutex;
128 struct kcm_psock_stats aggregate_psock_stats;
129 struct kcm_mux_stats aggregate_mux_stats;
130 struct strp_aggr_stats aggregate_strp_stats;
131 struct list_head mux_list;
132 int count;
133};
134
135/* Structure for a MUX */
136struct kcm_mux {
137 struct list_head kcm_mux_list;
138 struct rcu_head rcu;
139 struct kcm_net *knet;
140
141 struct list_head kcm_socks; /* All KCM sockets on MUX */
142 int kcm_socks_cnt; /* Total KCM socket count for MUX */
143 struct list_head psocks; /* List of all psocks on MUX */
144 int psocks_cnt; /* Total attached sockets */
145
146 struct kcm_mux_stats stats;
147 struct kcm_psock_stats aggregate_psock_stats;
148 struct strp_aggr_stats aggregate_strp_stats;
149
150 /* Receive */
151 spinlock_t rx_lock ____cacheline_aligned_in_smp;
152 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
153 struct list_head psocks_ready; /* List of psocks with a msg ready */
154 struct sk_buff_head rx_hold_queue;
155
156 /* Transmit */
157 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */
158 struct list_head psocks_avail; /* List of available psocks */
159 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
160};
161
162#ifdef CONFIG_PROC_FS
163int kcm_proc_init(void);
164void kcm_proc_exit(void);
165#else
166static inline int kcm_proc_init(void) { return 0; }
167static inline void kcm_proc_exit(void) { }
168#endif
169
170static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
171 struct kcm_psock_stats *agg_stats)
172{
173 /* Save psock statistics in the mux when psock is being unattached. */
174
175#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
176 SAVE_PSOCK_STATS(tx_msgs);
177 SAVE_PSOCK_STATS(tx_bytes);
178 SAVE_PSOCK_STATS(reserved);
179 SAVE_PSOCK_STATS(unreserved);
180 SAVE_PSOCK_STATS(tx_aborts);
181#undef SAVE_PSOCK_STATS
182}
183
184static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
185 struct kcm_mux_stats *agg_stats)
186{
187 /* Save psock statistics in the mux when psock is being unattached. */
188
189#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
190 SAVE_MUX_STATS(rx_msgs);
191 SAVE_MUX_STATS(rx_bytes);
192 SAVE_MUX_STATS(tx_msgs);
193 SAVE_MUX_STATS(tx_bytes);
194 SAVE_MUX_STATS(rx_ready_drops);
195 SAVE_MUX_STATS(psock_attach);
196 SAVE_MUX_STATS(psock_unattach_rsvd);
197 SAVE_MUX_STATS(psock_unattach);
198#undef SAVE_MUX_STATS
199}
200
201#endif /* __NET_KCM_H_ */