1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Kernel Connection Multiplexor
  4 *
  5 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
  6 */
  7
  8#ifndef __NET_KCM_H_
  9#define __NET_KCM_H_
 10
 11#include <linux/skbuff.h>
 12#include <net/sock.h>
 13#include <net/strparser.h>
 14#include <uapi/linux/kcm.h>
 15
 16extern unsigned int kcm_net_id;
 17
 18#define KCM_STATS_ADD(stat, count) ((stat) += (count))
 19#define KCM_STATS_INCR(stat) ((stat)++)
 20
 21struct kcm_psock_stats {
 22	unsigned long long tx_msgs;
 23	unsigned long long tx_bytes;
 24	unsigned long long reserved;
 25	unsigned long long unreserved;
 26	unsigned int tx_aborts;
 27};
 28
 29struct kcm_mux_stats {
 30	unsigned long long rx_msgs;
 31	unsigned long long rx_bytes;
 32	unsigned long long tx_msgs;
 33	unsigned long long tx_bytes;
 34	unsigned int rx_ready_drops;
 35	unsigned int tx_retries;
 36	unsigned int psock_attach;
 37	unsigned int psock_unattach_rsvd;
 38	unsigned int psock_unattach;
 39};
 40
 41struct kcm_stats {
 42	unsigned long long rx_msgs;
 43	unsigned long long rx_bytes;
 44	unsigned long long tx_msgs;
 45	unsigned long long tx_bytes;
 46};
 47
 48struct kcm_tx_msg {
 49	unsigned int sent;
 50	unsigned int frag_offset;
 51	unsigned int msg_flags;
 52	bool started_tx;
 53	struct sk_buff *frag_skb;
 54	struct sk_buff *last_skb;
 55};
 56
 57/* Socket structure for KCM client sockets */
 58struct kcm_sock {
 59	struct sock sk;
 60	struct kcm_mux *mux;
 61	struct list_head kcm_sock_list;
 62	int index;
 63	u32 done : 1;
 64	struct work_struct done_work;
 65
 66	struct kcm_stats stats;
 67
 68	/* Transmit */
 69	struct kcm_psock *tx_psock;
 70	struct work_struct tx_work;
 71	struct list_head wait_psock_list;
 72	struct sk_buff *seq_skb;
 73	struct mutex tx_mutex;
 74	u32 tx_stopped : 1;
 75
 76	/* Don't use bit fields here, these are set under different locks */
 77	bool tx_wait;
 78	bool tx_wait_more;
 79
 80	/* Receive */
 81	struct kcm_psock *rx_psock;
 82	struct list_head wait_rx_list; /* KCMs waiting for receiving */
 83	bool rx_wait;
 84	u32 rx_disabled : 1;
 85};
 86
 87struct bpf_prog;
 88
 89/* Structure for an attached lower socket */
 90struct kcm_psock {
 91	struct sock *sk;
 92	struct strparser strp;
 93	struct kcm_mux *mux;
 94	int index;
 95
 96	u32 tx_stopped : 1;
 97	u32 done : 1;
 98	u32 unattaching : 1;
 99
100	void (*save_state_change)(struct sock *sk);
101	void (*save_data_ready)(struct sock *sk);
102	void (*save_write_space)(struct sock *sk);
103
104	struct list_head psock_list;
105
106	struct kcm_psock_stats stats;
107
108	/* Receive */
109	struct list_head psock_ready_list;
110	struct bpf_prog *bpf_prog;
111	struct kcm_sock *rx_kcm;
112	unsigned long long saved_rx_bytes;
113	unsigned long long saved_rx_msgs;
114	struct sk_buff *ready_rx_msg;
115
116	/* Transmit */
117	struct kcm_sock *tx_kcm;
118	struct list_head psock_avail_list;
119	unsigned long long saved_tx_bytes;
120	unsigned long long saved_tx_msgs;
121};
122
123/* Per net MUX list */
124struct kcm_net {
125	struct mutex mutex;
126	struct kcm_psock_stats aggregate_psock_stats;
127	struct kcm_mux_stats aggregate_mux_stats;
128	struct strp_aggr_stats aggregate_strp_stats;
129	struct list_head mux_list;
130	int count;
131};
132
133/* Structure for a MUX */
134struct kcm_mux {
135	struct list_head kcm_mux_list;
136	struct rcu_head rcu;
137	struct kcm_net *knet;
138
139	struct list_head kcm_socks;	/* All KCM sockets on MUX */
140	int kcm_socks_cnt;		/* Total KCM socket count for MUX */
141	struct list_head psocks;	/* List of all psocks on MUX */
142	int psocks_cnt;		/* Total attached sockets */
143
144	struct kcm_mux_stats stats;
145	struct kcm_psock_stats aggregate_psock_stats;
146	struct strp_aggr_stats aggregate_strp_stats;
147
148	/* Receive */
149	spinlock_t rx_lock ____cacheline_aligned_in_smp;
150	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
151	struct list_head psocks_ready;	/* List of psocks with a msg ready */
152	struct sk_buff_head rx_hold_queue;
153
154	/* Transmit */
155	spinlock_t  lock ____cacheline_aligned_in_smp;	/* TX and mux locking */
156	struct list_head psocks_avail;	/* List of available psocks */
157	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
158};
159
160#ifdef CONFIG_PROC_FS
161int kcm_proc_init(void);
162void kcm_proc_exit(void);
163#else
164static inline int kcm_proc_init(void) { return 0; }
165static inline void kcm_proc_exit(void) { }
166#endif
167
168static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
169					 struct kcm_psock_stats *agg_stats)
170{
171	/* Save psock statistics in the mux when psock is being unattached. */
172
173#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
174	SAVE_PSOCK_STATS(tx_msgs);
175	SAVE_PSOCK_STATS(tx_bytes);
176	SAVE_PSOCK_STATS(reserved);
177	SAVE_PSOCK_STATS(unreserved);
178	SAVE_PSOCK_STATS(tx_aborts);
179#undef SAVE_PSOCK_STATS
180}
181
182static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
183				       struct kcm_mux_stats *agg_stats)
184{
185	/* Save psock statistics in the mux when psock is being unattached. */
186
187#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
188	SAVE_MUX_STATS(rx_msgs);
189	SAVE_MUX_STATS(rx_bytes);
190	SAVE_MUX_STATS(tx_msgs);
191	SAVE_MUX_STATS(tx_bytes);
192	SAVE_MUX_STATS(rx_ready_drops);
193	SAVE_MUX_STATS(psock_attach);
194	SAVE_MUX_STATS(psock_unattach_rsvd);
195	SAVE_MUX_STATS(psock_unattach);
196#undef SAVE_MUX_STATS
197}
198
199#endif /* __NET_KCM_H_ */