1/*
  2 * Kernel Connection Multiplexor
  3 *
  4 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
  5 *
  6 * This program is free software; you can redistribute it and/or modify
  7 * it under the terms of the GNU General Public License version 2
  8 * as published by the Free Software Foundation.
  9 */
 10
 11#ifndef __NET_KCM_H_
 12#define __NET_KCM_H_
 13
 14#include <linux/skbuff.h>
 15#include <net/sock.h>
 16#include <net/strparser.h>
 17#include <uapi/linux/kcm.h>
 18
 19extern unsigned int kcm_net_id;
 20
 21#define KCM_STATS_ADD(stat, count) ((stat) += (count))
 22#define KCM_STATS_INCR(stat) ((stat)++)
 23
 24struct kcm_psock_stats {
 25	unsigned long long tx_msgs;
 26	unsigned long long tx_bytes;
 27	unsigned long long reserved;
 28	unsigned long long unreserved;
 29	unsigned int tx_aborts;
 30};
 31
 32struct kcm_mux_stats {
 33	unsigned long long rx_msgs;
 34	unsigned long long rx_bytes;
 35	unsigned long long tx_msgs;
 36	unsigned long long tx_bytes;
 37	unsigned int rx_ready_drops;
 38	unsigned int tx_retries;
 39	unsigned int psock_attach;
 40	unsigned int psock_unattach_rsvd;
 41	unsigned int psock_unattach;
 42};
 43
 44struct kcm_stats {
 45	unsigned long long rx_msgs;
 46	unsigned long long rx_bytes;
 47	unsigned long long tx_msgs;
 48	unsigned long long tx_bytes;
 49};
 50
 51struct kcm_tx_msg {
 52	unsigned int sent;
 53	unsigned int fragidx;
 54	unsigned int frag_offset;
 55	unsigned int msg_flags;
 56	struct sk_buff *frag_skb;
 57	struct sk_buff *last_skb;
 58};
 59
 60/* Socket structure for KCM client sockets */
 61struct kcm_sock {
 62	struct sock sk;
 63	struct kcm_mux *mux;
 64	struct list_head kcm_sock_list;
 65	int index;
 66	u32 done : 1;
 67	struct work_struct done_work;
 68
 69	struct kcm_stats stats;
 70
 71	/* Transmit */
 72	struct kcm_psock *tx_psock;
 73	struct work_struct tx_work;
 74	struct list_head wait_psock_list;
 75	struct sk_buff *seq_skb;
 76	u32 tx_stopped : 1;
 77
 78	/* Don't use bit fields here, these are set under different locks */
 79	bool tx_wait;
 80	bool tx_wait_more;
 81
 82	/* Receive */
 83	struct kcm_psock *rx_psock;
 84	struct list_head wait_rx_list; /* KCMs waiting for receiving */
 85	bool rx_wait;
 86	u32 rx_disabled : 1;
 87};
 88
 89struct bpf_prog;
 90
 91/* Structure for an attached lower socket */
 92struct kcm_psock {
 93	struct sock *sk;
 94	struct strparser strp;
 95	struct kcm_mux *mux;
 96	int index;
 97
 98	u32 tx_stopped : 1;
 99	u32 done : 1;
100	u32 unattaching : 1;
101
102	void (*save_state_change)(struct sock *sk);
103	void (*save_data_ready)(struct sock *sk);
104	void (*save_write_space)(struct sock *sk);
105
106	struct list_head psock_list;
107
108	struct kcm_psock_stats stats;
109
110	/* Receive */
111	struct list_head psock_ready_list;
112	struct bpf_prog *bpf_prog;
113	struct kcm_sock *rx_kcm;
114	unsigned long long saved_rx_bytes;
115	unsigned long long saved_rx_msgs;
116	struct sk_buff *ready_rx_msg;
117
118	/* Transmit */
119	struct kcm_sock *tx_kcm;
120	struct list_head psock_avail_list;
121	unsigned long long saved_tx_bytes;
122	unsigned long long saved_tx_msgs;
123};
124
125/* Per net MUX list */
126struct kcm_net {
127	struct mutex mutex;
128	struct kcm_psock_stats aggregate_psock_stats;
129	struct kcm_mux_stats aggregate_mux_stats;
130	struct strp_aggr_stats aggregate_strp_stats;
131	struct list_head mux_list;
132	int count;
133};
134
135/* Structure for a MUX */
136struct kcm_mux {
137	struct list_head kcm_mux_list;
138	struct rcu_head rcu;
139	struct kcm_net *knet;
140
141	struct list_head kcm_socks;	/* All KCM sockets on MUX */
142	int kcm_socks_cnt;		/* Total KCM socket count for MUX */
143	struct list_head psocks;	/* List of all psocks on MUX */
144	int psocks_cnt;		/* Total attached sockets */
145
146	struct kcm_mux_stats stats;
147	struct kcm_psock_stats aggregate_psock_stats;
148	struct strp_aggr_stats aggregate_strp_stats;
149
150	/* Receive */
151	spinlock_t rx_lock ____cacheline_aligned_in_smp;
152	struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
153	struct list_head psocks_ready;	/* List of psocks with a msg ready */
154	struct sk_buff_head rx_hold_queue;
155
156	/* Transmit */
157	spinlock_t  lock ____cacheline_aligned_in_smp;	/* TX and mux locking */
158	struct list_head psocks_avail;	/* List of available psocks */
159	struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
160};
161
162#ifdef CONFIG_PROC_FS
163int kcm_proc_init(void);
164void kcm_proc_exit(void);
165#else
166static inline int kcm_proc_init(void) { return 0; }
167static inline void kcm_proc_exit(void) { }
168#endif
169
170static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
171					 struct kcm_psock_stats *agg_stats)
172{
173	/* Save psock statistics in the mux when psock is being unattached. */
174
175#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
176	SAVE_PSOCK_STATS(tx_msgs);
177	SAVE_PSOCK_STATS(tx_bytes);
178	SAVE_PSOCK_STATS(reserved);
179	SAVE_PSOCK_STATS(unreserved);
180	SAVE_PSOCK_STATS(tx_aborts);
181#undef SAVE_PSOCK_STATS
182}
183
184static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
185				       struct kcm_mux_stats *agg_stats)
186{
187	/* Save psock statistics in the mux when psock is being unattached. */
188
189#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
190	SAVE_MUX_STATS(rx_msgs);
191	SAVE_MUX_STATS(rx_bytes);
192	SAVE_MUX_STATS(tx_msgs);
193	SAVE_MUX_STATS(tx_bytes);
194	SAVE_MUX_STATS(rx_ready_drops);
195	SAVE_MUX_STATS(psock_attach);
196	SAVE_MUX_STATS(psock_unattach_rsvd);
197	SAVE_MUX_STATS(psock_unattach);
198#undef SAVE_MUX_STATS
199}
200
201#endif /* __NET_KCM_H_ */