Linux Audio

Check our new training course

Loading...
v3.5.6
  1/*
  2 * Copyright (c) 2006 Oracle.  All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 *
 32 */
 33#include <linux/kernel.h>
 34#include <linux/slab.h>
 35#include <linux/in.h>
 36#include <linux/module.h>
 37#include <net/tcp.h>
 
 
 38
 39#include "rds.h"
 40#include "tcp.h"
 41
 42/* only for info exporting */
 43static DEFINE_SPINLOCK(rds_tcp_tc_list_lock);
 44static LIST_HEAD(rds_tcp_tc_list);
 45static unsigned int rds_tcp_tc_count;
 46
 47/* Track rds_tcp_connection structs so they can be cleaned up */
 48static DEFINE_SPINLOCK(rds_tcp_conn_lock);
 49static LIST_HEAD(rds_tcp_conn_list);
 
 50
 51static struct kmem_cache *rds_tcp_conn_slab;
 52
 53#define RDS_TCP_DEFAULT_BUFSIZE (128 * 1024)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 54
 55/* doing it this way avoids calling tcp_sk() */
 56void rds_tcp_nonagle(struct socket *sock)
 57{
 58	mm_segment_t oldfs = get_fs();
 59	int val = 1;
 60
 61	set_fs(KERNEL_DS);
 62	sock->ops->setsockopt(sock, SOL_TCP, TCP_NODELAY, (char __user *)&val,
 63			      sizeof(val));
 64	set_fs(oldfs);
 65}
 66
 67void rds_tcp_tune(struct socket *sock)
 68{
 69	struct sock *sk = sock->sk;
 70
 71	rds_tcp_nonagle(sock);
 72
 73	/*
 74	 * We're trying to saturate gigabit with the default,
 75	 * see svc_sock_setbufsize().
 76	 */
 77	lock_sock(sk);
 78	sk->sk_sndbuf = RDS_TCP_DEFAULT_BUFSIZE;
 79	sk->sk_rcvbuf = RDS_TCP_DEFAULT_BUFSIZE;
 80	sk->sk_userlocks |= SOCK_SNDBUF_LOCK|SOCK_RCVBUF_LOCK;
 81	release_sock(sk);
 82}
 83
 84u32 rds_tcp_snd_nxt(struct rds_tcp_connection *tc)
 85{
 86	return tcp_sk(tc->t_sock->sk)->snd_nxt;
 
 87}
 88
 89u32 rds_tcp_snd_una(struct rds_tcp_connection *tc)
 90{
 91	return tcp_sk(tc->t_sock->sk)->snd_una;
 92}
 93
 94void rds_tcp_restore_callbacks(struct socket *sock,
 95			       struct rds_tcp_connection *tc)
 96{
 97	rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc);
 98	write_lock_bh(&sock->sk->sk_callback_lock);
 99
100	/* done under the callback_lock to serialize with write_space */
101	spin_lock(&rds_tcp_tc_list_lock);
102	list_del_init(&tc->t_list_item);
103	rds_tcp_tc_count--;
104	spin_unlock(&rds_tcp_tc_list_lock);
105
106	tc->t_sock = NULL;
107
108	sock->sk->sk_write_space = tc->t_orig_write_space;
109	sock->sk->sk_data_ready = tc->t_orig_data_ready;
110	sock->sk->sk_state_change = tc->t_orig_state_change;
111	sock->sk->sk_user_data = NULL;
112
113	write_unlock_bh(&sock->sk->sk_callback_lock);
114}
115
116/*
117 * This is the only path that sets tc->t_sock.  Send and receive trust that
118 * it is set.  The RDS_CONN_CONNECTED bit protects those paths from being
119 * called while it isn't set.
 
 
 
 
120 */
121void rds_tcp_set_callbacks(struct socket *sock, struct rds_connection *conn)
 
122{
123	struct rds_tcp_connection *tc = conn->c_transport_data;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
125	rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
126	write_lock_bh(&sock->sk->sk_callback_lock);
127
128	/* done under the callback_lock to serialize with write_space */
129	spin_lock(&rds_tcp_tc_list_lock);
130	list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
131	rds_tcp_tc_count++;
132	spin_unlock(&rds_tcp_tc_list_lock);
133
134	/* accepted sockets need our listen data ready undone */
135	if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
136		sock->sk->sk_data_ready = sock->sk->sk_user_data;
137
138	tc->t_sock = sock;
139	tc->conn = conn;
140	tc->t_orig_data_ready = sock->sk->sk_data_ready;
141	tc->t_orig_write_space = sock->sk->sk_write_space;
142	tc->t_orig_state_change = sock->sk->sk_state_change;
143
144	sock->sk->sk_user_data = conn;
145	sock->sk->sk_data_ready = rds_tcp_data_ready;
146	sock->sk->sk_write_space = rds_tcp_write_space;
147	sock->sk->sk_state_change = rds_tcp_state_change;
148
149	write_unlock_bh(&sock->sk->sk_callback_lock);
150}
151
152static void rds_tcp_tc_info(struct socket *sock, unsigned int len,
153			    struct rds_info_iterator *iter,
154			    struct rds_info_lengths *lens)
155{
156	struct rds_info_tcp_socket tsinfo;
157	struct rds_tcp_connection *tc;
158	unsigned long flags;
159	struct sockaddr_in sin;
160	int sinlen;
161
162	spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);
163
164	if (len / sizeof(tsinfo) < rds_tcp_tc_count)
165		goto out;
166
167	list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
168
169		sock->ops->getname(sock, (struct sockaddr *)&sin, &sinlen, 0);
170		tsinfo.local_addr = sin.sin_addr.s_addr;
171		tsinfo.local_port = sin.sin_port;
172		sock->ops->getname(sock, (struct sockaddr *)&sin, &sinlen, 1);
173		tsinfo.peer_addr = sin.sin_addr.s_addr;
174		tsinfo.peer_port = sin.sin_port;
 
 
 
175
176		tsinfo.hdr_rem = tc->t_tinc_hdr_rem;
177		tsinfo.data_rem = tc->t_tinc_data_rem;
178		tsinfo.last_sent_nxt = tc->t_last_sent_nxt;
179		tsinfo.last_expected_una = tc->t_last_expected_una;
180		tsinfo.last_seen_una = tc->t_last_seen_una;
181
182		rds_info_copy(iter, &tsinfo, sizeof(tsinfo));
183	}
184
185out:
186	lens->nr = rds_tcp_tc_count;
187	lens->each = sizeof(tsinfo);
188
189	spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
190}
191
192static int rds_tcp_laddr_check(__be32 addr)
193{
194	if (inet_addr_type(&init_net, addr) == RTN_LOCAL)
195		return 0;
196	return -EADDRNOTAVAIL;
197}
198
199static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp)
200{
201	struct rds_tcp_connection *tc;
 
202
203	tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp);
204	if (!tc)
205		return -ENOMEM;
206
207	tc->t_sock = NULL;
208	tc->t_tinc = NULL;
209	tc->t_tinc_hdr_rem = sizeof(struct rds_header);
210	tc->t_tinc_data_rem = 0;
 
 
 
 
 
 
 
 
 
211
212	conn->c_transport_data = tc;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
213
 
 
 
214	spin_lock_irq(&rds_tcp_conn_lock);
215	list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list);
 
 
 
 
216	spin_unlock_irq(&rds_tcp_conn_lock);
217
218	rdsdebug("alloced tc %p\n", conn->c_transport_data);
219	return 0;
 
 
 
220}
221
222static void rds_tcp_conn_free(void *arg)
223{
224	struct rds_tcp_connection *tc = arg;
225	unsigned long flags;
226	rdsdebug("freeing tc %p\n", tc);
227
228	spin_lock_irqsave(&rds_tcp_conn_lock, flags);
229	list_del(&tc->t_tcp_node);
230	spin_unlock_irqrestore(&rds_tcp_conn_lock, flags);
 
 
 
231
232	kmem_cache_free(rds_tcp_conn_slab, tc);
 
 
 
 
 
 
 
233}
234
235static void rds_tcp_destroy_conns(void)
236{
237	struct rds_tcp_connection *tc, *_tc;
238	LIST_HEAD(tmp_list);
239
240	/* avoid calling conn_destroy with irqs off */
241	spin_lock_irq(&rds_tcp_conn_lock);
242	list_splice(&rds_tcp_conn_list, &tmp_list);
243	INIT_LIST_HEAD(&rds_tcp_conn_list);
 
 
244	spin_unlock_irq(&rds_tcp_conn_lock);
245
246	list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node) {
247		if (tc->conn->c_passive)
248			rds_conn_destroy(tc->conn->c_passive);
249		rds_conn_destroy(tc->conn);
250	}
251}
252
253static void rds_tcp_exit(void)
254{
255	rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
256	rds_tcp_listen_stop();
257	rds_tcp_destroy_conns();
258	rds_trans_unregister(&rds_tcp_transport);
259	rds_tcp_recv_exit();
260	kmem_cache_destroy(rds_tcp_conn_slab);
261}
262module_exit(rds_tcp_exit);
263
264struct rds_transport rds_tcp_transport = {
265	.laddr_check		= rds_tcp_laddr_check,
266	.xmit_prepare		= rds_tcp_xmit_prepare,
267	.xmit_complete		= rds_tcp_xmit_complete,
268	.xmit			= rds_tcp_xmit,
269	.recv			= rds_tcp_recv,
270	.conn_alloc		= rds_tcp_conn_alloc,
271	.conn_free		= rds_tcp_conn_free,
272	.conn_connect		= rds_tcp_conn_connect,
273	.conn_shutdown		= rds_tcp_conn_shutdown,
274	.inc_copy_to_user	= rds_tcp_inc_copy_to_user,
275	.inc_free		= rds_tcp_inc_free,
276	.stats_info_copy	= rds_tcp_stats_info_copy,
277	.exit			= rds_tcp_exit,
278	.t_owner		= THIS_MODULE,
279	.t_name			= "tcp",
280	.t_type			= RDS_TRANS_TCP,
281	.t_prefer_loopback	= 1,
 
 
282};
283
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
284static int rds_tcp_init(void)
285{
286	int ret;
287
288	rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection",
289					      sizeof(struct rds_tcp_connection),
290					      0, 0, NULL);
291	if (!rds_tcp_conn_slab) {
292		ret = -ENOMEM;
293		goto out;
294	}
295
296	ret = rds_tcp_recv_init();
297	if (ret)
298		goto out_slab;
299
300	ret = rds_trans_register(&rds_tcp_transport);
301	if (ret)
302		goto out_recv;
303
304	ret = rds_tcp_listen_init();
305	if (ret)
306		goto out_register;
307
308	rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
309
310	goto out;
311
312out_register:
313	rds_trans_unregister(&rds_tcp_transport);
314out_recv:
315	rds_tcp_recv_exit();
316out_slab:
317	kmem_cache_destroy(rds_tcp_conn_slab);
318out:
319	return ret;
320}
321module_init(rds_tcp_init);
322
323MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
324MODULE_DESCRIPTION("RDS: TCP transport");
325MODULE_LICENSE("Dual BSD/GPL");
326
v4.17
  1/*
  2 * Copyright (c) 2006 Oracle.  All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 *
 32 */
 33#include <linux/kernel.h>
 34#include <linux/slab.h>
 35#include <linux/in.h>
 36#include <linux/module.h>
 37#include <net/tcp.h>
 38#include <net/net_namespace.h>
 39#include <net/netns/generic.h>
 40
 41#include "rds.h"
 42#include "tcp.h"
 43
 44/* only for info exporting */
 45static DEFINE_SPINLOCK(rds_tcp_tc_list_lock);
 46static LIST_HEAD(rds_tcp_tc_list);
 47static unsigned int rds_tcp_tc_count;
 48
 49/* Track rds_tcp_connection structs so they can be cleaned up */
 50static DEFINE_SPINLOCK(rds_tcp_conn_lock);
 51static LIST_HEAD(rds_tcp_conn_list);
 52static atomic_t rds_tcp_unloading = ATOMIC_INIT(0);
 53
 54static struct kmem_cache *rds_tcp_conn_slab;
 55
 56static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
 57				 void __user *buffer, size_t *lenp,
 58				 loff_t *fpos);
 59
 60static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF;
 61static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF;
 62
 63static struct ctl_table rds_tcp_sysctl_table[] = {
 64#define	RDS_TCP_SNDBUF	0
 65	{
 66		.procname       = "rds_tcp_sndbuf",
 67		/* data is per-net pointer */
 68		.maxlen         = sizeof(int),
 69		.mode           = 0644,
 70		.proc_handler   = rds_tcp_skbuf_handler,
 71		.extra1		= &rds_tcp_min_sndbuf,
 72	},
 73#define	RDS_TCP_RCVBUF	1
 74	{
 75		.procname       = "rds_tcp_rcvbuf",
 76		/* data is per-net pointer */
 77		.maxlen         = sizeof(int),
 78		.mode           = 0644,
 79		.proc_handler   = rds_tcp_skbuf_handler,
 80		.extra1		= &rds_tcp_min_rcvbuf,
 81	},
 82	{ }
 83};
 84
 85/* doing it this way avoids calling tcp_sk() */
 86void rds_tcp_nonagle(struct socket *sock)
 87{
 
 88	int val = 1;
 89
 90	kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&val,
 
 91			      sizeof(val));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 92}
 93
 94u32 rds_tcp_write_seq(struct rds_tcp_connection *tc)
 95{
 96	/* seq# of the last byte of data in tcp send buffer */
 97	return tcp_sk(tc->t_sock->sk)->write_seq;
 98}
 99
100u32 rds_tcp_snd_una(struct rds_tcp_connection *tc)
101{
102	return tcp_sk(tc->t_sock->sk)->snd_una;
103}
104
105void rds_tcp_restore_callbacks(struct socket *sock,
106			       struct rds_tcp_connection *tc)
107{
108	rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc);
109	write_lock_bh(&sock->sk->sk_callback_lock);
110
111	/* done under the callback_lock to serialize with write_space */
112	spin_lock(&rds_tcp_tc_list_lock);
113	list_del_init(&tc->t_list_item);
114	rds_tcp_tc_count--;
115	spin_unlock(&rds_tcp_tc_list_lock);
116
117	tc->t_sock = NULL;
118
119	sock->sk->sk_write_space = tc->t_orig_write_space;
120	sock->sk->sk_data_ready = tc->t_orig_data_ready;
121	sock->sk->sk_state_change = tc->t_orig_state_change;
122	sock->sk->sk_user_data = NULL;
123
124	write_unlock_bh(&sock->sk->sk_callback_lock);
125}
126
127/*
128 * rds_tcp_reset_callbacks() switches the to the new sock and
129 * returns the existing tc->t_sock.
130 *
131 * The only functions that set tc->t_sock are rds_tcp_set_callbacks
132 * and rds_tcp_reset_callbacks.  Send and receive trust that
133 * it is set.  The absence of RDS_CONN_UP bit protects those paths
134 * from being called while it isn't set.
135 */
136void rds_tcp_reset_callbacks(struct socket *sock,
137			     struct rds_conn_path *cp)
138{
139	struct rds_tcp_connection *tc = cp->cp_transport_data;
140	struct socket *osock = tc->t_sock;
141
142	if (!osock)
143		goto newsock;
144
145	/* Need to resolve a duelling SYN between peers.
146	 * We have an outstanding SYN to this peer, which may
147	 * potentially have transitioned to the RDS_CONN_UP state,
148	 * so we must quiesce any send threads before resetting
149	 * cp_transport_data. We quiesce these threads by setting
150	 * cp_state to something other than RDS_CONN_UP, and then
151	 * waiting for any existing threads in rds_send_xmit to
152	 * complete release_in_xmit(). (Subsequent threads entering
153	 * rds_send_xmit() will bail on !rds_conn_up().
154	 *
155	 * However an incoming syn-ack at this point would end up
156	 * marking the conn as RDS_CONN_UP, and would again permit
157	 * rds_send_xmi() threads through, so ideally we would
158	 * synchronize on RDS_CONN_UP after lock_sock(), but cannot
159	 * do that: waiting on !RDS_IN_XMIT after lock_sock() may
160	 * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT
161	 * would not get set. As a result, we set c_state to
162	 * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change
163	 * cannot mark rds_conn_path_up() in the window before lock_sock()
164	 */
165	atomic_set(&cp->cp_state, RDS_CONN_RESETTING);
166	wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags));
167	lock_sock(osock->sk);
168	/* reset receive side state for rds_tcp_data_recv() for osock  */
169	cancel_delayed_work_sync(&cp->cp_send_w);
170	cancel_delayed_work_sync(&cp->cp_recv_w);
171	if (tc->t_tinc) {
172		rds_inc_put(&tc->t_tinc->ti_inc);
173		tc->t_tinc = NULL;
174	}
175	tc->t_tinc_hdr_rem = sizeof(struct rds_header);
176	tc->t_tinc_data_rem = 0;
177	rds_tcp_restore_callbacks(osock, tc);
178	release_sock(osock->sk);
179	sock_release(osock);
180newsock:
181	rds_send_path_reset(cp);
182	lock_sock(sock->sk);
183	rds_tcp_set_callbacks(sock, cp);
184	release_sock(sock->sk);
185}
186
187/* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments
188 * above rds_tcp_reset_callbacks for notes about synchronization
189 * with data path
190 */
191void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
192{
193	struct rds_tcp_connection *tc = cp->cp_transport_data;
194
195	rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
196	write_lock_bh(&sock->sk->sk_callback_lock);
197
198	/* done under the callback_lock to serialize with write_space */
199	spin_lock(&rds_tcp_tc_list_lock);
200	list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
201	rds_tcp_tc_count++;
202	spin_unlock(&rds_tcp_tc_list_lock);
203
204	/* accepted sockets need our listen data ready undone */
205	if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
206		sock->sk->sk_data_ready = sock->sk->sk_user_data;
207
208	tc->t_sock = sock;
209	tc->t_cpath = cp;
210	tc->t_orig_data_ready = sock->sk->sk_data_ready;
211	tc->t_orig_write_space = sock->sk->sk_write_space;
212	tc->t_orig_state_change = sock->sk->sk_state_change;
213
214	sock->sk->sk_user_data = cp;
215	sock->sk->sk_data_ready = rds_tcp_data_ready;
216	sock->sk->sk_write_space = rds_tcp_write_space;
217	sock->sk->sk_state_change = rds_tcp_state_change;
218
219	write_unlock_bh(&sock->sk->sk_callback_lock);
220}
221
222static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len,
223			    struct rds_info_iterator *iter,
224			    struct rds_info_lengths *lens)
225{
226	struct rds_info_tcp_socket tsinfo;
227	struct rds_tcp_connection *tc;
228	unsigned long flags;
229	struct sockaddr_in sin;
230	struct socket *sock;
231
232	spin_lock_irqsave(&rds_tcp_tc_list_lock, flags);
233
234	if (len / sizeof(tsinfo) < rds_tcp_tc_count)
235		goto out;
236
237	list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) {
238
239		sock = tc->t_sock;
240		if (sock) {
241			sock->ops->getname(sock, (struct sockaddr *)&sin, 0);
242			tsinfo.local_addr = sin.sin_addr.s_addr;
243			tsinfo.local_port = sin.sin_port;
244			sock->ops->getname(sock, (struct sockaddr *)&sin, 1);
245			tsinfo.peer_addr = sin.sin_addr.s_addr;
246			tsinfo.peer_port = sin.sin_port;
247		}
248
249		tsinfo.hdr_rem = tc->t_tinc_hdr_rem;
250		tsinfo.data_rem = tc->t_tinc_data_rem;
251		tsinfo.last_sent_nxt = tc->t_last_sent_nxt;
252		tsinfo.last_expected_una = tc->t_last_expected_una;
253		tsinfo.last_seen_una = tc->t_last_seen_una;
254
255		rds_info_copy(iter, &tsinfo, sizeof(tsinfo));
256	}
257
258out:
259	lens->nr = rds_tcp_tc_count;
260	lens->each = sizeof(tsinfo);
261
262	spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags);
263}
264
265static int rds_tcp_laddr_check(struct net *net, __be32 addr)
266{
267	if (inet_addr_type(net, addr) == RTN_LOCAL)
268		return 0;
269	return -EADDRNOTAVAIL;
270}
271
272static void rds_tcp_conn_free(void *arg)
273{
274	struct rds_tcp_connection *tc = arg;
275	unsigned long flags;
276
277	rdsdebug("freeing tc %p\n", tc);
 
 
278
279	spin_lock_irqsave(&rds_tcp_conn_lock, flags);
280	if (!tc->t_tcp_node_detached)
281		list_del(&tc->t_tcp_node);
282	spin_unlock_irqrestore(&rds_tcp_conn_lock, flags);
283
284	kmem_cache_free(rds_tcp_conn_slab, tc);
285}
286
287static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp)
288{
289	struct rds_tcp_connection *tc;
290	int i, j;
291	int ret = 0;
292
293	for (i = 0; i < RDS_MPATH_WORKERS; i++) {
294		tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp);
295		if (!tc) {
296			ret = -ENOMEM;
297			goto fail;
298		}
299		mutex_init(&tc->t_conn_path_lock);
300		tc->t_sock = NULL;
301		tc->t_tinc = NULL;
302		tc->t_tinc_hdr_rem = sizeof(struct rds_header);
303		tc->t_tinc_data_rem = 0;
304
305		conn->c_path[i].cp_transport_data = tc;
306		tc->t_cpath = &conn->c_path[i];
307		tc->t_tcp_node_detached = true;
308
309		rdsdebug("rds_conn_path [%d] tc %p\n", i,
310			 conn->c_path[i].cp_transport_data);
311	}
312	spin_lock_irq(&rds_tcp_conn_lock);
313	for (i = 0; i < RDS_MPATH_WORKERS; i++) {
314		tc = conn->c_path[i].cp_transport_data;
315		tc->t_tcp_node_detached = false;
316		list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list);
317	}
318	spin_unlock_irq(&rds_tcp_conn_lock);
319fail:
320	if (ret) {
321		for (j = 0; j < i; j++)
322			rds_tcp_conn_free(conn->c_path[j].cp_transport_data);
323	}
324	return ret;
325}
326
327static bool list_has_conn(struct list_head *list, struct rds_connection *conn)
328{
329	struct rds_tcp_connection *tc, *_tc;
 
 
330
331	list_for_each_entry_safe(tc, _tc, list, t_tcp_node) {
332		if (tc->t_cpath->cp_conn == conn)
333			return true;
334	}
335	return false;
336}
337
338static void rds_tcp_set_unloading(void)
339{
340	atomic_set(&rds_tcp_unloading, 1);
341}
342
343static bool rds_tcp_is_unloading(struct rds_connection *conn)
344{
345	return atomic_read(&rds_tcp_unloading) != 0;
346}
347
348static void rds_tcp_destroy_conns(void)
349{
350	struct rds_tcp_connection *tc, *_tc;
351	LIST_HEAD(tmp_list);
352
353	/* avoid calling conn_destroy with irqs off */
354	spin_lock_irq(&rds_tcp_conn_lock);
355	list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
356		if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn))
357			list_move_tail(&tc->t_tcp_node, &tmp_list);
358	}
359	spin_unlock_irq(&rds_tcp_conn_lock);
360
361	list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
362		rds_conn_destroy(tc->t_cpath->cp_conn);
 
 
 
363}
364
365static void rds_tcp_exit(void);
 
 
 
 
 
 
 
 
 
366
367struct rds_transport rds_tcp_transport = {
368	.laddr_check		= rds_tcp_laddr_check,
369	.xmit_path_prepare	= rds_tcp_xmit_path_prepare,
370	.xmit_path_complete	= rds_tcp_xmit_path_complete,
371	.xmit			= rds_tcp_xmit,
372	.recv_path		= rds_tcp_recv_path,
373	.conn_alloc		= rds_tcp_conn_alloc,
374	.conn_free		= rds_tcp_conn_free,
375	.conn_path_connect	= rds_tcp_conn_path_connect,
376	.conn_path_shutdown	= rds_tcp_conn_path_shutdown,
377	.inc_copy_to_user	= rds_tcp_inc_copy_to_user,
378	.inc_free		= rds_tcp_inc_free,
379	.stats_info_copy	= rds_tcp_stats_info_copy,
380	.exit			= rds_tcp_exit,
381	.t_owner		= THIS_MODULE,
382	.t_name			= "tcp",
383	.t_type			= RDS_TRANS_TCP,
384	.t_prefer_loopback	= 1,
385	.t_mp_capable		= 1,
386	.t_unloading		= rds_tcp_is_unloading,
387};
388
389static unsigned int rds_tcp_netid;
390
391/* per-network namespace private data for this module */
392struct rds_tcp_net {
393	struct socket *rds_tcp_listen_sock;
394	struct work_struct rds_tcp_accept_w;
395	struct ctl_table_header *rds_tcp_sysctl;
396	struct ctl_table *ctl_table;
397	int sndbuf_size;
398	int rcvbuf_size;
399};
400
401/* All module specific customizations to the RDS-TCP socket should be done in
402 * rds_tcp_tune() and applied after socket creation.
403 */
404void rds_tcp_tune(struct socket *sock)
405{
406	struct sock *sk = sock->sk;
407	struct net *net = sock_net(sk);
408	struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
409
410	rds_tcp_nonagle(sock);
411	lock_sock(sk);
412	if (rtn->sndbuf_size > 0) {
413		sk->sk_sndbuf = rtn->sndbuf_size;
414		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
415	}
416	if (rtn->rcvbuf_size > 0) {
417		sk->sk_sndbuf = rtn->rcvbuf_size;
418		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
419	}
420	release_sock(sk);
421}
422
423static void rds_tcp_accept_worker(struct work_struct *work)
424{
425	struct rds_tcp_net *rtn = container_of(work,
426					       struct rds_tcp_net,
427					       rds_tcp_accept_w);
428
429	while (rds_tcp_accept_one(rtn->rds_tcp_listen_sock) == 0)
430		cond_resched();
431}
432
433void rds_tcp_accept_work(struct sock *sk)
434{
435	struct net *net = sock_net(sk);
436	struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
437
438	queue_work(rds_wq, &rtn->rds_tcp_accept_w);
439}
440
441static __net_init int rds_tcp_init_net(struct net *net)
442{
443	struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
444	struct ctl_table *tbl;
445	int err = 0;
446
447	memset(rtn, 0, sizeof(*rtn));
448
449	/* {snd, rcv}buf_size default to 0, which implies we let the
450	 * stack pick the value, and permit auto-tuning of buffer size.
451	 */
452	if (net == &init_net) {
453		tbl = rds_tcp_sysctl_table;
454	} else {
455		tbl = kmemdup(rds_tcp_sysctl_table,
456			      sizeof(rds_tcp_sysctl_table), GFP_KERNEL);
457		if (!tbl) {
458			pr_warn("could not set allocate syctl table\n");
459			return -ENOMEM;
460		}
461		rtn->ctl_table = tbl;
462	}
463	tbl[RDS_TCP_SNDBUF].data = &rtn->sndbuf_size;
464	tbl[RDS_TCP_RCVBUF].data = &rtn->rcvbuf_size;
465	rtn->rds_tcp_sysctl = register_net_sysctl(net, "net/rds/tcp", tbl);
466	if (!rtn->rds_tcp_sysctl) {
467		pr_warn("could not register sysctl\n");
468		err = -ENOMEM;
469		goto fail;
470	}
471	rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net);
472	if (!rtn->rds_tcp_listen_sock) {
473		pr_warn("could not set up listen sock\n");
474		unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
475		rtn->rds_tcp_sysctl = NULL;
476		err = -EAFNOSUPPORT;
477		goto fail;
478	}
479	INIT_WORK(&rtn->rds_tcp_accept_w, rds_tcp_accept_worker);
480	return 0;
481
482fail:
483	if (net != &init_net)
484		kfree(tbl);
485	return err;
486}
487
488static void rds_tcp_kill_sock(struct net *net)
489{
490	struct rds_tcp_connection *tc, *_tc;
491	LIST_HEAD(tmp_list);
492	struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
493	struct socket *lsock = rtn->rds_tcp_listen_sock;
494
495	rtn->rds_tcp_listen_sock = NULL;
496	rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w);
497	spin_lock_irq(&rds_tcp_conn_lock);
498	list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
499		struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net);
500
501		if (net != c_net || !tc->t_sock)
502			continue;
503		if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) {
504			list_move_tail(&tc->t_tcp_node, &tmp_list);
505		} else {
506			list_del(&tc->t_tcp_node);
507			tc->t_tcp_node_detached = true;
508		}
509	}
510	spin_unlock_irq(&rds_tcp_conn_lock);
511	list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node)
512		rds_conn_destroy(tc->t_cpath->cp_conn);
513}
514
515static void __net_exit rds_tcp_exit_net(struct net *net)
516{
517	struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
518
519	rds_tcp_kill_sock(net);
520
521	if (rtn->rds_tcp_sysctl)
522		unregister_net_sysctl_table(rtn->rds_tcp_sysctl);
523
524	if (net != &init_net && rtn->ctl_table)
525		kfree(rtn->ctl_table);
526}
527
528static struct pernet_operations rds_tcp_net_ops = {
529	.init = rds_tcp_init_net,
530	.exit = rds_tcp_exit_net,
531	.id = &rds_tcp_netid,
532	.size = sizeof(struct rds_tcp_net),
533};
534
535void *rds_tcp_listen_sock_def_readable(struct net *net)
536{
537	struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid);
538	struct socket *lsock = rtn->rds_tcp_listen_sock;
539
540	if (!lsock)
541		return NULL;
542
543	return lsock->sk->sk_user_data;
544}
545
546/* when sysctl is used to modify some kernel socket parameters,this
547 * function  resets the RDS connections in that netns  so that we can
548 * restart with new parameters.  The assumption is that such reset
549 * events are few and far-between.
550 */
551static void rds_tcp_sysctl_reset(struct net *net)
552{
553	struct rds_tcp_connection *tc, *_tc;
554
555	spin_lock_irq(&rds_tcp_conn_lock);
556	list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) {
557		struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net);
558
559		if (net != c_net || !tc->t_sock)
560			continue;
561
562		/* reconnect with new parameters */
563		rds_conn_path_drop(tc->t_cpath, false);
564	}
565	spin_unlock_irq(&rds_tcp_conn_lock);
566}
567
568static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write,
569				 void __user *buffer, size_t *lenp,
570				 loff_t *fpos)
571{
572	struct net *net = current->nsproxy->net_ns;
573	int err;
574
575	err = proc_dointvec_minmax(ctl, write, buffer, lenp, fpos);
576	if (err < 0) {
577		pr_warn("Invalid input. Must be >= %d\n",
578			*(int *)(ctl->extra1));
579		return err;
580	}
581	if (write)
582		rds_tcp_sysctl_reset(net);
583	return 0;
584}
585
586static void rds_tcp_exit(void)
587{
588	rds_tcp_set_unloading();
589	synchronize_rcu();
590	rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
591	unregister_pernet_device(&rds_tcp_net_ops);
592	rds_tcp_destroy_conns();
593	rds_trans_unregister(&rds_tcp_transport);
594	rds_tcp_recv_exit();
595	kmem_cache_destroy(rds_tcp_conn_slab);
596}
597module_exit(rds_tcp_exit);
598
599static int rds_tcp_init(void)
600{
601	int ret;
602
603	rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection",
604					      sizeof(struct rds_tcp_connection),
605					      0, 0, NULL);
606	if (!rds_tcp_conn_slab) {
607		ret = -ENOMEM;
608		goto out;
609	}
610
611	ret = rds_tcp_recv_init();
612	if (ret)
613		goto out_slab;
614
615	ret = register_pernet_device(&rds_tcp_net_ops);
616	if (ret)
617		goto out_recv;
618
619	rds_trans_register(&rds_tcp_transport);
 
 
620
621	rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info);
622
623	goto out;
 
 
 
624out_recv:
625	rds_tcp_recv_exit();
626out_slab:
627	kmem_cache_destroy(rds_tcp_conn_slab);
628out:
629	return ret;
630}
631module_init(rds_tcp_init);
632
633MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
634MODULE_DESCRIPTION("RDS: TCP transport");
635MODULE_LICENSE("Dual BSD/GPL");
636