Linux Audio

Check our new training course

Loading...
v6.8
  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Pluggable TCP congestion control support and newReno
  4 * congestion control.
  5 * Based on ideas from I/O scheduler support and Web100.
  6 *
  7 * Copyright (C) 2005 Stephen Hemminger <shemminger@osdl.org>
  8 */
  9
 10#define pr_fmt(fmt) "TCP: " fmt
 11
 12#include <linux/module.h>
 13#include <linux/mm.h>
 14#include <linux/types.h>
 15#include <linux/list.h>
 16#include <linux/gfp.h>
 17#include <linux/jhash.h>
 18#include <net/tcp.h>
 19#include <trace/events/tcp.h>
 20
 21static DEFINE_SPINLOCK(tcp_cong_list_lock);
 22static LIST_HEAD(tcp_cong_list);
 23
 24/* Simple linear search, don't expect many entries! */
 25struct tcp_congestion_ops *tcp_ca_find(const char *name)
 26{
 27	struct tcp_congestion_ops *e;
 28
 29	list_for_each_entry_rcu(e, &tcp_cong_list, list) {
 30		if (strcmp(e->name, name) == 0)
 31			return e;
 32	}
 33
 34	return NULL;
 35}
 36
 37void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
 38{
 39	struct inet_connection_sock *icsk = inet_csk(sk);
 40
 41	trace_tcp_cong_state_set(sk, ca_state);
 42
 43	if (icsk->icsk_ca_ops->set_state)
 44		icsk->icsk_ca_ops->set_state(sk, ca_state);
 45	icsk->icsk_ca_state = ca_state;
 46}
 47
 48/* Must be called with rcu lock held */
 49static struct tcp_congestion_ops *tcp_ca_find_autoload(struct net *net,
 50						       const char *name)
 51{
 52	struct tcp_congestion_ops *ca = tcp_ca_find(name);
 53
 54#ifdef CONFIG_MODULES
 55	if (!ca && capable(CAP_NET_ADMIN)) {
 56		rcu_read_unlock();
 57		request_module("tcp_%s", name);
 58		rcu_read_lock();
 59		ca = tcp_ca_find(name);
 60	}
 61#endif
 62	return ca;
 63}
 64
 65/* Simple linear search, not much in here. */
 66struct tcp_congestion_ops *tcp_ca_find_key(u32 key)
 67{
 68	struct tcp_congestion_ops *e;
 69
 70	list_for_each_entry_rcu(e, &tcp_cong_list, list) {
 71		if (e->key == key)
 72			return e;
 73	}
 74
 75	return NULL;
 76}
 77
 78int tcp_validate_congestion_control(struct tcp_congestion_ops *ca)
 79{
 80	/* all algorithms must implement these */
 81	if (!ca->ssthresh || !ca->undo_cwnd ||
 82	    !(ca->cong_avoid || ca->cong_control)) {
 83		pr_err("%s does not implement required ops\n", ca->name);
 84		return -EINVAL;
 85	}
 86
 87	return 0;
 88}
 89
 90/* Attach new congestion control algorithm to the list
 91 * of available options.
 92 */
 93int tcp_register_congestion_control(struct tcp_congestion_ops *ca)
 94{
 95	int ret;
 96
 97	ret = tcp_validate_congestion_control(ca);
 98	if (ret)
 99		return ret;
100
101	ca->key = jhash(ca->name, sizeof(ca->name), strlen(ca->name));
102
103	spin_lock(&tcp_cong_list_lock);
104	if (ca->key == TCP_CA_UNSPEC || tcp_ca_find_key(ca->key)) {
105		pr_notice("%s already registered or non-unique key\n",
106			  ca->name);
107		ret = -EEXIST;
108	} else {
109		list_add_tail_rcu(&ca->list, &tcp_cong_list);
110		pr_debug("%s registered\n", ca->name);
111	}
112	spin_unlock(&tcp_cong_list_lock);
113
114	return ret;
115}
116EXPORT_SYMBOL_GPL(tcp_register_congestion_control);
117
118/*
119 * Remove congestion control algorithm, called from
120 * the module's remove function.  Module ref counts are used
121 * to ensure that this can't be done till all sockets using
122 * that method are closed.
123 */
124void tcp_unregister_congestion_control(struct tcp_congestion_ops *ca)
125{
126	spin_lock(&tcp_cong_list_lock);
127	list_del_rcu(&ca->list);
128	spin_unlock(&tcp_cong_list_lock);
129
130	/* Wait for outstanding readers to complete before the
131	 * module gets removed entirely.
132	 *
133	 * A try_module_get() should fail by now as our module is
134	 * in "going" state since no refs are held anymore and
135	 * module_exit() handler being called.
136	 */
137	synchronize_rcu();
138}
139EXPORT_SYMBOL_GPL(tcp_unregister_congestion_control);
140
141/* Replace a registered old ca with a new one.
142 *
143 * The new ca must have the same name as the old one, that has been
144 * registered.
145 */
146int tcp_update_congestion_control(struct tcp_congestion_ops *ca, struct tcp_congestion_ops *old_ca)
147{
148	struct tcp_congestion_ops *existing;
149	int ret;
150
151	ret = tcp_validate_congestion_control(ca);
152	if (ret)
153		return ret;
154
155	ca->key = jhash(ca->name, sizeof(ca->name), strlen(ca->name));
156
157	spin_lock(&tcp_cong_list_lock);
158	existing = tcp_ca_find_key(old_ca->key);
159	if (ca->key == TCP_CA_UNSPEC || !existing || strcmp(existing->name, ca->name)) {
160		pr_notice("%s not registered or non-unique key\n",
161			  ca->name);
162		ret = -EINVAL;
163	} else if (existing != old_ca) {
164		pr_notice("invalid old congestion control algorithm to replace\n");
165		ret = -EINVAL;
166	} else {
167		/* Add the new one before removing the old one to keep
168		 * one implementation available all the time.
169		 */
170		list_add_tail_rcu(&ca->list, &tcp_cong_list);
171		list_del_rcu(&existing->list);
172		pr_debug("%s updated\n", ca->name);
173	}
174	spin_unlock(&tcp_cong_list_lock);
175
176	/* Wait for outstanding readers to complete before the
177	 * module or struct_ops gets removed entirely.
178	 */
179	if (!ret)
180		synchronize_rcu();
181
182	return ret;
183}
184
185u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca)
186{
187	const struct tcp_congestion_ops *ca;
188	u32 key = TCP_CA_UNSPEC;
189
190	might_sleep();
191
192	rcu_read_lock();
193	ca = tcp_ca_find_autoload(net, name);
194	if (ca) {
195		key = ca->key;
196		*ecn_ca = ca->flags & TCP_CONG_NEEDS_ECN;
197	}
198	rcu_read_unlock();
199
200	return key;
201}
202
203char *tcp_ca_get_name_by_key(u32 key, char *buffer)
204{
205	const struct tcp_congestion_ops *ca;
206	char *ret = NULL;
207
208	rcu_read_lock();
209	ca = tcp_ca_find_key(key);
210	if (ca)
211		ret = strncpy(buffer, ca->name,
212			      TCP_CA_NAME_MAX);
 
213	rcu_read_unlock();
214
215	return ret;
216}
217
218/* Assign choice of congestion control. */
219void tcp_assign_congestion_control(struct sock *sk)
220{
221	struct net *net = sock_net(sk);
222	struct inet_connection_sock *icsk = inet_csk(sk);
223	const struct tcp_congestion_ops *ca;
224
225	rcu_read_lock();
226	ca = rcu_dereference(net->ipv4.tcp_congestion_control);
227	if (unlikely(!bpf_try_module_get(ca, ca->owner)))
228		ca = &tcp_reno;
229	icsk->icsk_ca_ops = ca;
230	rcu_read_unlock();
231
232	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
233	if (ca->flags & TCP_CONG_NEEDS_ECN)
234		INET_ECN_xmit(sk);
235	else
236		INET_ECN_dontxmit(sk);
237}
238
239void tcp_init_congestion_control(struct sock *sk)
240{
241	struct inet_connection_sock *icsk = inet_csk(sk);
242
243	tcp_sk(sk)->prior_ssthresh = 0;
244	if (icsk->icsk_ca_ops->init)
245		icsk->icsk_ca_ops->init(sk);
246	if (tcp_ca_needs_ecn(sk))
247		INET_ECN_xmit(sk);
248	else
249		INET_ECN_dontxmit(sk);
250	icsk->icsk_ca_initialized = 1;
251}
252
253static void tcp_reinit_congestion_control(struct sock *sk,
254					  const struct tcp_congestion_ops *ca)
255{
256	struct inet_connection_sock *icsk = inet_csk(sk);
257
258	tcp_cleanup_congestion_control(sk);
259	icsk->icsk_ca_ops = ca;
260	icsk->icsk_ca_setsockopt = 1;
261	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
262
263	if (ca->flags & TCP_CONG_NEEDS_ECN)
264		INET_ECN_xmit(sk);
265	else
266		INET_ECN_dontxmit(sk);
267
268	if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
269		tcp_init_congestion_control(sk);
270}
271
272/* Manage refcounts on socket close. */
273void tcp_cleanup_congestion_control(struct sock *sk)
274{
275	struct inet_connection_sock *icsk = inet_csk(sk);
276
277	if (icsk->icsk_ca_ops->release)
278		icsk->icsk_ca_ops->release(sk);
 
279	bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
280}
281
282/* Used by sysctl to change default congestion control */
283int tcp_set_default_congestion_control(struct net *net, const char *name)
284{
285	struct tcp_congestion_ops *ca;
286	const struct tcp_congestion_ops *prev;
287	int ret;
288
289	rcu_read_lock();
290	ca = tcp_ca_find_autoload(net, name);
291	if (!ca) {
292		ret = -ENOENT;
293	} else if (!bpf_try_module_get(ca, ca->owner)) {
294		ret = -EBUSY;
295	} else if (!net_eq(net, &init_net) &&
296			!(ca->flags & TCP_CONG_NON_RESTRICTED)) {
297		/* Only init netns can set default to a restricted algorithm */
298		ret = -EPERM;
299	} else {
300		prev = xchg(&net->ipv4.tcp_congestion_control, ca);
301		if (prev)
302			bpf_module_put(prev, prev->owner);
303
304		ca->flags |= TCP_CONG_NON_RESTRICTED;
305		ret = 0;
306	}
307	rcu_read_unlock();
308
309	return ret;
310}
311
312/* Set default value from kernel configuration at bootup */
313static int __init tcp_congestion_default(void)
314{
315	return tcp_set_default_congestion_control(&init_net,
316						  CONFIG_DEFAULT_TCP_CONG);
317}
318late_initcall(tcp_congestion_default);
319
320/* Build string with list of available congestion control values */
321void tcp_get_available_congestion_control(char *buf, size_t maxlen)
322{
323	struct tcp_congestion_ops *ca;
324	size_t offs = 0;
325
326	rcu_read_lock();
327	list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
328		offs += snprintf(buf + offs, maxlen - offs,
329				 "%s%s",
330				 offs == 0 ? "" : " ", ca->name);
331
332		if (WARN_ON_ONCE(offs >= maxlen))
333			break;
334	}
335	rcu_read_unlock();
336}
337
338/* Get current default congestion control */
339void tcp_get_default_congestion_control(struct net *net, char *name)
340{
341	const struct tcp_congestion_ops *ca;
342
343	rcu_read_lock();
344	ca = rcu_dereference(net->ipv4.tcp_congestion_control);
345	strncpy(name, ca->name, TCP_CA_NAME_MAX);
346	rcu_read_unlock();
347}
348
349/* Built list of non-restricted congestion control values */
350void tcp_get_allowed_congestion_control(char *buf, size_t maxlen)
351{
352	struct tcp_congestion_ops *ca;
353	size_t offs = 0;
354
355	*buf = '\0';
356	rcu_read_lock();
357	list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
358		if (!(ca->flags & TCP_CONG_NON_RESTRICTED))
359			continue;
360		offs += snprintf(buf + offs, maxlen - offs,
361				 "%s%s",
362				 offs == 0 ? "" : " ", ca->name);
363
364		if (WARN_ON_ONCE(offs >= maxlen))
365			break;
366	}
367	rcu_read_unlock();
368}
369
370/* Change list of non-restricted congestion control */
371int tcp_set_allowed_congestion_control(char *val)
372{
373	struct tcp_congestion_ops *ca;
374	char *saved_clone, *clone, *name;
375	int ret = 0;
376
377	saved_clone = clone = kstrdup(val, GFP_USER);
378	if (!clone)
379		return -ENOMEM;
380
381	spin_lock(&tcp_cong_list_lock);
382	/* pass 1 check for bad entries */
383	while ((name = strsep(&clone, " ")) && *name) {
384		ca = tcp_ca_find(name);
385		if (!ca) {
386			ret = -ENOENT;
387			goto out;
388		}
389	}
390
391	/* pass 2 clear old values */
392	list_for_each_entry_rcu(ca, &tcp_cong_list, list)
393		ca->flags &= ~TCP_CONG_NON_RESTRICTED;
394
395	/* pass 3 mark as allowed */
396	while ((name = strsep(&val, " ")) && *name) {
397		ca = tcp_ca_find(name);
398		WARN_ON(!ca);
399		if (ca)
400			ca->flags |= TCP_CONG_NON_RESTRICTED;
401	}
402out:
403	spin_unlock(&tcp_cong_list_lock);
404	kfree(saved_clone);
405
406	return ret;
407}
408
409/* Change congestion control for socket. If load is false, then it is the
410 * responsibility of the caller to call tcp_init_congestion_control or
411 * tcp_reinit_congestion_control (if the current congestion control was
412 * already initialized.
413 */
414int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
415			       bool cap_net_admin)
416{
417	struct inet_connection_sock *icsk = inet_csk(sk);
418	const struct tcp_congestion_ops *ca;
419	int err = 0;
420
421	if (icsk->icsk_ca_dst_locked)
422		return -EPERM;
423
424	rcu_read_lock();
425	if (!load)
426		ca = tcp_ca_find(name);
427	else
428		ca = tcp_ca_find_autoload(sock_net(sk), name);
429
430	/* No change asking for existing value */
431	if (ca == icsk->icsk_ca_ops) {
432		icsk->icsk_ca_setsockopt = 1;
433		goto out;
434	}
435
436	if (!ca)
437		err = -ENOENT;
438	else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) || cap_net_admin))
439		err = -EPERM;
440	else if (!bpf_try_module_get(ca, ca->owner))
441		err = -EBUSY;
442	else
443		tcp_reinit_congestion_control(sk, ca);
444 out:
445	rcu_read_unlock();
446	return err;
447}
448
449/* Slow start is used when congestion window is no greater than the slow start
450 * threshold. We base on RFC2581 and also handle stretch ACKs properly.
451 * We do not implement RFC3465 Appropriate Byte Counting (ABC) per se but
452 * something better;) a packet is only considered (s)acked in its entirety to
453 * defend the ACK attacks described in the RFC. Slow start processes a stretch
454 * ACK of degree N as if N acks of degree 1 are received back to back except
455 * ABC caps N to 2. Slow start exits when cwnd grows over ssthresh and
456 * returns the leftover acks to adjust cwnd in congestion avoidance mode.
457 */
458__bpf_kfunc u32 tcp_slow_start(struct tcp_sock *tp, u32 acked)
459{
460	u32 cwnd = min(tcp_snd_cwnd(tp) + acked, tp->snd_ssthresh);
461
462	acked -= cwnd - tcp_snd_cwnd(tp);
463	tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp));
464
465	return acked;
466}
467EXPORT_SYMBOL_GPL(tcp_slow_start);
468
469/* In theory this is tp->snd_cwnd += 1 / tp->snd_cwnd (or alternative w),
470 * for every packet that was ACKed.
471 */
472__bpf_kfunc void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked)
473{
474	/* If credits accumulated at a higher w, apply them gently now. */
475	if (tp->snd_cwnd_cnt >= w) {
476		tp->snd_cwnd_cnt = 0;
477		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
478	}
479
480	tp->snd_cwnd_cnt += acked;
481	if (tp->snd_cwnd_cnt >= w) {
482		u32 delta = tp->snd_cwnd_cnt / w;
483
484		tp->snd_cwnd_cnt -= delta * w;
485		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + delta);
486	}
487	tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), tp->snd_cwnd_clamp));
488}
489EXPORT_SYMBOL_GPL(tcp_cong_avoid_ai);
490
491/*
492 * TCP Reno congestion control
493 * This is special case used for fallback as well.
494 */
495/* This is Jacobson's slow start and congestion avoidance.
496 * SIGCOMM '88, p. 328.
497 */
498__bpf_kfunc void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked)
499{
500	struct tcp_sock *tp = tcp_sk(sk);
501
502	if (!tcp_is_cwnd_limited(sk))
503		return;
504
505	/* In "safe" area, increase. */
506	if (tcp_in_slow_start(tp)) {
507		acked = tcp_slow_start(tp, acked);
508		if (!acked)
509			return;
510	}
511	/* In dangerous area, increase slowly. */
512	tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked);
513}
514EXPORT_SYMBOL_GPL(tcp_reno_cong_avoid);
515
516/* Slow start threshold is half the congestion window (min 2) */
517__bpf_kfunc u32 tcp_reno_ssthresh(struct sock *sk)
518{
519	const struct tcp_sock *tp = tcp_sk(sk);
520
521	return max(tcp_snd_cwnd(tp) >> 1U, 2U);
522}
523EXPORT_SYMBOL_GPL(tcp_reno_ssthresh);
524
525__bpf_kfunc u32 tcp_reno_undo_cwnd(struct sock *sk)
526{
527	const struct tcp_sock *tp = tcp_sk(sk);
528
529	return max(tcp_snd_cwnd(tp), tp->prior_cwnd);
530}
531EXPORT_SYMBOL_GPL(tcp_reno_undo_cwnd);
532
533struct tcp_congestion_ops tcp_reno = {
534	.flags		= TCP_CONG_NON_RESTRICTED,
535	.name		= "reno",
536	.owner		= THIS_MODULE,
537	.ssthresh	= tcp_reno_ssthresh,
538	.cong_avoid	= tcp_reno_cong_avoid,
539	.undo_cwnd	= tcp_reno_undo_cwnd,
540};
v6.13.7
  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Pluggable TCP congestion control support and newReno
  4 * congestion control.
  5 * Based on ideas from I/O scheduler support and Web100.
  6 *
  7 * Copyright (C) 2005 Stephen Hemminger <shemminger@osdl.org>
  8 */
  9
 10#define pr_fmt(fmt) "TCP: " fmt
 11
 12#include <linux/module.h>
 13#include <linux/mm.h>
 14#include <linux/types.h>
 15#include <linux/list.h>
 16#include <linux/gfp.h>
 17#include <linux/jhash.h>
 18#include <net/tcp.h>
 19#include <trace/events/tcp.h>
 20
 21static DEFINE_SPINLOCK(tcp_cong_list_lock);
 22static LIST_HEAD(tcp_cong_list);
 23
 24/* Simple linear search, don't expect many entries! */
 25struct tcp_congestion_ops *tcp_ca_find(const char *name)
 26{
 27	struct tcp_congestion_ops *e;
 28
 29	list_for_each_entry_rcu(e, &tcp_cong_list, list) {
 30		if (strcmp(e->name, name) == 0)
 31			return e;
 32	}
 33
 34	return NULL;
 35}
 36
 37void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
 38{
 39	struct inet_connection_sock *icsk = inet_csk(sk);
 40
 41	trace_tcp_cong_state_set(sk, ca_state);
 42
 43	if (icsk->icsk_ca_ops->set_state)
 44		icsk->icsk_ca_ops->set_state(sk, ca_state);
 45	icsk->icsk_ca_state = ca_state;
 46}
 47
 48/* Must be called with rcu lock held */
 49static struct tcp_congestion_ops *tcp_ca_find_autoload(const char *name)
 
 50{
 51	struct tcp_congestion_ops *ca = tcp_ca_find(name);
 52
 53#ifdef CONFIG_MODULES
 54	if (!ca && capable(CAP_NET_ADMIN)) {
 55		rcu_read_unlock();
 56		request_module("tcp_%s", name);
 57		rcu_read_lock();
 58		ca = tcp_ca_find(name);
 59	}
 60#endif
 61	return ca;
 62}
 63
 64/* Simple linear search, not much in here. */
 65struct tcp_congestion_ops *tcp_ca_find_key(u32 key)
 66{
 67	struct tcp_congestion_ops *e;
 68
 69	list_for_each_entry_rcu(e, &tcp_cong_list, list) {
 70		if (e->key == key)
 71			return e;
 72	}
 73
 74	return NULL;
 75}
 76
 77int tcp_validate_congestion_control(struct tcp_congestion_ops *ca)
 78{
 79	/* all algorithms must implement these */
 80	if (!ca->ssthresh || !ca->undo_cwnd ||
 81	    !(ca->cong_avoid || ca->cong_control)) {
 82		pr_err("%s does not implement required ops\n", ca->name);
 83		return -EINVAL;
 84	}
 85
 86	return 0;
 87}
 88
 89/* Attach new congestion control algorithm to the list
 90 * of available options.
 91 */
 92int tcp_register_congestion_control(struct tcp_congestion_ops *ca)
 93{
 94	int ret;
 95
 96	ret = tcp_validate_congestion_control(ca);
 97	if (ret)
 98		return ret;
 99
100	ca->key = jhash(ca->name, sizeof(ca->name), strlen(ca->name));
101
102	spin_lock(&tcp_cong_list_lock);
103	if (ca->key == TCP_CA_UNSPEC || tcp_ca_find_key(ca->key)) {
104		pr_notice("%s already registered or non-unique key\n",
105			  ca->name);
106		ret = -EEXIST;
107	} else {
108		list_add_tail_rcu(&ca->list, &tcp_cong_list);
109		pr_debug("%s registered\n", ca->name);
110	}
111	spin_unlock(&tcp_cong_list_lock);
112
113	return ret;
114}
115EXPORT_SYMBOL_GPL(tcp_register_congestion_control);
116
117/*
118 * Remove congestion control algorithm, called from
119 * the module's remove function.  Module ref counts are used
120 * to ensure that this can't be done till all sockets using
121 * that method are closed.
122 */
123void tcp_unregister_congestion_control(struct tcp_congestion_ops *ca)
124{
125	spin_lock(&tcp_cong_list_lock);
126	list_del_rcu(&ca->list);
127	spin_unlock(&tcp_cong_list_lock);
128
129	/* Wait for outstanding readers to complete before the
130	 * module gets removed entirely.
131	 *
132	 * A try_module_get() should fail by now as our module is
133	 * in "going" state since no refs are held anymore and
134	 * module_exit() handler being called.
135	 */
136	synchronize_rcu();
137}
138EXPORT_SYMBOL_GPL(tcp_unregister_congestion_control);
139
140/* Replace a registered old ca with a new one.
141 *
142 * The new ca must have the same name as the old one, that has been
143 * registered.
144 */
145int tcp_update_congestion_control(struct tcp_congestion_ops *ca, struct tcp_congestion_ops *old_ca)
146{
147	struct tcp_congestion_ops *existing;
148	int ret = 0;
 
 
 
 
149
150	ca->key = jhash(ca->name, sizeof(ca->name), strlen(ca->name));
151
152	spin_lock(&tcp_cong_list_lock);
153	existing = tcp_ca_find_key(old_ca->key);
154	if (ca->key == TCP_CA_UNSPEC || !existing || strcmp(existing->name, ca->name)) {
155		pr_notice("%s not registered or non-unique key\n",
156			  ca->name);
157		ret = -EINVAL;
158	} else if (existing != old_ca) {
159		pr_notice("invalid old congestion control algorithm to replace\n");
160		ret = -EINVAL;
161	} else {
162		/* Add the new one before removing the old one to keep
163		 * one implementation available all the time.
164		 */
165		list_add_tail_rcu(&ca->list, &tcp_cong_list);
166		list_del_rcu(&existing->list);
167		pr_debug("%s updated\n", ca->name);
168	}
169	spin_unlock(&tcp_cong_list_lock);
170
171	/* Wait for outstanding readers to complete before the
172	 * module or struct_ops gets removed entirely.
173	 */
174	if (!ret)
175		synchronize_rcu();
176
177	return ret;
178}
179
180u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca)
181{
182	const struct tcp_congestion_ops *ca;
183	u32 key = TCP_CA_UNSPEC;
184
185	might_sleep();
186
187	rcu_read_lock();
188	ca = tcp_ca_find_autoload(name);
189	if (ca) {
190		key = ca->key;
191		*ecn_ca = ca->flags & TCP_CONG_NEEDS_ECN;
192	}
193	rcu_read_unlock();
194
195	return key;
196}
197
198char *tcp_ca_get_name_by_key(u32 key, char *buffer)
199{
200	const struct tcp_congestion_ops *ca;
201	char *ret = NULL;
202
203	rcu_read_lock();
204	ca = tcp_ca_find_key(key);
205	if (ca) {
206		strscpy(buffer, ca->name, TCP_CA_NAME_MAX);
207		ret = buffer;
208	}
209	rcu_read_unlock();
210
211	return ret;
212}
213
214/* Assign choice of congestion control. */
215void tcp_assign_congestion_control(struct sock *sk)
216{
217	struct net *net = sock_net(sk);
218	struct inet_connection_sock *icsk = inet_csk(sk);
219	const struct tcp_congestion_ops *ca;
220
221	rcu_read_lock();
222	ca = rcu_dereference(net->ipv4.tcp_congestion_control);
223	if (unlikely(!bpf_try_module_get(ca, ca->owner)))
224		ca = &tcp_reno;
225	icsk->icsk_ca_ops = ca;
226	rcu_read_unlock();
227
228	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
229	if (ca->flags & TCP_CONG_NEEDS_ECN)
230		INET_ECN_xmit(sk);
231	else
232		INET_ECN_dontxmit(sk);
233}
234
235void tcp_init_congestion_control(struct sock *sk)
236{
237	struct inet_connection_sock *icsk = inet_csk(sk);
238
239	tcp_sk(sk)->prior_ssthresh = 0;
240	if (icsk->icsk_ca_ops->init)
241		icsk->icsk_ca_ops->init(sk);
242	if (tcp_ca_needs_ecn(sk))
243		INET_ECN_xmit(sk);
244	else
245		INET_ECN_dontxmit(sk);
246	icsk->icsk_ca_initialized = 1;
247}
248
249static void tcp_reinit_congestion_control(struct sock *sk,
250					  const struct tcp_congestion_ops *ca)
251{
252	struct inet_connection_sock *icsk = inet_csk(sk);
253
254	tcp_cleanup_congestion_control(sk);
255	icsk->icsk_ca_ops = ca;
256	icsk->icsk_ca_setsockopt = 1;
257	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
258
259	if (ca->flags & TCP_CONG_NEEDS_ECN)
260		INET_ECN_xmit(sk);
261	else
262		INET_ECN_dontxmit(sk);
263
264	if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
265		tcp_init_congestion_control(sk);
266}
267
268/* Manage refcounts on socket close. */
269void tcp_cleanup_congestion_control(struct sock *sk)
270{
271	struct inet_connection_sock *icsk = inet_csk(sk);
272
273	if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release)
274		icsk->icsk_ca_ops->release(sk);
275	icsk->icsk_ca_initialized = 0;
276	bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
277}
278
279/* Used by sysctl to change default congestion control */
280int tcp_set_default_congestion_control(struct net *net, const char *name)
281{
282	struct tcp_congestion_ops *ca;
283	const struct tcp_congestion_ops *prev;
284	int ret;
285
286	rcu_read_lock();
287	ca = tcp_ca_find_autoload(name);
288	if (!ca) {
289		ret = -ENOENT;
290	} else if (!bpf_try_module_get(ca, ca->owner)) {
291		ret = -EBUSY;
292	} else if (!net_eq(net, &init_net) &&
293			!(ca->flags & TCP_CONG_NON_RESTRICTED)) {
294		/* Only init netns can set default to a restricted algorithm */
295		ret = -EPERM;
296	} else {
297		prev = xchg(&net->ipv4.tcp_congestion_control, ca);
298		if (prev)
299			bpf_module_put(prev, prev->owner);
300
301		ca->flags |= TCP_CONG_NON_RESTRICTED;
302		ret = 0;
303	}
304	rcu_read_unlock();
305
306	return ret;
307}
308
309/* Set default value from kernel configuration at bootup */
310static int __init tcp_congestion_default(void)
311{
312	return tcp_set_default_congestion_control(&init_net,
313						  CONFIG_DEFAULT_TCP_CONG);
314}
315late_initcall(tcp_congestion_default);
316
317/* Build string with list of available congestion control values */
318void tcp_get_available_congestion_control(char *buf, size_t maxlen)
319{
320	struct tcp_congestion_ops *ca;
321	size_t offs = 0;
322
323	rcu_read_lock();
324	list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
325		offs += snprintf(buf + offs, maxlen - offs,
326				 "%s%s",
327				 offs == 0 ? "" : " ", ca->name);
328
329		if (WARN_ON_ONCE(offs >= maxlen))
330			break;
331	}
332	rcu_read_unlock();
333}
334
335/* Get current default congestion control */
336void tcp_get_default_congestion_control(struct net *net, char *name)
337{
338	const struct tcp_congestion_ops *ca;
339
340	rcu_read_lock();
341	ca = rcu_dereference(net->ipv4.tcp_congestion_control);
342	strscpy(name, ca->name, TCP_CA_NAME_MAX);
343	rcu_read_unlock();
344}
345
346/* Built list of non-restricted congestion control values */
347void tcp_get_allowed_congestion_control(char *buf, size_t maxlen)
348{
349	struct tcp_congestion_ops *ca;
350	size_t offs = 0;
351
352	*buf = '\0';
353	rcu_read_lock();
354	list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
355		if (!(ca->flags & TCP_CONG_NON_RESTRICTED))
356			continue;
357		offs += snprintf(buf + offs, maxlen - offs,
358				 "%s%s",
359				 offs == 0 ? "" : " ", ca->name);
360
361		if (WARN_ON_ONCE(offs >= maxlen))
362			break;
363	}
364	rcu_read_unlock();
365}
366
367/* Change list of non-restricted congestion control */
368int tcp_set_allowed_congestion_control(char *val)
369{
370	struct tcp_congestion_ops *ca;
371	char *saved_clone, *clone, *name;
372	int ret = 0;
373
374	saved_clone = clone = kstrdup(val, GFP_USER);
375	if (!clone)
376		return -ENOMEM;
377
378	spin_lock(&tcp_cong_list_lock);
379	/* pass 1 check for bad entries */
380	while ((name = strsep(&clone, " ")) && *name) {
381		ca = tcp_ca_find(name);
382		if (!ca) {
383			ret = -ENOENT;
384			goto out;
385		}
386	}
387
388	/* pass 2 clear old values */
389	list_for_each_entry_rcu(ca, &tcp_cong_list, list)
390		ca->flags &= ~TCP_CONG_NON_RESTRICTED;
391
392	/* pass 3 mark as allowed */
393	while ((name = strsep(&val, " ")) && *name) {
394		ca = tcp_ca_find(name);
395		WARN_ON(!ca);
396		if (ca)
397			ca->flags |= TCP_CONG_NON_RESTRICTED;
398	}
399out:
400	spin_unlock(&tcp_cong_list_lock);
401	kfree(saved_clone);
402
403	return ret;
404}
405
406/* Change congestion control for socket. If load is false, then it is the
407 * responsibility of the caller to call tcp_init_congestion_control or
408 * tcp_reinit_congestion_control (if the current congestion control was
409 * already initialized.
410 */
411int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
412			       bool cap_net_admin)
413{
414	struct inet_connection_sock *icsk = inet_csk(sk);
415	const struct tcp_congestion_ops *ca;
416	int err = 0;
417
418	if (icsk->icsk_ca_dst_locked)
419		return -EPERM;
420
421	rcu_read_lock();
422	if (!load)
423		ca = tcp_ca_find(name);
424	else
425		ca = tcp_ca_find_autoload(name);
426
427	/* No change asking for existing value */
428	if (ca == icsk->icsk_ca_ops) {
429		icsk->icsk_ca_setsockopt = 1;
430		goto out;
431	}
432
433	if (!ca)
434		err = -ENOENT;
435	else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) || cap_net_admin))
436		err = -EPERM;
437	else if (!bpf_try_module_get(ca, ca->owner))
438		err = -EBUSY;
439	else
440		tcp_reinit_congestion_control(sk, ca);
441 out:
442	rcu_read_unlock();
443	return err;
444}
445
446/* Slow start is used when congestion window is no greater than the slow start
447 * threshold. We base on RFC2581 and also handle stretch ACKs properly.
448 * We do not implement RFC3465 Appropriate Byte Counting (ABC) per se but
449 * something better;) a packet is only considered (s)acked in its entirety to
450 * defend the ACK attacks described in the RFC. Slow start processes a stretch
451 * ACK of degree N as if N acks of degree 1 are received back to back except
452 * ABC caps N to 2. Slow start exits when cwnd grows over ssthresh and
453 * returns the leftover acks to adjust cwnd in congestion avoidance mode.
454 */
455__bpf_kfunc u32 tcp_slow_start(struct tcp_sock *tp, u32 acked)
456{
457	u32 cwnd = min(tcp_snd_cwnd(tp) + acked, tp->snd_ssthresh);
458
459	acked -= cwnd - tcp_snd_cwnd(tp);
460	tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp));
461
462	return acked;
463}
464EXPORT_SYMBOL_GPL(tcp_slow_start);
465
466/* In theory this is tp->snd_cwnd += 1 / tp->snd_cwnd (or alternative w),
467 * for every packet that was ACKed.
468 */
469__bpf_kfunc void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked)
470{
471	/* If credits accumulated at a higher w, apply them gently now. */
472	if (tp->snd_cwnd_cnt >= w) {
473		tp->snd_cwnd_cnt = 0;
474		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
475	}
476
477	tp->snd_cwnd_cnt += acked;
478	if (tp->snd_cwnd_cnt >= w) {
479		u32 delta = tp->snd_cwnd_cnt / w;
480
481		tp->snd_cwnd_cnt -= delta * w;
482		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + delta);
483	}
484	tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), tp->snd_cwnd_clamp));
485}
486EXPORT_SYMBOL_GPL(tcp_cong_avoid_ai);
487
488/*
489 * TCP Reno congestion control
490 * This is special case used for fallback as well.
491 */
492/* This is Jacobson's slow start and congestion avoidance.
493 * SIGCOMM '88, p. 328.
494 */
495__bpf_kfunc void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked)
496{
497	struct tcp_sock *tp = tcp_sk(sk);
498
499	if (!tcp_is_cwnd_limited(sk))
500		return;
501
502	/* In "safe" area, increase. */
503	if (tcp_in_slow_start(tp)) {
504		acked = tcp_slow_start(tp, acked);
505		if (!acked)
506			return;
507	}
508	/* In dangerous area, increase slowly. */
509	tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked);
510}
511EXPORT_SYMBOL_GPL(tcp_reno_cong_avoid);
512
513/* Slow start threshold is half the congestion window (min 2) */
514__bpf_kfunc u32 tcp_reno_ssthresh(struct sock *sk)
515{
516	const struct tcp_sock *tp = tcp_sk(sk);
517
518	return max(tcp_snd_cwnd(tp) >> 1U, 2U);
519}
520EXPORT_SYMBOL_GPL(tcp_reno_ssthresh);
521
522__bpf_kfunc u32 tcp_reno_undo_cwnd(struct sock *sk)
523{
524	const struct tcp_sock *tp = tcp_sk(sk);
525
526	return max(tcp_snd_cwnd(tp), tp->prior_cwnd);
527}
528EXPORT_SYMBOL_GPL(tcp_reno_undo_cwnd);
529
530struct tcp_congestion_ops tcp_reno = {
531	.flags		= TCP_CONG_NON_RESTRICTED,
532	.name		= "reno",
533	.owner		= THIS_MODULE,
534	.ssthresh	= tcp_reno_ssthresh,
535	.cong_avoid	= tcp_reno_cong_avoid,
536	.undo_cwnd	= tcp_reno_undo_cwnd,
537};