1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2019 Facebook  */
  3
  4#include <linux/init.h>
  5#include <linux/types.h>
  6#include <linux/bpf_verifier.h>
  7#include <linux/bpf.h>
  8#include <linux/btf.h>
  9#include <linux/btf_ids.h>
 10#include <linux/filter.h>
 11#include <net/tcp.h>
 12#include <net/bpf_sk_storage.h>
 13
 14/* "extern" is to avoid sparse warning.  It is only used in bpf_struct_ops.c. */
 15extern struct bpf_struct_ops bpf_tcp_congestion_ops;
 16
 17static u32 unsupported_ops[] = {
 18	offsetof(struct tcp_congestion_ops, get_info),
 19};
 20
 21static const struct btf_type *tcp_sock_type;
 22static u32 tcp_sock_id, sock_id;
 23
 24static int bpf_tcp_ca_init(struct btf *btf)
 25{
 26	s32 type_id;
 27
 28	type_id = btf_find_by_name_kind(btf, "sock", BTF_KIND_STRUCT);
 29	if (type_id < 0)
 30		return -EINVAL;
 31	sock_id = type_id;
 32
 33	type_id = btf_find_by_name_kind(btf, "tcp_sock", BTF_KIND_STRUCT);
 34	if (type_id < 0)
 35		return -EINVAL;
 36	tcp_sock_id = type_id;
 37	tcp_sock_type = btf_type_by_id(btf, tcp_sock_id);
 38
 39	return 0;
 40}
 41
 42static bool is_unsupported(u32 member_offset)
 43{
 44	unsigned int i;
 45
 46	for (i = 0; i < ARRAY_SIZE(unsupported_ops); i++) {
 47		if (member_offset == unsupported_ops[i])
 48			return true;
 49	}
 50
 51	return false;
 52}
 53
 54extern struct btf *btf_vmlinux;
 55
 56static bool bpf_tcp_ca_is_valid_access(int off, int size,
 57				       enum bpf_access_type type,
 58				       const struct bpf_prog *prog,
 59				       struct bpf_insn_access_aux *info)
 60{
 61	if (!bpf_tracing_btf_ctx_access(off, size, type, prog, info))
 62		return false;
 63
 64	if (base_type(info->reg_type) == PTR_TO_BTF_ID &&
 65	    !bpf_type_has_unsafe_modifiers(info->reg_type) &&
 66	    info->btf_id == sock_id)
 67		/* promote it to tcp_sock */
 68		info->btf_id = tcp_sock_id;
 69
 70	return true;
 71}
 72
 73static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
 74					const struct bpf_reg_state *reg,
 75					int off, int size, enum bpf_access_type atype,
 76					u32 *next_btf_id, enum bpf_type_flag *flag)
 77{
 78	const struct btf_type *t;
 79	size_t end;
 80
 81	if (atype == BPF_READ)
 82		return btf_struct_access(log, reg, off, size, atype, next_btf_id, flag);
 83
 84	t = btf_type_by_id(reg->btf, reg->btf_id);
 85	if (t != tcp_sock_type) {
 86		bpf_log(log, "only read is supported\n");
 87		return -EACCES;
 88	}
 89
 90	switch (off) {
 91	case offsetof(struct sock, sk_pacing_rate):
 92		end = offsetofend(struct sock, sk_pacing_rate);
 93		break;
 94	case offsetof(struct sock, sk_pacing_status):
 95		end = offsetofend(struct sock, sk_pacing_status);
 96		break;
 97	case bpf_ctx_range(struct inet_connection_sock, icsk_ca_priv):
 98		end = offsetofend(struct inet_connection_sock, icsk_ca_priv);
 99		break;
100	case offsetof(struct inet_connection_sock, icsk_ack.pending):
101		end = offsetofend(struct inet_connection_sock,
102				  icsk_ack.pending);
103		break;
104	case offsetof(struct tcp_sock, snd_cwnd):
105		end = offsetofend(struct tcp_sock, snd_cwnd);
106		break;
107	case offsetof(struct tcp_sock, snd_cwnd_cnt):
108		end = offsetofend(struct tcp_sock, snd_cwnd_cnt);
109		break;
110	case offsetof(struct tcp_sock, snd_ssthresh):
111		end = offsetofend(struct tcp_sock, snd_ssthresh);
112		break;
113	case offsetof(struct tcp_sock, ecn_flags):
114		end = offsetofend(struct tcp_sock, ecn_flags);
115		break;
116	default:
117		bpf_log(log, "no write support to tcp_sock at off %d\n", off);
118		return -EACCES;
119	}
120
121	if (off + size > end) {
122		bpf_log(log,
123			"write access at off %d with size %d beyond the member of tcp_sock ended at %zu\n",
124			off, size, end);
125		return -EACCES;
126	}
127
128	return 0;
129}
130
131BPF_CALL_2(bpf_tcp_send_ack, struct tcp_sock *, tp, u32, rcv_nxt)
132{
133	/* bpf_tcp_ca prog cannot have NULL tp */
134	__tcp_send_ack((struct sock *)tp, rcv_nxt);
135	return 0;
136}
137
138static const struct bpf_func_proto bpf_tcp_send_ack_proto = {
139	.func		= bpf_tcp_send_ack,
140	.gpl_only	= false,
141	/* In case we want to report error later */
142	.ret_type	= RET_INTEGER,
143	.arg1_type	= ARG_PTR_TO_BTF_ID,
144	.arg1_btf_id	= &tcp_sock_id,
145	.arg2_type	= ARG_ANYTHING,
146};
147
148static u32 prog_ops_moff(const struct bpf_prog *prog)
149{
150	const struct btf_member *m;
151	const struct btf_type *t;
152	u32 midx;
153
154	midx = prog->expected_attach_type;
155	t = bpf_tcp_congestion_ops.type;
156	m = &btf_type_member(t)[midx];
157
158	return __btf_member_bit_offset(t, m) / 8;
159}
160
161static const struct bpf_func_proto *
162bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id,
163			  const struct bpf_prog *prog)
164{
165	switch (func_id) {
166	case BPF_FUNC_tcp_send_ack:
167		return &bpf_tcp_send_ack_proto;
168	case BPF_FUNC_sk_storage_get:
169		return &bpf_sk_storage_get_proto;
170	case BPF_FUNC_sk_storage_delete:
171		return &bpf_sk_storage_delete_proto;
172	case BPF_FUNC_setsockopt:
173		/* Does not allow release() to call setsockopt.
174		 * release() is called when the current bpf-tcp-cc
175		 * is retiring.  It is not allowed to call
176		 * setsockopt() to make further changes which
177		 * may potentially allocate new resources.
178		 */
179		if (prog_ops_moff(prog) !=
180		    offsetof(struct tcp_congestion_ops, release))
181			return &bpf_sk_setsockopt_proto;
182		return NULL;
183	case BPF_FUNC_getsockopt:
184		/* Since get/setsockopt is usually expected to
185		 * be available together, disable getsockopt for
186		 * release also to avoid usage surprise.
187		 * The bpf-tcp-cc already has a more powerful way
188		 * to read tcp_sock from the PTR_TO_BTF_ID.
189		 */
190		if (prog_ops_moff(prog) !=
191		    offsetof(struct tcp_congestion_ops, release))
192			return &bpf_sk_getsockopt_proto;
193		return NULL;
194	case BPF_FUNC_ktime_get_coarse_ns:
195		return &bpf_ktime_get_coarse_ns_proto;
196	default:
197		return bpf_base_func_proto(func_id);
198	}
199}
200
201BTF_SET8_START(bpf_tcp_ca_check_kfunc_ids)
202BTF_ID_FLAGS(func, tcp_reno_ssthresh)
203BTF_ID_FLAGS(func, tcp_reno_cong_avoid)
204BTF_ID_FLAGS(func, tcp_reno_undo_cwnd)
205BTF_ID_FLAGS(func, tcp_slow_start)
206BTF_ID_FLAGS(func, tcp_cong_avoid_ai)
207BTF_SET8_END(bpf_tcp_ca_check_kfunc_ids)
208
209static const struct btf_kfunc_id_set bpf_tcp_ca_kfunc_set = {
210	.owner = THIS_MODULE,
211	.set   = &bpf_tcp_ca_check_kfunc_ids,
212};
213
214static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {
215	.get_func_proto		= bpf_tcp_ca_get_func_proto,
216	.is_valid_access	= bpf_tcp_ca_is_valid_access,
217	.btf_struct_access	= bpf_tcp_ca_btf_struct_access,
218};
219
220static int bpf_tcp_ca_init_member(const struct btf_type *t,
221				  const struct btf_member *member,
222				  void *kdata, const void *udata)
223{
224	const struct tcp_congestion_ops *utcp_ca;
225	struct tcp_congestion_ops *tcp_ca;
226	u32 moff;
227
228	utcp_ca = (const struct tcp_congestion_ops *)udata;
229	tcp_ca = (struct tcp_congestion_ops *)kdata;
230
231	moff = __btf_member_bit_offset(t, member) / 8;
232	switch (moff) {
233	case offsetof(struct tcp_congestion_ops, flags):
234		if (utcp_ca->flags & ~TCP_CONG_MASK)
235			return -EINVAL;
236		tcp_ca->flags = utcp_ca->flags;
237		return 1;
238	case offsetof(struct tcp_congestion_ops, name):
239		if (bpf_obj_name_cpy(tcp_ca->name, utcp_ca->name,
240				     sizeof(tcp_ca->name)) <= 0)
241			return -EINVAL;
242		if (tcp_ca_find(utcp_ca->name))
243			return -EEXIST;
244		return 1;
245	}
246
247	return 0;
248}
249
250static int bpf_tcp_ca_check_member(const struct btf_type *t,
251				   const struct btf_member *member)
252{
253	if (is_unsupported(__btf_member_bit_offset(t, member) / 8))
254		return -ENOTSUPP;
255	return 0;
256}
257
258static int bpf_tcp_ca_reg(void *kdata)
259{
260	return tcp_register_congestion_control(kdata);
261}
262
263static void bpf_tcp_ca_unreg(void *kdata)
264{
265	tcp_unregister_congestion_control(kdata);
266}
267
268struct bpf_struct_ops bpf_tcp_congestion_ops = {
269	.verifier_ops = &bpf_tcp_ca_verifier_ops,
270	.reg = bpf_tcp_ca_reg,
271	.unreg = bpf_tcp_ca_unreg,
272	.check_member = bpf_tcp_ca_check_member,
273	.init_member = bpf_tcp_ca_init_member,
274	.init = bpf_tcp_ca_init,
275	.name = "tcp_congestion_ops",
276};
277
278static int __init bpf_tcp_ca_kfunc_init(void)
279{
280	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_tcp_ca_kfunc_set);
281}
282late_initcall(bpf_tcp_ca_kfunc_init);