Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2019 Facebook */
3
4#include <linux/bpf.h>
5#include <linux/bpf_verifier.h>
6#include <linux/btf.h>
7#include <linux/filter.h>
8#include <linux/slab.h>
9#include <linux/numa.h>
10#include <linux/seq_file.h>
11#include <linux/refcount.h>
12#include <linux/mutex.h>
13#include <linux/btf_ids.h>
14
15enum bpf_struct_ops_state {
16 BPF_STRUCT_OPS_STATE_INIT,
17 BPF_STRUCT_OPS_STATE_INUSE,
18 BPF_STRUCT_OPS_STATE_TOBEFREE,
19};
20
21#define BPF_STRUCT_OPS_COMMON_VALUE \
22 refcount_t refcnt; \
23 enum bpf_struct_ops_state state
24
25struct bpf_struct_ops_value {
26 BPF_STRUCT_OPS_COMMON_VALUE;
27 char data[] ____cacheline_aligned_in_smp;
28};
29
30struct bpf_struct_ops_map {
31 struct bpf_map map;
32 struct rcu_head rcu;
33 const struct bpf_struct_ops *st_ops;
34 /* protect map_update */
35 struct mutex lock;
36 /* link has all the bpf_links that is populated
37 * to the func ptr of the kernel's struct
38 * (in kvalue.data).
39 */
40 struct bpf_link **links;
41 /* image is a page that has all the trampolines
42 * that stores the func args before calling the bpf_prog.
43 * A PAGE_SIZE "image" is enough to store all trampoline for
44 * "links[]".
45 */
46 void *image;
47 /* uvalue->data stores the kernel struct
48 * (e.g. tcp_congestion_ops) that is more useful
49 * to userspace than the kvalue. For example,
50 * the bpf_prog's id is stored instead of the kernel
51 * address of a func ptr.
52 */
53 struct bpf_struct_ops_value *uvalue;
54 /* kvalue.data stores the actual kernel's struct
55 * (e.g. tcp_congestion_ops) that will be
56 * registered to the kernel subsystem.
57 */
58 struct bpf_struct_ops_value kvalue;
59};
60
61#define VALUE_PREFIX "bpf_struct_ops_"
62#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
63
64/* bpf_struct_ops_##_name (e.g. bpf_struct_ops_tcp_congestion_ops) is
65 * the map's value exposed to the userspace and its btf-type-id is
66 * stored at the map->btf_vmlinux_value_type_id.
67 *
68 */
69#define BPF_STRUCT_OPS_TYPE(_name) \
70extern struct bpf_struct_ops bpf_##_name; \
71 \
72struct bpf_struct_ops_##_name { \
73 BPF_STRUCT_OPS_COMMON_VALUE; \
74 struct _name data ____cacheline_aligned_in_smp; \
75};
76#include "bpf_struct_ops_types.h"
77#undef BPF_STRUCT_OPS_TYPE
78
79enum {
80#define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name,
81#include "bpf_struct_ops_types.h"
82#undef BPF_STRUCT_OPS_TYPE
83 __NR_BPF_STRUCT_OPS_TYPE,
84};
85
86static struct bpf_struct_ops * const bpf_struct_ops[] = {
87#define BPF_STRUCT_OPS_TYPE(_name) \
88 [BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name,
89#include "bpf_struct_ops_types.h"
90#undef BPF_STRUCT_OPS_TYPE
91};
92
93const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
94};
95
96const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
97#ifdef CONFIG_NET
98 .test_run = bpf_struct_ops_test_run,
99#endif
100};
101
102static const struct btf_type *module_type;
103
104void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log)
105{
106 s32 type_id, value_id, module_id;
107 const struct btf_member *member;
108 struct bpf_struct_ops *st_ops;
109 const struct btf_type *t;
110 char value_name[128];
111 const char *mname;
112 u32 i, j;
113
114 /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */
115#define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name);
116#include "bpf_struct_ops_types.h"
117#undef BPF_STRUCT_OPS_TYPE
118
119 module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT);
120 if (module_id < 0) {
121 pr_warn("Cannot find struct module in btf_vmlinux\n");
122 return;
123 }
124 module_type = btf_type_by_id(btf, module_id);
125
126 for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
127 st_ops = bpf_struct_ops[i];
128
129 if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
130 sizeof(value_name)) {
131 pr_warn("struct_ops name %s is too long\n",
132 st_ops->name);
133 continue;
134 }
135 sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
136
137 value_id = btf_find_by_name_kind(btf, value_name,
138 BTF_KIND_STRUCT);
139 if (value_id < 0) {
140 pr_warn("Cannot find struct %s in btf_vmlinux\n",
141 value_name);
142 continue;
143 }
144
145 type_id = btf_find_by_name_kind(btf, st_ops->name,
146 BTF_KIND_STRUCT);
147 if (type_id < 0) {
148 pr_warn("Cannot find struct %s in btf_vmlinux\n",
149 st_ops->name);
150 continue;
151 }
152 t = btf_type_by_id(btf, type_id);
153 if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
154 pr_warn("Cannot support #%u members in struct %s\n",
155 btf_type_vlen(t), st_ops->name);
156 continue;
157 }
158
159 for_each_member(j, t, member) {
160 const struct btf_type *func_proto;
161
162 mname = btf_name_by_offset(btf, member->name_off);
163 if (!*mname) {
164 pr_warn("anon member in struct %s is not supported\n",
165 st_ops->name);
166 break;
167 }
168
169 if (__btf_member_bitfield_size(t, member)) {
170 pr_warn("bit field member %s in struct %s is not supported\n",
171 mname, st_ops->name);
172 break;
173 }
174
175 func_proto = btf_type_resolve_func_ptr(btf,
176 member->type,
177 NULL);
178 if (func_proto &&
179 btf_distill_func_proto(log, btf,
180 func_proto, mname,
181 &st_ops->func_models[j])) {
182 pr_warn("Error in parsing func ptr %s in struct %s\n",
183 mname, st_ops->name);
184 break;
185 }
186 }
187
188 if (j == btf_type_vlen(t)) {
189 if (st_ops->init(btf)) {
190 pr_warn("Error in init bpf_struct_ops %s\n",
191 st_ops->name);
192 } else {
193 st_ops->type_id = type_id;
194 st_ops->type = t;
195 st_ops->value_id = value_id;
196 st_ops->value_type = btf_type_by_id(btf,
197 value_id);
198 }
199 }
200 }
201}
202
203extern struct btf *btf_vmlinux;
204
205static const struct bpf_struct_ops *
206bpf_struct_ops_find_value(u32 value_id)
207{
208 unsigned int i;
209
210 if (!value_id || !btf_vmlinux)
211 return NULL;
212
213 for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
214 if (bpf_struct_ops[i]->value_id == value_id)
215 return bpf_struct_ops[i];
216 }
217
218 return NULL;
219}
220
221const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
222{
223 unsigned int i;
224
225 if (!type_id || !btf_vmlinux)
226 return NULL;
227
228 for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
229 if (bpf_struct_ops[i]->type_id == type_id)
230 return bpf_struct_ops[i];
231 }
232
233 return NULL;
234}
235
236static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
237 void *next_key)
238{
239 if (key && *(u32 *)key == 0)
240 return -ENOENT;
241
242 *(u32 *)next_key = 0;
243 return 0;
244}
245
246int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
247 void *value)
248{
249 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
250 struct bpf_struct_ops_value *uvalue, *kvalue;
251 enum bpf_struct_ops_state state;
252
253 if (unlikely(*(u32 *)key != 0))
254 return -ENOENT;
255
256 kvalue = &st_map->kvalue;
257 /* Pair with smp_store_release() during map_update */
258 state = smp_load_acquire(&kvalue->state);
259 if (state == BPF_STRUCT_OPS_STATE_INIT) {
260 memset(value, 0, map->value_size);
261 return 0;
262 }
263
264 /* No lock is needed. state and refcnt do not need
265 * to be updated together under atomic context.
266 */
267 uvalue = value;
268 memcpy(uvalue, st_map->uvalue, map->value_size);
269 uvalue->state = state;
270 refcount_set(&uvalue->refcnt, refcount_read(&kvalue->refcnt));
271
272 return 0;
273}
274
275static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
276{
277 return ERR_PTR(-EINVAL);
278}
279
280static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
281{
282 const struct btf_type *t = st_map->st_ops->type;
283 u32 i;
284
285 for (i = 0; i < btf_type_vlen(t); i++) {
286 if (st_map->links[i]) {
287 bpf_link_put(st_map->links[i]);
288 st_map->links[i] = NULL;
289 }
290 }
291}
292
293static int check_zero_holes(const struct btf_type *t, void *data)
294{
295 const struct btf_member *member;
296 u32 i, moff, msize, prev_mend = 0;
297 const struct btf_type *mtype;
298
299 for_each_member(i, t, member) {
300 moff = __btf_member_bit_offset(t, member) / 8;
301 if (moff > prev_mend &&
302 memchr_inv(data + prev_mend, 0, moff - prev_mend))
303 return -EINVAL;
304
305 mtype = btf_type_by_id(btf_vmlinux, member->type);
306 mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
307 if (IS_ERR(mtype))
308 return PTR_ERR(mtype);
309 prev_mend = moff + msize;
310 }
311
312 if (t->size > prev_mend &&
313 memchr_inv(data + prev_mend, 0, t->size - prev_mend))
314 return -EINVAL;
315
316 return 0;
317}
318
319static void bpf_struct_ops_link_release(struct bpf_link *link)
320{
321}
322
323static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
324{
325 struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
326
327 kfree(tlink);
328}
329
330const struct bpf_link_ops bpf_struct_ops_link_lops = {
331 .release = bpf_struct_ops_link_release,
332 .dealloc = bpf_struct_ops_link_dealloc,
333};
334
335int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
336 struct bpf_tramp_link *link,
337 const struct btf_func_model *model,
338 void *image, void *image_end)
339{
340 u32 flags;
341
342 tlinks[BPF_TRAMP_FENTRY].links[0] = link;
343 tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
344 /* BPF_TRAMP_F_RET_FENTRY_RET is only used by bpf_struct_ops,
345 * and it must be used alone.
346 */
347 flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0;
348 return arch_prepare_bpf_trampoline(NULL, image, image_end,
349 model, flags, tlinks, NULL);
350}
351
352static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
353 void *value, u64 flags)
354{
355 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
356 const struct bpf_struct_ops *st_ops = st_map->st_ops;
357 struct bpf_struct_ops_value *uvalue, *kvalue;
358 const struct btf_member *member;
359 const struct btf_type *t = st_ops->type;
360 struct bpf_tramp_links *tlinks = NULL;
361 void *udata, *kdata;
362 int prog_fd, err = 0;
363 void *image, *image_end;
364 u32 i;
365
366 if (flags)
367 return -EINVAL;
368
369 if (*(u32 *)key != 0)
370 return -E2BIG;
371
372 err = check_zero_holes(st_ops->value_type, value);
373 if (err)
374 return err;
375
376 uvalue = value;
377 err = check_zero_holes(t, uvalue->data);
378 if (err)
379 return err;
380
381 if (uvalue->state || refcount_read(&uvalue->refcnt))
382 return -EINVAL;
383
384 tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
385 if (!tlinks)
386 return -ENOMEM;
387
388 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
389 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
390
391 mutex_lock(&st_map->lock);
392
393 if (kvalue->state != BPF_STRUCT_OPS_STATE_INIT) {
394 err = -EBUSY;
395 goto unlock;
396 }
397
398 memcpy(uvalue, value, map->value_size);
399
400 udata = &uvalue->data;
401 kdata = &kvalue->data;
402 image = st_map->image;
403 image_end = st_map->image + PAGE_SIZE;
404
405 for_each_member(i, t, member) {
406 const struct btf_type *mtype, *ptype;
407 struct bpf_prog *prog;
408 struct bpf_tramp_link *link;
409 u32 moff;
410
411 moff = __btf_member_bit_offset(t, member) / 8;
412 ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
413 if (ptype == module_type) {
414 if (*(void **)(udata + moff))
415 goto reset_unlock;
416 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
417 continue;
418 }
419
420 err = st_ops->init_member(t, member, kdata, udata);
421 if (err < 0)
422 goto reset_unlock;
423
424 /* The ->init_member() has handled this member */
425 if (err > 0)
426 continue;
427
428 /* If st_ops->init_member does not handle it,
429 * we will only handle func ptrs and zero-ed members
430 * here. Reject everything else.
431 */
432
433 /* All non func ptr member must be 0 */
434 if (!ptype || !btf_type_is_func_proto(ptype)) {
435 u32 msize;
436
437 mtype = btf_type_by_id(btf_vmlinux, member->type);
438 mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
439 if (IS_ERR(mtype)) {
440 err = PTR_ERR(mtype);
441 goto reset_unlock;
442 }
443
444 if (memchr_inv(udata + moff, 0, msize)) {
445 err = -EINVAL;
446 goto reset_unlock;
447 }
448
449 continue;
450 }
451
452 prog_fd = (int)(*(unsigned long *)(udata + moff));
453 /* Similar check as the attr->attach_prog_fd */
454 if (!prog_fd)
455 continue;
456
457 prog = bpf_prog_get(prog_fd);
458 if (IS_ERR(prog)) {
459 err = PTR_ERR(prog);
460 goto reset_unlock;
461 }
462
463 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
464 prog->aux->attach_btf_id != st_ops->type_id ||
465 prog->expected_attach_type != i) {
466 bpf_prog_put(prog);
467 err = -EINVAL;
468 goto reset_unlock;
469 }
470
471 link = kzalloc(sizeof(*link), GFP_USER);
472 if (!link) {
473 bpf_prog_put(prog);
474 err = -ENOMEM;
475 goto reset_unlock;
476 }
477 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
478 &bpf_struct_ops_link_lops, prog);
479 st_map->links[i] = &link->link;
480
481 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
482 &st_ops->func_models[i],
483 image, image_end);
484 if (err < 0)
485 goto reset_unlock;
486
487 *(void **)(kdata + moff) = image;
488 image += err;
489
490 /* put prog_id to udata */
491 *(unsigned long *)(udata + moff) = prog->aux->id;
492 }
493
494 refcount_set(&kvalue->refcnt, 1);
495 bpf_map_inc(map);
496
497 set_memory_rox((long)st_map->image, 1);
498 err = st_ops->reg(kdata);
499 if (likely(!err)) {
500 /* Pair with smp_load_acquire() during lookup_elem().
501 * It ensures the above udata updates (e.g. prog->aux->id)
502 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
503 */
504 smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_INUSE);
505 goto unlock;
506 }
507
508 /* Error during st_ops->reg(). Can happen if this struct_ops needs to be
509 * verified as a whole, after all init_member() calls. Can also happen if
510 * there was a race in registering the struct_ops (under the same name) to
511 * a sub-system through different struct_ops's maps.
512 */
513 set_memory_nx((long)st_map->image, 1);
514 set_memory_rw((long)st_map->image, 1);
515 bpf_map_put(map);
516
517reset_unlock:
518 bpf_struct_ops_map_put_progs(st_map);
519 memset(uvalue, 0, map->value_size);
520 memset(kvalue, 0, map->value_size);
521unlock:
522 kfree(tlinks);
523 mutex_unlock(&st_map->lock);
524 return err;
525}
526
527static int bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
528{
529 enum bpf_struct_ops_state prev_state;
530 struct bpf_struct_ops_map *st_map;
531
532 st_map = (struct bpf_struct_ops_map *)map;
533 prev_state = cmpxchg(&st_map->kvalue.state,
534 BPF_STRUCT_OPS_STATE_INUSE,
535 BPF_STRUCT_OPS_STATE_TOBEFREE);
536 switch (prev_state) {
537 case BPF_STRUCT_OPS_STATE_INUSE:
538 st_map->st_ops->unreg(&st_map->kvalue.data);
539 if (refcount_dec_and_test(&st_map->kvalue.refcnt))
540 bpf_map_put(map);
541 return 0;
542 case BPF_STRUCT_OPS_STATE_TOBEFREE:
543 return -EINPROGRESS;
544 case BPF_STRUCT_OPS_STATE_INIT:
545 return -ENOENT;
546 default:
547 WARN_ON_ONCE(1);
548 /* Should never happen. Treat it as not found. */
549 return -ENOENT;
550 }
551}
552
553static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
554 struct seq_file *m)
555{
556 void *value;
557 int err;
558
559 value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
560 if (!value)
561 return;
562
563 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
564 if (!err) {
565 btf_type_seq_show(btf_vmlinux, map->btf_vmlinux_value_type_id,
566 value, m);
567 seq_puts(m, "\n");
568 }
569
570 kfree(value);
571}
572
573static void bpf_struct_ops_map_free(struct bpf_map *map)
574{
575 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
576
577 if (st_map->links)
578 bpf_struct_ops_map_put_progs(st_map);
579 bpf_map_area_free(st_map->links);
580 bpf_jit_free_exec(st_map->image);
581 bpf_map_area_free(st_map->uvalue);
582 bpf_map_area_free(st_map);
583}
584
585static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
586{
587 if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
588 attr->map_flags || !attr->btf_vmlinux_value_type_id)
589 return -EINVAL;
590 return 0;
591}
592
593static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
594{
595 const struct bpf_struct_ops *st_ops;
596 size_t st_map_size;
597 struct bpf_struct_ops_map *st_map;
598 const struct btf_type *t, *vt;
599 struct bpf_map *map;
600
601 if (!bpf_capable())
602 return ERR_PTR(-EPERM);
603
604 st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id);
605 if (!st_ops)
606 return ERR_PTR(-ENOTSUPP);
607
608 vt = st_ops->value_type;
609 if (attr->value_size != vt->size)
610 return ERR_PTR(-EINVAL);
611
612 t = st_ops->type;
613
614 st_map_size = sizeof(*st_map) +
615 /* kvalue stores the
616 * struct bpf_struct_ops_tcp_congestions_ops
617 */
618 (vt->size - sizeof(struct bpf_struct_ops_value));
619
620 st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
621 if (!st_map)
622 return ERR_PTR(-ENOMEM);
623
624 st_map->st_ops = st_ops;
625 map = &st_map->map;
626
627 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
628 st_map->links =
629 bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *),
630 NUMA_NO_NODE);
631 st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
632 if (!st_map->uvalue || !st_map->links || !st_map->image) {
633 bpf_struct_ops_map_free(map);
634 return ERR_PTR(-ENOMEM);
635 }
636
637 mutex_init(&st_map->lock);
638 set_vm_flush_reset_perms(st_map->image);
639 bpf_map_init_from_attr(map, attr);
640
641 return map;
642}
643
644BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map)
645const struct bpf_map_ops bpf_struct_ops_map_ops = {
646 .map_alloc_check = bpf_struct_ops_map_alloc_check,
647 .map_alloc = bpf_struct_ops_map_alloc,
648 .map_free = bpf_struct_ops_map_free,
649 .map_get_next_key = bpf_struct_ops_map_get_next_key,
650 .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
651 .map_delete_elem = bpf_struct_ops_map_delete_elem,
652 .map_update_elem = bpf_struct_ops_map_update_elem,
653 .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
654 .map_btf_id = &bpf_struct_ops_map_btf_ids[0],
655};
656
657/* "const void *" because some subsystem is
658 * passing a const (e.g. const struct tcp_congestion_ops *)
659 */
660bool bpf_struct_ops_get(const void *kdata)
661{
662 struct bpf_struct_ops_value *kvalue;
663
664 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
665
666 return refcount_inc_not_zero(&kvalue->refcnt);
667}
668
669static void bpf_struct_ops_put_rcu(struct rcu_head *head)
670{
671 struct bpf_struct_ops_map *st_map;
672
673 st_map = container_of(head, struct bpf_struct_ops_map, rcu);
674 bpf_map_put(&st_map->map);
675}
676
677void bpf_struct_ops_put(const void *kdata)
678{
679 struct bpf_struct_ops_value *kvalue;
680
681 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
682 if (refcount_dec_and_test(&kvalue->refcnt)) {
683 struct bpf_struct_ops_map *st_map;
684
685 st_map = container_of(kvalue, struct bpf_struct_ops_map,
686 kvalue);
687 /* The struct_ops's function may switch to another struct_ops.
688 *
689 * For example, bpf_tcp_cc_x->init() may switch to
690 * another tcp_cc_y by calling
691 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
692 * During the switch, bpf_struct_ops_put(tcp_cc_x) is called
693 * and its map->refcnt may reach 0 which then free its
694 * trampoline image while tcp_cc_x is still running.
695 *
696 * Thus, a rcu grace period is needed here.
697 */
698 call_rcu(&st_map->rcu, bpf_struct_ops_put_rcu);
699 }
700}
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2019 Facebook */
3
4#include <linux/bpf.h>
5#include <linux/bpf_verifier.h>
6#include <linux/btf.h>
7#include <linux/filter.h>
8#include <linux/slab.h>
9#include <linux/numa.h>
10#include <linux/seq_file.h>
11#include <linux/refcount.h>
12#include <linux/mutex.h>
13#include <linux/btf_ids.h>
14#include <linux/rcupdate_wait.h>
15
16struct bpf_struct_ops_value {
17 struct bpf_struct_ops_common_value common;
18 char data[] ____cacheline_aligned_in_smp;
19};
20
21#define MAX_TRAMP_IMAGE_PAGES 8
22
23struct bpf_struct_ops_map {
24 struct bpf_map map;
25 struct rcu_head rcu;
26 const struct bpf_struct_ops_desc *st_ops_desc;
27 /* protect map_update */
28 struct mutex lock;
29 /* link has all the bpf_links that is populated
30 * to the func ptr of the kernel's struct
31 * (in kvalue.data).
32 */
33 struct bpf_link **links;
34 u32 links_cnt;
35 u32 image_pages_cnt;
36 /* image_pages is an array of pages that has all the trampolines
37 * that stores the func args before calling the bpf_prog.
38 */
39 void *image_pages[MAX_TRAMP_IMAGE_PAGES];
40 /* The owner moduler's btf. */
41 struct btf *btf;
42 /* uvalue->data stores the kernel struct
43 * (e.g. tcp_congestion_ops) that is more useful
44 * to userspace than the kvalue. For example,
45 * the bpf_prog's id is stored instead of the kernel
46 * address of a func ptr.
47 */
48 struct bpf_struct_ops_value *uvalue;
49 /* kvalue.data stores the actual kernel's struct
50 * (e.g. tcp_congestion_ops) that will be
51 * registered to the kernel subsystem.
52 */
53 struct bpf_struct_ops_value kvalue;
54};
55
56struct bpf_struct_ops_link {
57 struct bpf_link link;
58 struct bpf_map __rcu *map;
59};
60
61static DEFINE_MUTEX(update_mutex);
62
63#define VALUE_PREFIX "bpf_struct_ops_"
64#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
65
66const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
67};
68
69const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
70#ifdef CONFIG_NET
71 .test_run = bpf_struct_ops_test_run,
72#endif
73};
74
75BTF_ID_LIST(st_ops_ids)
76BTF_ID(struct, module)
77BTF_ID(struct, bpf_struct_ops_common_value)
78
79enum {
80 IDX_MODULE_ID,
81 IDX_ST_OPS_COMMON_VALUE_ID,
82};
83
84extern struct btf *btf_vmlinux;
85
86static bool is_valid_value_type(struct btf *btf, s32 value_id,
87 const struct btf_type *type,
88 const char *value_name)
89{
90 const struct btf_type *common_value_type;
91 const struct btf_member *member;
92 const struct btf_type *vt, *mt;
93
94 vt = btf_type_by_id(btf, value_id);
95 if (btf_vlen(vt) != 2) {
96 pr_warn("The number of %s's members should be 2, but we get %d\n",
97 value_name, btf_vlen(vt));
98 return false;
99 }
100 member = btf_type_member(vt);
101 mt = btf_type_by_id(btf, member->type);
102 common_value_type = btf_type_by_id(btf_vmlinux,
103 st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]);
104 if (mt != common_value_type) {
105 pr_warn("The first member of %s should be bpf_struct_ops_common_value\n",
106 value_name);
107 return false;
108 }
109 member++;
110 mt = btf_type_by_id(btf, member->type);
111 if (mt != type) {
112 pr_warn("The second member of %s should be %s\n",
113 value_name, btf_name_by_offset(btf, type->name_off));
114 return false;
115 }
116
117 return true;
118}
119
120static void *bpf_struct_ops_image_alloc(void)
121{
122 void *image;
123 int err;
124
125 err = bpf_jit_charge_modmem(PAGE_SIZE);
126 if (err)
127 return ERR_PTR(err);
128 image = arch_alloc_bpf_trampoline(PAGE_SIZE);
129 if (!image) {
130 bpf_jit_uncharge_modmem(PAGE_SIZE);
131 return ERR_PTR(-ENOMEM);
132 }
133
134 return image;
135}
136
137void bpf_struct_ops_image_free(void *image)
138{
139 if (image) {
140 arch_free_bpf_trampoline(image, PAGE_SIZE);
141 bpf_jit_uncharge_modmem(PAGE_SIZE);
142 }
143}
144
145#define MAYBE_NULL_SUFFIX "__nullable"
146#define MAX_STUB_NAME 128
147
148/* Return the type info of a stub function, if it exists.
149 *
150 * The name of a stub function is made up of the name of the struct_ops and
151 * the name of the function pointer member, separated by "__". For example,
152 * if the struct_ops type is named "foo_ops" and the function pointer
153 * member is named "bar", the stub function name would be "foo_ops__bar".
154 */
155static const struct btf_type *
156find_stub_func_proto(const struct btf *btf, const char *st_op_name,
157 const char *member_name)
158{
159 char stub_func_name[MAX_STUB_NAME];
160 const struct btf_type *func_type;
161 s32 btf_id;
162 int cp;
163
164 cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
165 st_op_name, member_name);
166 if (cp >= MAX_STUB_NAME) {
167 pr_warn("Stub function name too long\n");
168 return NULL;
169 }
170 btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
171 if (btf_id < 0)
172 return NULL;
173 func_type = btf_type_by_id(btf, btf_id);
174 if (!func_type)
175 return NULL;
176
177 return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */
178}
179
180/* Prepare argument info for every nullable argument of a member of a
181 * struct_ops type.
182 *
183 * Initialize a struct bpf_struct_ops_arg_info according to type info of
184 * the arguments of a stub function. (Check kCFI for more information about
185 * stub functions.)
186 *
187 * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info
188 * to provide an array of struct bpf_ctx_arg_aux, which in turn provides
189 * the information that used by the verifier to check the arguments of the
190 * BPF struct_ops program assigned to the member. Here, we only care about
191 * the arguments that are marked as __nullable.
192 *
193 * The array of struct bpf_ctx_arg_aux is eventually assigned to
194 * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the
195 * verifier. (See check_struct_ops_btf_id())
196 *
197 * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If
198 * fails, it will be kept untouched.
199 */
200static int prepare_arg_info(struct btf *btf,
201 const char *st_ops_name,
202 const char *member_name,
203 const struct btf_type *func_proto,
204 struct bpf_struct_ops_arg_info *arg_info)
205{
206 const struct btf_type *stub_func_proto, *pointed_type;
207 const struct btf_param *stub_args, *args;
208 struct bpf_ctx_arg_aux *info, *info_buf;
209 u32 nargs, arg_no, info_cnt = 0;
210 u32 arg_btf_id;
211 int offset;
212
213 stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name);
214 if (!stub_func_proto)
215 return 0;
216
217 /* Check if the number of arguments of the stub function is the same
218 * as the number of arguments of the function pointer.
219 */
220 nargs = btf_type_vlen(func_proto);
221 if (nargs != btf_type_vlen(stub_func_proto)) {
222 pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n",
223 st_ops_name, member_name, member_name, st_ops_name);
224 return -EINVAL;
225 }
226
227 if (!nargs)
228 return 0;
229
230 args = btf_params(func_proto);
231 stub_args = btf_params(stub_func_proto);
232
233 info_buf = kcalloc(nargs, sizeof(*info_buf), GFP_KERNEL);
234 if (!info_buf)
235 return -ENOMEM;
236
237 /* Prepare info for every nullable argument */
238 info = info_buf;
239 for (arg_no = 0; arg_no < nargs; arg_no++) {
240 /* Skip arguments that is not suffixed with
241 * "__nullable".
242 */
243 if (!btf_param_match_suffix(btf, &stub_args[arg_no],
244 MAYBE_NULL_SUFFIX))
245 continue;
246
247 /* Should be a pointer to struct */
248 pointed_type = btf_type_resolve_ptr(btf,
249 args[arg_no].type,
250 &arg_btf_id);
251 if (!pointed_type ||
252 !btf_type_is_struct(pointed_type)) {
253 pr_warn("stub function %s__%s has %s tagging to an unsupported type\n",
254 st_ops_name, member_name, MAYBE_NULL_SUFFIX);
255 goto err_out;
256 }
257
258 offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
259 if (offset < 0) {
260 pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n",
261 st_ops_name, member_name, arg_no);
262 goto err_out;
263 }
264
265 if (args[arg_no].type != stub_args[arg_no].type) {
266 pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n",
267 arg_no, st_ops_name, member_name);
268 goto err_out;
269 }
270
271 /* Fill the information of the new argument */
272 info->reg_type =
273 PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL;
274 info->btf_id = arg_btf_id;
275 info->btf = btf;
276 info->offset = offset;
277
278 info++;
279 info_cnt++;
280 }
281
282 if (info_cnt) {
283 arg_info->info = info_buf;
284 arg_info->cnt = info_cnt;
285 } else {
286 kfree(info_buf);
287 }
288
289 return 0;
290
291err_out:
292 kfree(info_buf);
293
294 return -EINVAL;
295}
296
297/* Clean up the arg_info in a struct bpf_struct_ops_desc. */
298void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
299{
300 struct bpf_struct_ops_arg_info *arg_info;
301 int i;
302
303 arg_info = st_ops_desc->arg_info;
304 for (i = 0; i < btf_type_vlen(st_ops_desc->type); i++)
305 kfree(arg_info[i].info);
306
307 kfree(arg_info);
308}
309
310int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
311 struct btf *btf,
312 struct bpf_verifier_log *log)
313{
314 struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
315 struct bpf_struct_ops_arg_info *arg_info;
316 const struct btf_member *member;
317 const struct btf_type *t;
318 s32 type_id, value_id;
319 char value_name[128];
320 const char *mname;
321 int i, err;
322
323 if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
324 sizeof(value_name)) {
325 pr_warn("struct_ops name %s is too long\n",
326 st_ops->name);
327 return -EINVAL;
328 }
329 sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
330
331 if (!st_ops->cfi_stubs) {
332 pr_warn("struct_ops for %s has no cfi_stubs\n", st_ops->name);
333 return -EINVAL;
334 }
335
336 type_id = btf_find_by_name_kind(btf, st_ops->name,
337 BTF_KIND_STRUCT);
338 if (type_id < 0) {
339 pr_warn("Cannot find struct %s in %s\n",
340 st_ops->name, btf_get_name(btf));
341 return -EINVAL;
342 }
343 t = btf_type_by_id(btf, type_id);
344 if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
345 pr_warn("Cannot support #%u members in struct %s\n",
346 btf_type_vlen(t), st_ops->name);
347 return -EINVAL;
348 }
349
350 value_id = btf_find_by_name_kind(btf, value_name,
351 BTF_KIND_STRUCT);
352 if (value_id < 0) {
353 pr_warn("Cannot find struct %s in %s\n",
354 value_name, btf_get_name(btf));
355 return -EINVAL;
356 }
357 if (!is_valid_value_type(btf, value_id, t, value_name))
358 return -EINVAL;
359
360 arg_info = kcalloc(btf_type_vlen(t), sizeof(*arg_info),
361 GFP_KERNEL);
362 if (!arg_info)
363 return -ENOMEM;
364
365 st_ops_desc->arg_info = arg_info;
366 st_ops_desc->type = t;
367 st_ops_desc->type_id = type_id;
368 st_ops_desc->value_id = value_id;
369 st_ops_desc->value_type = btf_type_by_id(btf, value_id);
370
371 for_each_member(i, t, member) {
372 const struct btf_type *func_proto;
373
374 mname = btf_name_by_offset(btf, member->name_off);
375 if (!*mname) {
376 pr_warn("anon member in struct %s is not supported\n",
377 st_ops->name);
378 err = -EOPNOTSUPP;
379 goto errout;
380 }
381
382 if (__btf_member_bitfield_size(t, member)) {
383 pr_warn("bit field member %s in struct %s is not supported\n",
384 mname, st_ops->name);
385 err = -EOPNOTSUPP;
386 goto errout;
387 }
388
389 func_proto = btf_type_resolve_func_ptr(btf,
390 member->type,
391 NULL);
392 if (!func_proto)
393 continue;
394
395 if (btf_distill_func_proto(log, btf,
396 func_proto, mname,
397 &st_ops->func_models[i])) {
398 pr_warn("Error in parsing func ptr %s in struct %s\n",
399 mname, st_ops->name);
400 err = -EINVAL;
401 goto errout;
402 }
403
404 err = prepare_arg_info(btf, st_ops->name, mname,
405 func_proto,
406 arg_info + i);
407 if (err)
408 goto errout;
409 }
410
411 if (st_ops->init(btf)) {
412 pr_warn("Error in init bpf_struct_ops %s\n",
413 st_ops->name);
414 err = -EINVAL;
415 goto errout;
416 }
417
418 return 0;
419
420errout:
421 bpf_struct_ops_desc_release(st_ops_desc);
422
423 return err;
424}
425
426static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
427 void *next_key)
428{
429 if (key && *(u32 *)key == 0)
430 return -ENOENT;
431
432 *(u32 *)next_key = 0;
433 return 0;
434}
435
436int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
437 void *value)
438{
439 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
440 struct bpf_struct_ops_value *uvalue, *kvalue;
441 enum bpf_struct_ops_state state;
442 s64 refcnt;
443
444 if (unlikely(*(u32 *)key != 0))
445 return -ENOENT;
446
447 kvalue = &st_map->kvalue;
448 /* Pair with smp_store_release() during map_update */
449 state = smp_load_acquire(&kvalue->common.state);
450 if (state == BPF_STRUCT_OPS_STATE_INIT) {
451 memset(value, 0, map->value_size);
452 return 0;
453 }
454
455 /* No lock is needed. state and refcnt do not need
456 * to be updated together under atomic context.
457 */
458 uvalue = value;
459 memcpy(uvalue, st_map->uvalue, map->value_size);
460 uvalue->common.state = state;
461
462 /* This value offers the user space a general estimate of how
463 * many sockets are still utilizing this struct_ops for TCP
464 * congestion control. The number might not be exact, but it
465 * should sufficiently meet our present goals.
466 */
467 refcnt = atomic64_read(&map->refcnt) - atomic64_read(&map->usercnt);
468 refcount_set(&uvalue->common.refcnt, max_t(s64, refcnt, 0));
469
470 return 0;
471}
472
473static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
474{
475 return ERR_PTR(-EINVAL);
476}
477
478static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
479{
480 u32 i;
481
482 for (i = 0; i < st_map->links_cnt; i++) {
483 if (st_map->links[i]) {
484 bpf_link_put(st_map->links[i]);
485 st_map->links[i] = NULL;
486 }
487 }
488}
489
490static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)
491{
492 int i;
493
494 for (i = 0; i < st_map->image_pages_cnt; i++)
495 bpf_struct_ops_image_free(st_map->image_pages[i]);
496 st_map->image_pages_cnt = 0;
497}
498
499static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data)
500{
501 const struct btf_member *member;
502 u32 i, moff, msize, prev_mend = 0;
503 const struct btf_type *mtype;
504
505 for_each_member(i, t, member) {
506 moff = __btf_member_bit_offset(t, member) / 8;
507 if (moff > prev_mend &&
508 memchr_inv(data + prev_mend, 0, moff - prev_mend))
509 return -EINVAL;
510
511 mtype = btf_type_by_id(btf, member->type);
512 mtype = btf_resolve_size(btf, mtype, &msize);
513 if (IS_ERR(mtype))
514 return PTR_ERR(mtype);
515 prev_mend = moff + msize;
516 }
517
518 if (t->size > prev_mend &&
519 memchr_inv(data + prev_mend, 0, t->size - prev_mend))
520 return -EINVAL;
521
522 return 0;
523}
524
525static void bpf_struct_ops_link_release(struct bpf_link *link)
526{
527}
528
529static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
530{
531 struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
532
533 kfree(tlink);
534}
535
536const struct bpf_link_ops bpf_struct_ops_link_lops = {
537 .release = bpf_struct_ops_link_release,
538 .dealloc = bpf_struct_ops_link_dealloc,
539};
540
541int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
542 struct bpf_tramp_link *link,
543 const struct btf_func_model *model,
544 void *stub_func,
545 void **_image, u32 *_image_off,
546 bool allow_alloc)
547{
548 u32 image_off = *_image_off, flags = BPF_TRAMP_F_INDIRECT;
549 void *image = *_image;
550 int size;
551
552 tlinks[BPF_TRAMP_FENTRY].links[0] = link;
553 tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
554
555 if (model->ret_size > 0)
556 flags |= BPF_TRAMP_F_RET_FENTRY_RET;
557
558 size = arch_bpf_trampoline_size(model, flags, tlinks, NULL);
559 if (size <= 0)
560 return size ? : -EFAULT;
561
562 /* Allocate image buffer if necessary */
563 if (!image || size > PAGE_SIZE - image_off) {
564 if (!allow_alloc)
565 return -E2BIG;
566
567 image = bpf_struct_ops_image_alloc();
568 if (IS_ERR(image))
569 return PTR_ERR(image);
570 image_off = 0;
571 }
572
573 size = arch_prepare_bpf_trampoline(NULL, image + image_off,
574 image + PAGE_SIZE,
575 model, flags, tlinks, stub_func);
576 if (size <= 0) {
577 if (image != *_image)
578 bpf_struct_ops_image_free(image);
579 return size ? : -EFAULT;
580 }
581
582 *_image = image;
583 *_image_off = image_off + size;
584 return 0;
585}
586
587static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
588 void *value, u64 flags)
589{
590 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
591 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
592 const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops;
593 struct bpf_struct_ops_value *uvalue, *kvalue;
594 const struct btf_type *module_type;
595 const struct btf_member *member;
596 const struct btf_type *t = st_ops_desc->type;
597 struct bpf_tramp_links *tlinks;
598 void *udata, *kdata;
599 int prog_fd, err;
600 u32 i, trampoline_start, image_off = 0;
601 void *cur_image = NULL, *image = NULL;
602
603 if (flags)
604 return -EINVAL;
605
606 if (*(u32 *)key != 0)
607 return -E2BIG;
608
609 err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);
610 if (err)
611 return err;
612
613 uvalue = value;
614 err = check_zero_holes(st_map->btf, t, uvalue->data);
615 if (err)
616 return err;
617
618 if (uvalue->common.state || refcount_read(&uvalue->common.refcnt))
619 return -EINVAL;
620
621 tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
622 if (!tlinks)
623 return -ENOMEM;
624
625 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
626 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
627
628 mutex_lock(&st_map->lock);
629
630 if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) {
631 err = -EBUSY;
632 goto unlock;
633 }
634
635 memcpy(uvalue, value, map->value_size);
636
637 udata = &uvalue->data;
638 kdata = &kvalue->data;
639
640 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
641 for_each_member(i, t, member) {
642 const struct btf_type *mtype, *ptype;
643 struct bpf_prog *prog;
644 struct bpf_tramp_link *link;
645 u32 moff;
646
647 moff = __btf_member_bit_offset(t, member) / 8;
648 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
649 if (ptype == module_type) {
650 if (*(void **)(udata + moff))
651 goto reset_unlock;
652 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
653 continue;
654 }
655
656 err = st_ops->init_member(t, member, kdata, udata);
657 if (err < 0)
658 goto reset_unlock;
659
660 /* The ->init_member() has handled this member */
661 if (err > 0)
662 continue;
663
664 /* If st_ops->init_member does not handle it,
665 * we will only handle func ptrs and zero-ed members
666 * here. Reject everything else.
667 */
668
669 /* All non func ptr member must be 0 */
670 if (!ptype || !btf_type_is_func_proto(ptype)) {
671 u32 msize;
672
673 mtype = btf_type_by_id(st_map->btf, member->type);
674 mtype = btf_resolve_size(st_map->btf, mtype, &msize);
675 if (IS_ERR(mtype)) {
676 err = PTR_ERR(mtype);
677 goto reset_unlock;
678 }
679
680 if (memchr_inv(udata + moff, 0, msize)) {
681 err = -EINVAL;
682 goto reset_unlock;
683 }
684
685 continue;
686 }
687
688 prog_fd = (int)(*(unsigned long *)(udata + moff));
689 /* Similar check as the attr->attach_prog_fd */
690 if (!prog_fd)
691 continue;
692
693 prog = bpf_prog_get(prog_fd);
694 if (IS_ERR(prog)) {
695 err = PTR_ERR(prog);
696 goto reset_unlock;
697 }
698
699 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
700 prog->aux->attach_btf_id != st_ops_desc->type_id ||
701 prog->expected_attach_type != i) {
702 bpf_prog_put(prog);
703 err = -EINVAL;
704 goto reset_unlock;
705 }
706
707 link = kzalloc(sizeof(*link), GFP_USER);
708 if (!link) {
709 bpf_prog_put(prog);
710 err = -ENOMEM;
711 goto reset_unlock;
712 }
713 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
714 &bpf_struct_ops_link_lops, prog);
715 st_map->links[i] = &link->link;
716
717 trampoline_start = image_off;
718 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
719 &st_ops->func_models[i],
720 *(void **)(st_ops->cfi_stubs + moff),
721 &image, &image_off,
722 st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);
723 if (err)
724 goto reset_unlock;
725
726 if (cur_image != image) {
727 st_map->image_pages[st_map->image_pages_cnt++] = image;
728 cur_image = image;
729 trampoline_start = 0;
730 }
731 if (err < 0)
732 goto reset_unlock;
733
734 *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
735
736 /* put prog_id to udata */
737 *(unsigned long *)(udata + moff) = prog->aux->id;
738 }
739
740 if (st_ops->validate) {
741 err = st_ops->validate(kdata);
742 if (err)
743 goto reset_unlock;
744 }
745 for (i = 0; i < st_map->image_pages_cnt; i++)
746 arch_protect_bpf_trampoline(st_map->image_pages[i], PAGE_SIZE);
747
748 if (st_map->map.map_flags & BPF_F_LINK) {
749 err = 0;
750 /* Let bpf_link handle registration & unregistration.
751 *
752 * Pair with smp_load_acquire() during lookup_elem().
753 */
754 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY);
755 goto unlock;
756 }
757
758 err = st_ops->reg(kdata);
759 if (likely(!err)) {
760 /* This refcnt increment on the map here after
761 * 'st_ops->reg()' is secure since the state of the
762 * map must be set to INIT at this moment, and thus
763 * bpf_struct_ops_map_delete_elem() can't unregister
764 * or transition it to TOBEFREE concurrently.
765 */
766 bpf_map_inc(map);
767 /* Pair with smp_load_acquire() during lookup_elem().
768 * It ensures the above udata updates (e.g. prog->aux->id)
769 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
770 */
771 smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE);
772 goto unlock;
773 }
774
775 /* Error during st_ops->reg(). Can happen if this struct_ops needs to be
776 * verified as a whole, after all init_member() calls. Can also happen if
777 * there was a race in registering the struct_ops (under the same name) to
778 * a sub-system through different struct_ops's maps.
779 */
780
781reset_unlock:
782 bpf_struct_ops_map_free_image(st_map);
783 bpf_struct_ops_map_put_progs(st_map);
784 memset(uvalue, 0, map->value_size);
785 memset(kvalue, 0, map->value_size);
786unlock:
787 kfree(tlinks);
788 mutex_unlock(&st_map->lock);
789 return err;
790}
791
792static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
793{
794 enum bpf_struct_ops_state prev_state;
795 struct bpf_struct_ops_map *st_map;
796
797 st_map = (struct bpf_struct_ops_map *)map;
798 if (st_map->map.map_flags & BPF_F_LINK)
799 return -EOPNOTSUPP;
800
801 prev_state = cmpxchg(&st_map->kvalue.common.state,
802 BPF_STRUCT_OPS_STATE_INUSE,
803 BPF_STRUCT_OPS_STATE_TOBEFREE);
804 switch (prev_state) {
805 case BPF_STRUCT_OPS_STATE_INUSE:
806 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data);
807 bpf_map_put(map);
808 return 0;
809 case BPF_STRUCT_OPS_STATE_TOBEFREE:
810 return -EINPROGRESS;
811 case BPF_STRUCT_OPS_STATE_INIT:
812 return -ENOENT;
813 default:
814 WARN_ON_ONCE(1);
815 /* Should never happen. Treat it as not found. */
816 return -ENOENT;
817 }
818}
819
820static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
821 struct seq_file *m)
822{
823 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
824 void *value;
825 int err;
826
827 value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
828 if (!value)
829 return;
830
831 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
832 if (!err) {
833 btf_type_seq_show(st_map->btf,
834 map->btf_vmlinux_value_type_id,
835 value, m);
836 seq_puts(m, "\n");
837 }
838
839 kfree(value);
840}
841
842static void __bpf_struct_ops_map_free(struct bpf_map *map)
843{
844 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
845
846 if (st_map->links)
847 bpf_struct_ops_map_put_progs(st_map);
848 bpf_map_area_free(st_map->links);
849 bpf_struct_ops_map_free_image(st_map);
850 bpf_map_area_free(st_map->uvalue);
851 bpf_map_area_free(st_map);
852}
853
854static void bpf_struct_ops_map_free(struct bpf_map *map)
855{
856 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
857
858 /* st_ops->owner was acquired during map_alloc to implicitly holds
859 * the btf's refcnt. The acquire was only done when btf_is_module()
860 * st_map->btf cannot be NULL here.
861 */
862 if (btf_is_module(st_map->btf))
863 module_put(st_map->st_ops_desc->st_ops->owner);
864
865 /* The struct_ops's function may switch to another struct_ops.
866 *
867 * For example, bpf_tcp_cc_x->init() may switch to
868 * another tcp_cc_y by calling
869 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
870 * During the switch, bpf_struct_ops_put(tcp_cc_x) is called
871 * and its refcount may reach 0 which then free its
872 * trampoline image while tcp_cc_x is still running.
873 *
874 * A vanilla rcu gp is to wait for all bpf-tcp-cc prog
875 * to finish. bpf-tcp-cc prog is non sleepable.
876 * A rcu_tasks gp is to wait for the last few insn
877 * in the tramopline image to finish before releasing
878 * the trampoline image.
879 */
880 synchronize_rcu_mult(call_rcu, call_rcu_tasks);
881
882 __bpf_struct_ops_map_free(map);
883}
884
885static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
886{
887 if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
888 (attr->map_flags & ~(BPF_F_LINK | BPF_F_VTYPE_BTF_OBJ_FD)) ||
889 !attr->btf_vmlinux_value_type_id)
890 return -EINVAL;
891 return 0;
892}
893
894static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
895{
896 const struct bpf_struct_ops_desc *st_ops_desc;
897 size_t st_map_size;
898 struct bpf_struct_ops_map *st_map;
899 const struct btf_type *t, *vt;
900 struct module *mod = NULL;
901 struct bpf_map *map;
902 struct btf *btf;
903 int ret;
904
905 if (attr->map_flags & BPF_F_VTYPE_BTF_OBJ_FD) {
906 /* The map holds btf for its whole life time. */
907 btf = btf_get_by_fd(attr->value_type_btf_obj_fd);
908 if (IS_ERR(btf))
909 return ERR_CAST(btf);
910 if (!btf_is_module(btf)) {
911 btf_put(btf);
912 return ERR_PTR(-EINVAL);
913 }
914
915 mod = btf_try_get_module(btf);
916 /* mod holds a refcnt to btf. We don't need an extra refcnt
917 * here.
918 */
919 btf_put(btf);
920 if (!mod)
921 return ERR_PTR(-EINVAL);
922 } else {
923 btf = bpf_get_btf_vmlinux();
924 if (IS_ERR(btf))
925 return ERR_CAST(btf);
926 if (!btf)
927 return ERR_PTR(-ENOTSUPP);
928 }
929
930 st_ops_desc = bpf_struct_ops_find_value(btf, attr->btf_vmlinux_value_type_id);
931 if (!st_ops_desc) {
932 ret = -ENOTSUPP;
933 goto errout;
934 }
935
936 vt = st_ops_desc->value_type;
937 if (attr->value_size != vt->size) {
938 ret = -EINVAL;
939 goto errout;
940 }
941
942 t = st_ops_desc->type;
943
944 st_map_size = sizeof(*st_map) +
945 /* kvalue stores the
946 * struct bpf_struct_ops_tcp_congestions_ops
947 */
948 (vt->size - sizeof(struct bpf_struct_ops_value));
949
950 st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
951 if (!st_map) {
952 ret = -ENOMEM;
953 goto errout;
954 }
955
956 st_map->st_ops_desc = st_ops_desc;
957 map = &st_map->map;
958
959 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
960 st_map->links_cnt = btf_type_vlen(t);
961 st_map->links =
962 bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *),
963 NUMA_NO_NODE);
964 if (!st_map->uvalue || !st_map->links) {
965 ret = -ENOMEM;
966 goto errout_free;
967 }
968 st_map->btf = btf;
969
970 mutex_init(&st_map->lock);
971 bpf_map_init_from_attr(map, attr);
972
973 return map;
974
975errout_free:
976 __bpf_struct_ops_map_free(map);
977errout:
978 module_put(mod);
979
980 return ERR_PTR(ret);
981}
982
983static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map)
984{
985 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
986 const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;
987 const struct btf_type *vt = st_ops_desc->value_type;
988 u64 usage;
989
990 usage = sizeof(*st_map) +
991 vt->size - sizeof(struct bpf_struct_ops_value);
992 usage += vt->size;
993 usage += btf_type_vlen(vt) * sizeof(struct bpf_links *);
994 usage += PAGE_SIZE;
995 return usage;
996}
997
998BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map)
999const struct bpf_map_ops bpf_struct_ops_map_ops = {
1000 .map_alloc_check = bpf_struct_ops_map_alloc_check,
1001 .map_alloc = bpf_struct_ops_map_alloc,
1002 .map_free = bpf_struct_ops_map_free,
1003 .map_get_next_key = bpf_struct_ops_map_get_next_key,
1004 .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
1005 .map_delete_elem = bpf_struct_ops_map_delete_elem,
1006 .map_update_elem = bpf_struct_ops_map_update_elem,
1007 .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
1008 .map_mem_usage = bpf_struct_ops_map_mem_usage,
1009 .map_btf_id = &bpf_struct_ops_map_btf_ids[0],
1010};
1011
1012/* "const void *" because some subsystem is
1013 * passing a const (e.g. const struct tcp_congestion_ops *)
1014 */
1015bool bpf_struct_ops_get(const void *kdata)
1016{
1017 struct bpf_struct_ops_value *kvalue;
1018 struct bpf_struct_ops_map *st_map;
1019 struct bpf_map *map;
1020
1021 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1022 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1023
1024 map = __bpf_map_inc_not_zero(&st_map->map, false);
1025 return !IS_ERR(map);
1026}
1027
1028void bpf_struct_ops_put(const void *kdata)
1029{
1030 struct bpf_struct_ops_value *kvalue;
1031 struct bpf_struct_ops_map *st_map;
1032
1033 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
1034 st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);
1035
1036 bpf_map_put(&st_map->map);
1037}
1038
1039static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
1040{
1041 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1042
1043 return map->map_type == BPF_MAP_TYPE_STRUCT_OPS &&
1044 map->map_flags & BPF_F_LINK &&
1045 /* Pair with smp_store_release() during map_update */
1046 smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;
1047}
1048
1049static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)
1050{
1051 struct bpf_struct_ops_link *st_link;
1052 struct bpf_struct_ops_map *st_map;
1053
1054 st_link = container_of(link, struct bpf_struct_ops_link, link);
1055 st_map = (struct bpf_struct_ops_map *)
1056 rcu_dereference_protected(st_link->map, true);
1057 if (st_map) {
1058 /* st_link->map can be NULL if
1059 * bpf_struct_ops_link_create() fails to register.
1060 */
1061 st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data);
1062 bpf_map_put(&st_map->map);
1063 }
1064 kfree(st_link);
1065}
1066
1067static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link,
1068 struct seq_file *seq)
1069{
1070 struct bpf_struct_ops_link *st_link;
1071 struct bpf_map *map;
1072
1073 st_link = container_of(link, struct bpf_struct_ops_link, link);
1074 rcu_read_lock();
1075 map = rcu_dereference(st_link->map);
1076 seq_printf(seq, "map_id:\t%d\n", map->id);
1077 rcu_read_unlock();
1078}
1079
1080static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
1081 struct bpf_link_info *info)
1082{
1083 struct bpf_struct_ops_link *st_link;
1084 struct bpf_map *map;
1085
1086 st_link = container_of(link, struct bpf_struct_ops_link, link);
1087 rcu_read_lock();
1088 map = rcu_dereference(st_link->map);
1089 info->struct_ops.map_id = map->id;
1090 rcu_read_unlock();
1091 return 0;
1092}
1093
1094static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map,
1095 struct bpf_map *expected_old_map)
1096{
1097 struct bpf_struct_ops_map *st_map, *old_st_map;
1098 struct bpf_map *old_map;
1099 struct bpf_struct_ops_link *st_link;
1100 int err;
1101
1102 st_link = container_of(link, struct bpf_struct_ops_link, link);
1103 st_map = container_of(new_map, struct bpf_struct_ops_map, map);
1104
1105 if (!bpf_struct_ops_valid_to_reg(new_map))
1106 return -EINVAL;
1107
1108 if (!st_map->st_ops_desc->st_ops->update)
1109 return -EOPNOTSUPP;
1110
1111 mutex_lock(&update_mutex);
1112
1113 old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
1114 if (expected_old_map && old_map != expected_old_map) {
1115 err = -EPERM;
1116 goto err_out;
1117 }
1118
1119 old_st_map = container_of(old_map, struct bpf_struct_ops_map, map);
1120 /* The new and old struct_ops must be the same type. */
1121 if (st_map->st_ops_desc != old_st_map->st_ops_desc) {
1122 err = -EINVAL;
1123 goto err_out;
1124 }
1125
1126 err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data);
1127 if (err)
1128 goto err_out;
1129
1130 bpf_map_inc(new_map);
1131 rcu_assign_pointer(st_link->map, new_map);
1132 bpf_map_put(old_map);
1133
1134err_out:
1135 mutex_unlock(&update_mutex);
1136
1137 return err;
1138}
1139
1140static const struct bpf_link_ops bpf_struct_ops_map_lops = {
1141 .dealloc = bpf_struct_ops_map_link_dealloc,
1142 .show_fdinfo = bpf_struct_ops_map_link_show_fdinfo,
1143 .fill_link_info = bpf_struct_ops_map_link_fill_link_info,
1144 .update_map = bpf_struct_ops_map_link_update,
1145};
1146
1147int bpf_struct_ops_link_create(union bpf_attr *attr)
1148{
1149 struct bpf_struct_ops_link *link = NULL;
1150 struct bpf_link_primer link_primer;
1151 struct bpf_struct_ops_map *st_map;
1152 struct bpf_map *map;
1153 int err;
1154
1155 map = bpf_map_get(attr->link_create.map_fd);
1156 if (IS_ERR(map))
1157 return PTR_ERR(map);
1158
1159 st_map = (struct bpf_struct_ops_map *)map;
1160
1161 if (!bpf_struct_ops_valid_to_reg(map)) {
1162 err = -EINVAL;
1163 goto err_out;
1164 }
1165
1166 link = kzalloc(sizeof(*link), GFP_USER);
1167 if (!link) {
1168 err = -ENOMEM;
1169 goto err_out;
1170 }
1171 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL);
1172
1173 err = bpf_link_prime(&link->link, &link_primer);
1174 if (err)
1175 goto err_out;
1176
1177 err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data);
1178 if (err) {
1179 bpf_link_cleanup(&link_primer);
1180 link = NULL;
1181 goto err_out;
1182 }
1183 RCU_INIT_POINTER(link->map, map);
1184
1185 return bpf_link_settle(&link_primer);
1186
1187err_out:
1188 bpf_map_put(map);
1189 kfree(link);
1190 return err;
1191}
1192
1193void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1194{
1195 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
1196
1197 info->btf_vmlinux_id = btf_obj_id(st_map->btf);
1198}