Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2020 Facebook */
3
4#include <linux/fs.h>
5#include <linux/anon_inodes.h>
6#include <linux/filter.h>
7#include <linux/bpf.h>
8#include <linux/rcupdate_trace.h>
9
10struct bpf_iter_target_info {
11 struct list_head list;
12 const struct bpf_iter_reg *reg_info;
13 u32 btf_id; /* cached value */
14};
15
16struct bpf_iter_link {
17 struct bpf_link link;
18 struct bpf_iter_aux_info aux;
19 struct bpf_iter_target_info *tinfo;
20};
21
22struct bpf_iter_priv_data {
23 struct bpf_iter_target_info *tinfo;
24 const struct bpf_iter_seq_info *seq_info;
25 struct bpf_prog *prog;
26 u64 session_id;
27 u64 seq_num;
28 bool done_stop;
29 u8 target_private[] __aligned(8);
30};
31
32static struct list_head targets = LIST_HEAD_INIT(targets);
33static DEFINE_MUTEX(targets_mutex);
34
35/* protect bpf_iter_link changes */
36static DEFINE_MUTEX(link_mutex);
37
38/* incremented on every opened seq_file */
39static atomic64_t session_id;
40
41static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
42 const struct bpf_iter_seq_info *seq_info);
43
44static void bpf_iter_inc_seq_num(struct seq_file *seq)
45{
46 struct bpf_iter_priv_data *iter_priv;
47
48 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
49 target_private);
50 iter_priv->seq_num++;
51}
52
53static void bpf_iter_dec_seq_num(struct seq_file *seq)
54{
55 struct bpf_iter_priv_data *iter_priv;
56
57 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
58 target_private);
59 iter_priv->seq_num--;
60}
61
62static void bpf_iter_done_stop(struct seq_file *seq)
63{
64 struct bpf_iter_priv_data *iter_priv;
65
66 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
67 target_private);
68 iter_priv->done_stop = true;
69}
70
71static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo)
72{
73 return tinfo->reg_info->feature & BPF_ITER_RESCHED;
74}
75
76static bool bpf_iter_support_resched(struct seq_file *seq)
77{
78 struct bpf_iter_priv_data *iter_priv;
79
80 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
81 target_private);
82 return bpf_iter_target_support_resched(iter_priv->tinfo);
83}
84
85/* maximum visited objects before bailing out */
86#define MAX_ITER_OBJECTS 1000000
87
88/* bpf_seq_read, a customized and simpler version for bpf iterator.
89 * The following are differences from seq_read():
90 * . fixed buffer size (PAGE_SIZE)
91 * . assuming NULL ->llseek()
92 * . stop() may call bpf program, handling potential overflow there
93 */
94static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
95 loff_t *ppos)
96{
97 struct seq_file *seq = file->private_data;
98 size_t n, offs, copied = 0;
99 int err = 0, num_objs = 0;
100 bool can_resched;
101 void *p;
102
103 mutex_lock(&seq->lock);
104
105 if (!seq->buf) {
106 seq->size = PAGE_SIZE << 3;
107 seq->buf = kvmalloc(seq->size, GFP_KERNEL);
108 if (!seq->buf) {
109 err = -ENOMEM;
110 goto done;
111 }
112 }
113
114 if (seq->count) {
115 n = min(seq->count, size);
116 err = copy_to_user(buf, seq->buf + seq->from, n);
117 if (err) {
118 err = -EFAULT;
119 goto done;
120 }
121 seq->count -= n;
122 seq->from += n;
123 copied = n;
124 goto done;
125 }
126
127 seq->from = 0;
128 p = seq->op->start(seq, &seq->index);
129 if (!p)
130 goto stop;
131 if (IS_ERR(p)) {
132 err = PTR_ERR(p);
133 seq->op->stop(seq, p);
134 seq->count = 0;
135 goto done;
136 }
137
138 err = seq->op->show(seq, p);
139 if (err > 0) {
140 /* object is skipped, decrease seq_num, so next
141 * valid object can reuse the same seq_num.
142 */
143 bpf_iter_dec_seq_num(seq);
144 seq->count = 0;
145 } else if (err < 0 || seq_has_overflowed(seq)) {
146 if (!err)
147 err = -E2BIG;
148 seq->op->stop(seq, p);
149 seq->count = 0;
150 goto done;
151 }
152
153 can_resched = bpf_iter_support_resched(seq);
154 while (1) {
155 loff_t pos = seq->index;
156
157 num_objs++;
158 offs = seq->count;
159 p = seq->op->next(seq, p, &seq->index);
160 if (pos == seq->index) {
161 pr_info_ratelimited("buggy seq_file .next function %ps "
162 "did not updated position index\n",
163 seq->op->next);
164 seq->index++;
165 }
166
167 if (IS_ERR_OR_NULL(p))
168 break;
169
170 /* got a valid next object, increase seq_num */
171 bpf_iter_inc_seq_num(seq);
172
173 if (seq->count >= size)
174 break;
175
176 if (num_objs >= MAX_ITER_OBJECTS) {
177 if (offs == 0) {
178 err = -EAGAIN;
179 seq->op->stop(seq, p);
180 goto done;
181 }
182 break;
183 }
184
185 err = seq->op->show(seq, p);
186 if (err > 0) {
187 bpf_iter_dec_seq_num(seq);
188 seq->count = offs;
189 } else if (err < 0 || seq_has_overflowed(seq)) {
190 seq->count = offs;
191 if (offs == 0) {
192 if (!err)
193 err = -E2BIG;
194 seq->op->stop(seq, p);
195 goto done;
196 }
197 break;
198 }
199
200 if (can_resched)
201 cond_resched();
202 }
203stop:
204 offs = seq->count;
205 if (IS_ERR(p)) {
206 seq->op->stop(seq, NULL);
207 err = PTR_ERR(p);
208 goto done;
209 }
210 /* bpf program called if !p */
211 seq->op->stop(seq, p);
212 if (!p) {
213 if (!seq_has_overflowed(seq)) {
214 bpf_iter_done_stop(seq);
215 } else {
216 seq->count = offs;
217 if (offs == 0) {
218 err = -E2BIG;
219 goto done;
220 }
221 }
222 }
223
224 n = min(seq->count, size);
225 err = copy_to_user(buf, seq->buf, n);
226 if (err) {
227 err = -EFAULT;
228 goto done;
229 }
230 copied = n;
231 seq->count -= n;
232 seq->from = n;
233done:
234 if (!copied)
235 copied = err;
236 else
237 *ppos += copied;
238 mutex_unlock(&seq->lock);
239 return copied;
240}
241
242static const struct bpf_iter_seq_info *
243__get_seq_info(struct bpf_iter_link *link)
244{
245 const struct bpf_iter_seq_info *seq_info;
246
247 if (link->aux.map) {
248 seq_info = link->aux.map->ops->iter_seq_info;
249 if (seq_info)
250 return seq_info;
251 }
252
253 return link->tinfo->reg_info->seq_info;
254}
255
256static int iter_open(struct inode *inode, struct file *file)
257{
258 struct bpf_iter_link *link = inode->i_private;
259
260 return prepare_seq_file(file, link, __get_seq_info(link));
261}
262
263static int iter_release(struct inode *inode, struct file *file)
264{
265 struct bpf_iter_priv_data *iter_priv;
266 struct seq_file *seq;
267
268 seq = file->private_data;
269 if (!seq)
270 return 0;
271
272 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
273 target_private);
274
275 if (iter_priv->seq_info->fini_seq_private)
276 iter_priv->seq_info->fini_seq_private(seq->private);
277
278 bpf_prog_put(iter_priv->prog);
279 seq->private = iter_priv;
280
281 return seq_release_private(inode, file);
282}
283
284const struct file_operations bpf_iter_fops = {
285 .open = iter_open,
286 .llseek = no_llseek,
287 .read = bpf_seq_read,
288 .release = iter_release,
289};
290
291/* The argument reg_info will be cached in bpf_iter_target_info.
292 * The common practice is to declare target reg_info as
293 * a const static variable and passed as an argument to
294 * bpf_iter_reg_target().
295 */
296int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
297{
298 struct bpf_iter_target_info *tinfo;
299
300 tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
301 if (!tinfo)
302 return -ENOMEM;
303
304 tinfo->reg_info = reg_info;
305 INIT_LIST_HEAD(&tinfo->list);
306
307 mutex_lock(&targets_mutex);
308 list_add(&tinfo->list, &targets);
309 mutex_unlock(&targets_mutex);
310
311 return 0;
312}
313
314void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
315{
316 struct bpf_iter_target_info *tinfo;
317 bool found = false;
318
319 mutex_lock(&targets_mutex);
320 list_for_each_entry(tinfo, &targets, list) {
321 if (reg_info == tinfo->reg_info) {
322 list_del(&tinfo->list);
323 kfree(tinfo);
324 found = true;
325 break;
326 }
327 }
328 mutex_unlock(&targets_mutex);
329
330 WARN_ON(found == false);
331}
332
333static void cache_btf_id(struct bpf_iter_target_info *tinfo,
334 struct bpf_prog *prog)
335{
336 tinfo->btf_id = prog->aux->attach_btf_id;
337}
338
339bool bpf_iter_prog_supported(struct bpf_prog *prog)
340{
341 const char *attach_fname = prog->aux->attach_func_name;
342 struct bpf_iter_target_info *tinfo = NULL, *iter;
343 u32 prog_btf_id = prog->aux->attach_btf_id;
344 const char *prefix = BPF_ITER_FUNC_PREFIX;
345 int prefix_len = strlen(prefix);
346
347 if (strncmp(attach_fname, prefix, prefix_len))
348 return false;
349
350 mutex_lock(&targets_mutex);
351 list_for_each_entry(iter, &targets, list) {
352 if (iter->btf_id && iter->btf_id == prog_btf_id) {
353 tinfo = iter;
354 break;
355 }
356 if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) {
357 cache_btf_id(iter, prog);
358 tinfo = iter;
359 break;
360 }
361 }
362 mutex_unlock(&targets_mutex);
363
364 if (tinfo) {
365 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
366 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
367 }
368
369 return tinfo != NULL;
370}
371
372const struct bpf_func_proto *
373bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
374{
375 const struct bpf_iter_target_info *tinfo;
376 const struct bpf_func_proto *fn = NULL;
377
378 mutex_lock(&targets_mutex);
379 list_for_each_entry(tinfo, &targets, list) {
380 if (tinfo->btf_id == prog->aux->attach_btf_id) {
381 const struct bpf_iter_reg *reg_info;
382
383 reg_info = tinfo->reg_info;
384 if (reg_info->get_func_proto)
385 fn = reg_info->get_func_proto(func_id, prog);
386 break;
387 }
388 }
389 mutex_unlock(&targets_mutex);
390
391 return fn;
392}
393
394static void bpf_iter_link_release(struct bpf_link *link)
395{
396 struct bpf_iter_link *iter_link =
397 container_of(link, struct bpf_iter_link, link);
398
399 if (iter_link->tinfo->reg_info->detach_target)
400 iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
401}
402
403static void bpf_iter_link_dealloc(struct bpf_link *link)
404{
405 struct bpf_iter_link *iter_link =
406 container_of(link, struct bpf_iter_link, link);
407
408 kfree(iter_link);
409}
410
411static int bpf_iter_link_replace(struct bpf_link *link,
412 struct bpf_prog *new_prog,
413 struct bpf_prog *old_prog)
414{
415 int ret = 0;
416
417 mutex_lock(&link_mutex);
418 if (old_prog && link->prog != old_prog) {
419 ret = -EPERM;
420 goto out_unlock;
421 }
422
423 if (link->prog->type != new_prog->type ||
424 link->prog->expected_attach_type != new_prog->expected_attach_type ||
425 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
426 ret = -EINVAL;
427 goto out_unlock;
428 }
429
430 old_prog = xchg(&link->prog, new_prog);
431 bpf_prog_put(old_prog);
432
433out_unlock:
434 mutex_unlock(&link_mutex);
435 return ret;
436}
437
438static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
439 struct seq_file *seq)
440{
441 struct bpf_iter_link *iter_link =
442 container_of(link, struct bpf_iter_link, link);
443 bpf_iter_show_fdinfo_t show_fdinfo;
444
445 seq_printf(seq,
446 "target_name:\t%s\n",
447 iter_link->tinfo->reg_info->target);
448
449 show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
450 if (show_fdinfo)
451 show_fdinfo(&iter_link->aux, seq);
452}
453
454static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
455 struct bpf_link_info *info)
456{
457 struct bpf_iter_link *iter_link =
458 container_of(link, struct bpf_iter_link, link);
459 char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
460 bpf_iter_fill_link_info_t fill_link_info;
461 u32 ulen = info->iter.target_name_len;
462 const char *target_name;
463 u32 target_len;
464
465 if (!ulen ^ !ubuf)
466 return -EINVAL;
467
468 target_name = iter_link->tinfo->reg_info->target;
469 target_len = strlen(target_name);
470 info->iter.target_name_len = target_len + 1;
471
472 if (ubuf) {
473 if (ulen >= target_len + 1) {
474 if (copy_to_user(ubuf, target_name, target_len + 1))
475 return -EFAULT;
476 } else {
477 char zero = '\0';
478
479 if (copy_to_user(ubuf, target_name, ulen - 1))
480 return -EFAULT;
481 if (put_user(zero, ubuf + ulen - 1))
482 return -EFAULT;
483 return -ENOSPC;
484 }
485 }
486
487 fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
488 if (fill_link_info)
489 return fill_link_info(&iter_link->aux, info);
490
491 return 0;
492}
493
494static const struct bpf_link_ops bpf_iter_link_lops = {
495 .release = bpf_iter_link_release,
496 .dealloc = bpf_iter_link_dealloc,
497 .update_prog = bpf_iter_link_replace,
498 .show_fdinfo = bpf_iter_link_show_fdinfo,
499 .fill_link_info = bpf_iter_link_fill_link_info,
500};
501
502bool bpf_link_is_iter(struct bpf_link *link)
503{
504 return link->ops == &bpf_iter_link_lops;
505}
506
507int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
508 struct bpf_prog *prog)
509{
510 struct bpf_iter_target_info *tinfo = NULL, *iter;
511 struct bpf_link_primer link_primer;
512 union bpf_iter_link_info linfo;
513 struct bpf_iter_link *link;
514 u32 prog_btf_id, linfo_len;
515 bpfptr_t ulinfo;
516 int err;
517
518 if (attr->link_create.target_fd || attr->link_create.flags)
519 return -EINVAL;
520
521 memset(&linfo, 0, sizeof(union bpf_iter_link_info));
522
523 ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
524 linfo_len = attr->link_create.iter_info_len;
525 if (bpfptr_is_null(ulinfo) ^ !linfo_len)
526 return -EINVAL;
527
528 if (!bpfptr_is_null(ulinfo)) {
529 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
530 linfo_len);
531 if (err)
532 return err;
533 linfo_len = min_t(u32, linfo_len, sizeof(linfo));
534 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
535 return -EFAULT;
536 }
537
538 prog_btf_id = prog->aux->attach_btf_id;
539 mutex_lock(&targets_mutex);
540 list_for_each_entry(iter, &targets, list) {
541 if (iter->btf_id == prog_btf_id) {
542 tinfo = iter;
543 break;
544 }
545 }
546 mutex_unlock(&targets_mutex);
547 if (!tinfo)
548 return -ENOENT;
549
550 /* Only allow sleepable program for resched-able iterator */
551 if (prog->aux->sleepable && !bpf_iter_target_support_resched(tinfo))
552 return -EINVAL;
553
554 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
555 if (!link)
556 return -ENOMEM;
557
558 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
559 link->tinfo = tinfo;
560
561 err = bpf_link_prime(&link->link, &link_primer);
562 if (err) {
563 kfree(link);
564 return err;
565 }
566
567 if (tinfo->reg_info->attach_target) {
568 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
569 if (err) {
570 bpf_link_cleanup(&link_primer);
571 return err;
572 }
573 }
574
575 return bpf_link_settle(&link_primer);
576}
577
578static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
579 struct bpf_iter_target_info *tinfo,
580 const struct bpf_iter_seq_info *seq_info,
581 struct bpf_prog *prog)
582{
583 priv_data->tinfo = tinfo;
584 priv_data->seq_info = seq_info;
585 priv_data->prog = prog;
586 priv_data->session_id = atomic64_inc_return(&session_id);
587 priv_data->seq_num = 0;
588 priv_data->done_stop = false;
589}
590
591static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
592 const struct bpf_iter_seq_info *seq_info)
593{
594 struct bpf_iter_priv_data *priv_data;
595 struct bpf_iter_target_info *tinfo;
596 struct bpf_prog *prog;
597 u32 total_priv_dsize;
598 struct seq_file *seq;
599 int err = 0;
600
601 mutex_lock(&link_mutex);
602 prog = link->link.prog;
603 bpf_prog_inc(prog);
604 mutex_unlock(&link_mutex);
605
606 tinfo = link->tinfo;
607 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
608 seq_info->seq_priv_size;
609 priv_data = __seq_open_private(file, seq_info->seq_ops,
610 total_priv_dsize);
611 if (!priv_data) {
612 err = -ENOMEM;
613 goto release_prog;
614 }
615
616 if (seq_info->init_seq_private) {
617 err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
618 if (err)
619 goto release_seq_file;
620 }
621
622 init_seq_meta(priv_data, tinfo, seq_info, prog);
623 seq = file->private_data;
624 seq->private = priv_data->target_private;
625
626 return 0;
627
628release_seq_file:
629 seq_release_private(file->f_inode, file);
630 file->private_data = NULL;
631release_prog:
632 bpf_prog_put(prog);
633 return err;
634}
635
636int bpf_iter_new_fd(struct bpf_link *link)
637{
638 struct bpf_iter_link *iter_link;
639 struct file *file;
640 unsigned int flags;
641 int err, fd;
642
643 if (link->ops != &bpf_iter_link_lops)
644 return -EINVAL;
645
646 flags = O_RDONLY | O_CLOEXEC;
647 fd = get_unused_fd_flags(flags);
648 if (fd < 0)
649 return fd;
650
651 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
652 if (IS_ERR(file)) {
653 err = PTR_ERR(file);
654 goto free_fd;
655 }
656
657 iter_link = container_of(link, struct bpf_iter_link, link);
658 err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
659 if (err)
660 goto free_file;
661
662 fd_install(fd, file);
663 return fd;
664
665free_file:
666 fput(file);
667free_fd:
668 put_unused_fd(fd);
669 return err;
670}
671
672struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
673{
674 struct bpf_iter_priv_data *iter_priv;
675 struct seq_file *seq;
676 void *seq_priv;
677
678 seq = meta->seq;
679 if (seq->file->f_op != &bpf_iter_fops)
680 return NULL;
681
682 seq_priv = seq->private;
683 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
684 target_private);
685
686 if (in_stop && iter_priv->done_stop)
687 return NULL;
688
689 meta->session_id = iter_priv->session_id;
690 meta->seq_num = iter_priv->seq_num;
691
692 return iter_priv->prog;
693}
694
695int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
696{
697 struct bpf_run_ctx run_ctx, *old_run_ctx;
698 int ret;
699
700 if (prog->aux->sleepable) {
701 rcu_read_lock_trace();
702 migrate_disable();
703 might_fault();
704 old_run_ctx = bpf_set_run_ctx(&run_ctx);
705 ret = bpf_prog_run(prog, ctx);
706 bpf_reset_run_ctx(old_run_ctx);
707 migrate_enable();
708 rcu_read_unlock_trace();
709 } else {
710 rcu_read_lock();
711 migrate_disable();
712 old_run_ctx = bpf_set_run_ctx(&run_ctx);
713 ret = bpf_prog_run(prog, ctx);
714 bpf_reset_run_ctx(old_run_ctx);
715 migrate_enable();
716 rcu_read_unlock();
717 }
718
719 /* bpf program can only return 0 or 1:
720 * 0 : okay
721 * 1 : retry the same object
722 * The bpf_iter_run_prog() return value
723 * will be seq_ops->show() return value.
724 */
725 return ret == 0 ? 0 : -EAGAIN;
726}
727
728BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
729 void *, callback_ctx, u64, flags)
730{
731 return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
732}
733
734const struct bpf_func_proto bpf_for_each_map_elem_proto = {
735 .func = bpf_for_each_map_elem,
736 .gpl_only = false,
737 .ret_type = RET_INTEGER,
738 .arg1_type = ARG_CONST_MAP_PTR,
739 .arg2_type = ARG_PTR_TO_FUNC,
740 .arg3_type = ARG_PTR_TO_STACK_OR_NULL,
741 .arg4_type = ARG_ANYTHING,
742};
743
744BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx,
745 u64, flags)
746{
747 bpf_callback_t callback = (bpf_callback_t)callback_fn;
748 u64 ret;
749 u32 i;
750
751 /* Note: these safety checks are also verified when bpf_loop
752 * is inlined, be careful to modify this code in sync. See
753 * function verifier.c:inline_bpf_loop.
754 */
755 if (flags)
756 return -EINVAL;
757 if (nr_loops > BPF_MAX_LOOPS)
758 return -E2BIG;
759
760 for (i = 0; i < nr_loops; i++) {
761 ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0);
762 /* return value: 0 - continue, 1 - stop and return */
763 if (ret)
764 return i + 1;
765 }
766
767 return i;
768}
769
770const struct bpf_func_proto bpf_loop_proto = {
771 .func = bpf_loop,
772 .gpl_only = false,
773 .ret_type = RET_INTEGER,
774 .arg1_type = ARG_ANYTHING,
775 .arg2_type = ARG_PTR_TO_FUNC,
776 .arg3_type = ARG_PTR_TO_STACK_OR_NULL,
777 .arg4_type = ARG_ANYTHING,
778};
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2020 Facebook */
3
4#include <linux/fs.h>
5#include <linux/anon_inodes.h>
6#include <linux/filter.h>
7#include <linux/bpf.h>
8
9struct bpf_iter_target_info {
10 struct list_head list;
11 const struct bpf_iter_reg *reg_info;
12 u32 btf_id; /* cached value */
13};
14
15struct bpf_iter_link {
16 struct bpf_link link;
17 struct bpf_iter_aux_info aux;
18 struct bpf_iter_target_info *tinfo;
19};
20
21struct bpf_iter_priv_data {
22 struct bpf_iter_target_info *tinfo;
23 const struct bpf_iter_seq_info *seq_info;
24 struct bpf_prog *prog;
25 u64 session_id;
26 u64 seq_num;
27 bool done_stop;
28 u8 target_private[] __aligned(8);
29};
30
31static struct list_head targets = LIST_HEAD_INIT(targets);
32static DEFINE_MUTEX(targets_mutex);
33
34/* protect bpf_iter_link changes */
35static DEFINE_MUTEX(link_mutex);
36
37/* incremented on every opened seq_file */
38static atomic64_t session_id;
39
40static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
41 const struct bpf_iter_seq_info *seq_info);
42
43static void bpf_iter_inc_seq_num(struct seq_file *seq)
44{
45 struct bpf_iter_priv_data *iter_priv;
46
47 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
48 target_private);
49 iter_priv->seq_num++;
50}
51
52static void bpf_iter_dec_seq_num(struct seq_file *seq)
53{
54 struct bpf_iter_priv_data *iter_priv;
55
56 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
57 target_private);
58 iter_priv->seq_num--;
59}
60
61static void bpf_iter_done_stop(struct seq_file *seq)
62{
63 struct bpf_iter_priv_data *iter_priv;
64
65 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
66 target_private);
67 iter_priv->done_stop = true;
68}
69
70static bool bpf_iter_support_resched(struct seq_file *seq)
71{
72 struct bpf_iter_priv_data *iter_priv;
73
74 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
75 target_private);
76 return iter_priv->tinfo->reg_info->feature & BPF_ITER_RESCHED;
77}
78
79/* maximum visited objects before bailing out */
80#define MAX_ITER_OBJECTS 1000000
81
82/* bpf_seq_read, a customized and simpler version for bpf iterator.
83 * no_llseek is assumed for this file.
84 * The following are differences from seq_read():
85 * . fixed buffer size (PAGE_SIZE)
86 * . assuming no_llseek
87 * . stop() may call bpf program, handling potential overflow there
88 */
89static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
90 loff_t *ppos)
91{
92 struct seq_file *seq = file->private_data;
93 size_t n, offs, copied = 0;
94 int err = 0, num_objs = 0;
95 bool can_resched;
96 void *p;
97
98 mutex_lock(&seq->lock);
99
100 if (!seq->buf) {
101 seq->size = PAGE_SIZE << 3;
102 seq->buf = kvmalloc(seq->size, GFP_KERNEL);
103 if (!seq->buf) {
104 err = -ENOMEM;
105 goto done;
106 }
107 }
108
109 if (seq->count) {
110 n = min(seq->count, size);
111 err = copy_to_user(buf, seq->buf + seq->from, n);
112 if (err) {
113 err = -EFAULT;
114 goto done;
115 }
116 seq->count -= n;
117 seq->from += n;
118 copied = n;
119 goto done;
120 }
121
122 seq->from = 0;
123 p = seq->op->start(seq, &seq->index);
124 if (!p)
125 goto stop;
126 if (IS_ERR(p)) {
127 err = PTR_ERR(p);
128 seq->op->stop(seq, p);
129 seq->count = 0;
130 goto done;
131 }
132
133 err = seq->op->show(seq, p);
134 if (err > 0) {
135 /* object is skipped, decrease seq_num, so next
136 * valid object can reuse the same seq_num.
137 */
138 bpf_iter_dec_seq_num(seq);
139 seq->count = 0;
140 } else if (err < 0 || seq_has_overflowed(seq)) {
141 if (!err)
142 err = -E2BIG;
143 seq->op->stop(seq, p);
144 seq->count = 0;
145 goto done;
146 }
147
148 can_resched = bpf_iter_support_resched(seq);
149 while (1) {
150 loff_t pos = seq->index;
151
152 num_objs++;
153 offs = seq->count;
154 p = seq->op->next(seq, p, &seq->index);
155 if (pos == seq->index) {
156 pr_info_ratelimited("buggy seq_file .next function %ps "
157 "did not updated position index\n",
158 seq->op->next);
159 seq->index++;
160 }
161
162 if (IS_ERR_OR_NULL(p))
163 break;
164
165 /* got a valid next object, increase seq_num */
166 bpf_iter_inc_seq_num(seq);
167
168 if (seq->count >= size)
169 break;
170
171 if (num_objs >= MAX_ITER_OBJECTS) {
172 if (offs == 0) {
173 err = -EAGAIN;
174 seq->op->stop(seq, p);
175 goto done;
176 }
177 break;
178 }
179
180 err = seq->op->show(seq, p);
181 if (err > 0) {
182 bpf_iter_dec_seq_num(seq);
183 seq->count = offs;
184 } else if (err < 0 || seq_has_overflowed(seq)) {
185 seq->count = offs;
186 if (offs == 0) {
187 if (!err)
188 err = -E2BIG;
189 seq->op->stop(seq, p);
190 goto done;
191 }
192 break;
193 }
194
195 if (can_resched)
196 cond_resched();
197 }
198stop:
199 offs = seq->count;
200 /* bpf program called if !p */
201 seq->op->stop(seq, p);
202 if (!p) {
203 if (!seq_has_overflowed(seq)) {
204 bpf_iter_done_stop(seq);
205 } else {
206 seq->count = offs;
207 if (offs == 0) {
208 err = -E2BIG;
209 goto done;
210 }
211 }
212 }
213
214 n = min(seq->count, size);
215 err = copy_to_user(buf, seq->buf, n);
216 if (err) {
217 err = -EFAULT;
218 goto done;
219 }
220 copied = n;
221 seq->count -= n;
222 seq->from = n;
223done:
224 if (!copied)
225 copied = err;
226 else
227 *ppos += copied;
228 mutex_unlock(&seq->lock);
229 return copied;
230}
231
232static const struct bpf_iter_seq_info *
233__get_seq_info(struct bpf_iter_link *link)
234{
235 const struct bpf_iter_seq_info *seq_info;
236
237 if (link->aux.map) {
238 seq_info = link->aux.map->ops->iter_seq_info;
239 if (seq_info)
240 return seq_info;
241 }
242
243 return link->tinfo->reg_info->seq_info;
244}
245
246static int iter_open(struct inode *inode, struct file *file)
247{
248 struct bpf_iter_link *link = inode->i_private;
249
250 return prepare_seq_file(file, link, __get_seq_info(link));
251}
252
253static int iter_release(struct inode *inode, struct file *file)
254{
255 struct bpf_iter_priv_data *iter_priv;
256 struct seq_file *seq;
257
258 seq = file->private_data;
259 if (!seq)
260 return 0;
261
262 iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
263 target_private);
264
265 if (iter_priv->seq_info->fini_seq_private)
266 iter_priv->seq_info->fini_seq_private(seq->private);
267
268 bpf_prog_put(iter_priv->prog);
269 seq->private = iter_priv;
270
271 return seq_release_private(inode, file);
272}
273
274const struct file_operations bpf_iter_fops = {
275 .open = iter_open,
276 .llseek = no_llseek,
277 .read = bpf_seq_read,
278 .release = iter_release,
279};
280
281/* The argument reg_info will be cached in bpf_iter_target_info.
282 * The common practice is to declare target reg_info as
283 * a const static variable and passed as an argument to
284 * bpf_iter_reg_target().
285 */
286int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
287{
288 struct bpf_iter_target_info *tinfo;
289
290 tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
291 if (!tinfo)
292 return -ENOMEM;
293
294 tinfo->reg_info = reg_info;
295 INIT_LIST_HEAD(&tinfo->list);
296
297 mutex_lock(&targets_mutex);
298 list_add(&tinfo->list, &targets);
299 mutex_unlock(&targets_mutex);
300
301 return 0;
302}
303
304void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
305{
306 struct bpf_iter_target_info *tinfo;
307 bool found = false;
308
309 mutex_lock(&targets_mutex);
310 list_for_each_entry(tinfo, &targets, list) {
311 if (reg_info == tinfo->reg_info) {
312 list_del(&tinfo->list);
313 kfree(tinfo);
314 found = true;
315 break;
316 }
317 }
318 mutex_unlock(&targets_mutex);
319
320 WARN_ON(found == false);
321}
322
323static void cache_btf_id(struct bpf_iter_target_info *tinfo,
324 struct bpf_prog *prog)
325{
326 tinfo->btf_id = prog->aux->attach_btf_id;
327}
328
329bool bpf_iter_prog_supported(struct bpf_prog *prog)
330{
331 const char *attach_fname = prog->aux->attach_func_name;
332 u32 prog_btf_id = prog->aux->attach_btf_id;
333 const char *prefix = BPF_ITER_FUNC_PREFIX;
334 struct bpf_iter_target_info *tinfo;
335 int prefix_len = strlen(prefix);
336 bool supported = false;
337
338 if (strncmp(attach_fname, prefix, prefix_len))
339 return false;
340
341 mutex_lock(&targets_mutex);
342 list_for_each_entry(tinfo, &targets, list) {
343 if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
344 supported = true;
345 break;
346 }
347 if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
348 cache_btf_id(tinfo, prog);
349 supported = true;
350 break;
351 }
352 }
353 mutex_unlock(&targets_mutex);
354
355 if (supported) {
356 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
357 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
358 }
359
360 return supported;
361}
362
363static void bpf_iter_link_release(struct bpf_link *link)
364{
365 struct bpf_iter_link *iter_link =
366 container_of(link, struct bpf_iter_link, link);
367
368 if (iter_link->tinfo->reg_info->detach_target)
369 iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
370}
371
372static void bpf_iter_link_dealloc(struct bpf_link *link)
373{
374 struct bpf_iter_link *iter_link =
375 container_of(link, struct bpf_iter_link, link);
376
377 kfree(iter_link);
378}
379
380static int bpf_iter_link_replace(struct bpf_link *link,
381 struct bpf_prog *new_prog,
382 struct bpf_prog *old_prog)
383{
384 int ret = 0;
385
386 mutex_lock(&link_mutex);
387 if (old_prog && link->prog != old_prog) {
388 ret = -EPERM;
389 goto out_unlock;
390 }
391
392 if (link->prog->type != new_prog->type ||
393 link->prog->expected_attach_type != new_prog->expected_attach_type ||
394 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
395 ret = -EINVAL;
396 goto out_unlock;
397 }
398
399 old_prog = xchg(&link->prog, new_prog);
400 bpf_prog_put(old_prog);
401
402out_unlock:
403 mutex_unlock(&link_mutex);
404 return ret;
405}
406
407static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
408 struct seq_file *seq)
409{
410 struct bpf_iter_link *iter_link =
411 container_of(link, struct bpf_iter_link, link);
412 bpf_iter_show_fdinfo_t show_fdinfo;
413
414 seq_printf(seq,
415 "target_name:\t%s\n",
416 iter_link->tinfo->reg_info->target);
417
418 show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
419 if (show_fdinfo)
420 show_fdinfo(&iter_link->aux, seq);
421}
422
423static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
424 struct bpf_link_info *info)
425{
426 struct bpf_iter_link *iter_link =
427 container_of(link, struct bpf_iter_link, link);
428 char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
429 bpf_iter_fill_link_info_t fill_link_info;
430 u32 ulen = info->iter.target_name_len;
431 const char *target_name;
432 u32 target_len;
433
434 if (!ulen ^ !ubuf)
435 return -EINVAL;
436
437 target_name = iter_link->tinfo->reg_info->target;
438 target_len = strlen(target_name);
439 info->iter.target_name_len = target_len + 1;
440
441 if (ubuf) {
442 if (ulen >= target_len + 1) {
443 if (copy_to_user(ubuf, target_name, target_len + 1))
444 return -EFAULT;
445 } else {
446 char zero = '\0';
447
448 if (copy_to_user(ubuf, target_name, ulen - 1))
449 return -EFAULT;
450 if (put_user(zero, ubuf + ulen - 1))
451 return -EFAULT;
452 return -ENOSPC;
453 }
454 }
455
456 fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
457 if (fill_link_info)
458 return fill_link_info(&iter_link->aux, info);
459
460 return 0;
461}
462
463static const struct bpf_link_ops bpf_iter_link_lops = {
464 .release = bpf_iter_link_release,
465 .dealloc = bpf_iter_link_dealloc,
466 .update_prog = bpf_iter_link_replace,
467 .show_fdinfo = bpf_iter_link_show_fdinfo,
468 .fill_link_info = bpf_iter_link_fill_link_info,
469};
470
471bool bpf_link_is_iter(struct bpf_link *link)
472{
473 return link->ops == &bpf_iter_link_lops;
474}
475
476int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
477 struct bpf_prog *prog)
478{
479 struct bpf_link_primer link_primer;
480 struct bpf_iter_target_info *tinfo;
481 union bpf_iter_link_info linfo;
482 struct bpf_iter_link *link;
483 u32 prog_btf_id, linfo_len;
484 bool existed = false;
485 bpfptr_t ulinfo;
486 int err;
487
488 if (attr->link_create.target_fd || attr->link_create.flags)
489 return -EINVAL;
490
491 memset(&linfo, 0, sizeof(union bpf_iter_link_info));
492
493 ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
494 linfo_len = attr->link_create.iter_info_len;
495 if (bpfptr_is_null(ulinfo) ^ !linfo_len)
496 return -EINVAL;
497
498 if (!bpfptr_is_null(ulinfo)) {
499 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
500 linfo_len);
501 if (err)
502 return err;
503 linfo_len = min_t(u32, linfo_len, sizeof(linfo));
504 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
505 return -EFAULT;
506 }
507
508 prog_btf_id = prog->aux->attach_btf_id;
509 mutex_lock(&targets_mutex);
510 list_for_each_entry(tinfo, &targets, list) {
511 if (tinfo->btf_id == prog_btf_id) {
512 existed = true;
513 break;
514 }
515 }
516 mutex_unlock(&targets_mutex);
517 if (!existed)
518 return -ENOENT;
519
520 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
521 if (!link)
522 return -ENOMEM;
523
524 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
525 link->tinfo = tinfo;
526
527 err = bpf_link_prime(&link->link, &link_primer);
528 if (err) {
529 kfree(link);
530 return err;
531 }
532
533 if (tinfo->reg_info->attach_target) {
534 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
535 if (err) {
536 bpf_link_cleanup(&link_primer);
537 return err;
538 }
539 }
540
541 return bpf_link_settle(&link_primer);
542}
543
544static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
545 struct bpf_iter_target_info *tinfo,
546 const struct bpf_iter_seq_info *seq_info,
547 struct bpf_prog *prog)
548{
549 priv_data->tinfo = tinfo;
550 priv_data->seq_info = seq_info;
551 priv_data->prog = prog;
552 priv_data->session_id = atomic64_inc_return(&session_id);
553 priv_data->seq_num = 0;
554 priv_data->done_stop = false;
555}
556
557static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
558 const struct bpf_iter_seq_info *seq_info)
559{
560 struct bpf_iter_priv_data *priv_data;
561 struct bpf_iter_target_info *tinfo;
562 struct bpf_prog *prog;
563 u32 total_priv_dsize;
564 struct seq_file *seq;
565 int err = 0;
566
567 mutex_lock(&link_mutex);
568 prog = link->link.prog;
569 bpf_prog_inc(prog);
570 mutex_unlock(&link_mutex);
571
572 tinfo = link->tinfo;
573 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
574 seq_info->seq_priv_size;
575 priv_data = __seq_open_private(file, seq_info->seq_ops,
576 total_priv_dsize);
577 if (!priv_data) {
578 err = -ENOMEM;
579 goto release_prog;
580 }
581
582 if (seq_info->init_seq_private) {
583 err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
584 if (err)
585 goto release_seq_file;
586 }
587
588 init_seq_meta(priv_data, tinfo, seq_info, prog);
589 seq = file->private_data;
590 seq->private = priv_data->target_private;
591
592 return 0;
593
594release_seq_file:
595 seq_release_private(file->f_inode, file);
596 file->private_data = NULL;
597release_prog:
598 bpf_prog_put(prog);
599 return err;
600}
601
602int bpf_iter_new_fd(struct bpf_link *link)
603{
604 struct bpf_iter_link *iter_link;
605 struct file *file;
606 unsigned int flags;
607 int err, fd;
608
609 if (link->ops != &bpf_iter_link_lops)
610 return -EINVAL;
611
612 flags = O_RDONLY | O_CLOEXEC;
613 fd = get_unused_fd_flags(flags);
614 if (fd < 0)
615 return fd;
616
617 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
618 if (IS_ERR(file)) {
619 err = PTR_ERR(file);
620 goto free_fd;
621 }
622
623 iter_link = container_of(link, struct bpf_iter_link, link);
624 err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
625 if (err)
626 goto free_file;
627
628 fd_install(fd, file);
629 return fd;
630
631free_file:
632 fput(file);
633free_fd:
634 put_unused_fd(fd);
635 return err;
636}
637
638struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
639{
640 struct bpf_iter_priv_data *iter_priv;
641 struct seq_file *seq;
642 void *seq_priv;
643
644 seq = meta->seq;
645 if (seq->file->f_op != &bpf_iter_fops)
646 return NULL;
647
648 seq_priv = seq->private;
649 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
650 target_private);
651
652 if (in_stop && iter_priv->done_stop)
653 return NULL;
654
655 meta->session_id = iter_priv->session_id;
656 meta->seq_num = iter_priv->seq_num;
657
658 return iter_priv->prog;
659}
660
661int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
662{
663 int ret;
664
665 rcu_read_lock();
666 migrate_disable();
667 ret = BPF_PROG_RUN(prog, ctx);
668 migrate_enable();
669 rcu_read_unlock();
670
671 /* bpf program can only return 0 or 1:
672 * 0 : okay
673 * 1 : retry the same object
674 * The bpf_iter_run_prog() return value
675 * will be seq_ops->show() return value.
676 */
677 return ret == 0 ? 0 : -EAGAIN;
678}
679
680BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
681 void *, callback_ctx, u64, flags)
682{
683 return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
684}
685
686const struct bpf_func_proto bpf_for_each_map_elem_proto = {
687 .func = bpf_for_each_map_elem,
688 .gpl_only = false,
689 .ret_type = RET_INTEGER,
690 .arg1_type = ARG_CONST_MAP_PTR,
691 .arg2_type = ARG_PTR_TO_FUNC,
692 .arg3_type = ARG_PTR_TO_STACK_OR_NULL,
693 .arg4_type = ARG_ANYTHING,
694};