Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_tracing.h>
6#include <bpf/bpf_helpers.h>
7#include <bpf/bpf_core_read.h>
8#include "../bpf_experimental.h"
9#include "../bpf_testmod/bpf_testmod_kfunc.h"
10
11struct plain_local;
12
13struct node_data {
14 long key;
15 long data;
16 struct plain_local __kptr * stashed_in_local_kptr;
17 struct bpf_rb_node node;
18};
19
20struct refcounted_node {
21 long data;
22 struct bpf_rb_node rb_node;
23 struct bpf_refcount refcount;
24};
25
26struct stash {
27 struct bpf_spin_lock l;
28 struct refcounted_node __kptr *stashed;
29};
30
31struct {
32 __uint(type, BPF_MAP_TYPE_ARRAY);
33 __type(key, int);
34 __type(value, struct stash);
35 __uint(max_entries, 10);
36} refcounted_node_stash SEC(".maps");
37
38struct plain_local {
39 long key;
40 long data;
41};
42
43struct local_with_root {
44 long key;
45 struct bpf_spin_lock l;
46 struct bpf_rb_root r __contains(node_data, node);
47};
48
49struct map_value {
50 struct prog_test_ref_kfunc *not_kptr;
51 struct prog_test_ref_kfunc __kptr *val;
52 struct node_data __kptr *node;
53 struct plain_local __kptr *plain;
54 struct local_with_root __kptr *local_root;
55};
56
57/* This is necessary so that LLVM generates BTF for node_data struct
58 * If it's not included, a fwd reference for node_data will be generated but
59 * no struct. Example BTF of "node" field in map_value when not included:
60 *
61 * [10] PTR '(anon)' type_id=35
62 * [34] FWD 'node_data' fwd_kind=struct
63 * [35] TYPE_TAG 'kptr_ref' type_id=34
64 *
65 * (with no node_data struct defined)
66 * Had to do the same w/ bpf_kfunc_call_test_release below
67 */
68struct node_data *just_here_because_btf_bug;
69struct refcounted_node *just_here_because_btf_bug2;
70
71struct {
72 __uint(type, BPF_MAP_TYPE_ARRAY);
73 __type(key, int);
74 __type(value, struct map_value);
75 __uint(max_entries, 2);
76} some_nodes SEC(".maps");
77
78static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
79{
80 struct node_data *node_a;
81 struct node_data *node_b;
82
83 node_a = container_of(a, struct node_data, node);
84 node_b = container_of(b, struct node_data, node);
85
86 return node_a->key < node_b->key;
87}
88
89static int create_and_stash(int idx, int val)
90{
91 struct plain_local *inner_local_kptr;
92 struct map_value *mapval;
93 struct node_data *res;
94
95 mapval = bpf_map_lookup_elem(&some_nodes, &idx);
96 if (!mapval)
97 return 1;
98
99 inner_local_kptr = bpf_obj_new(typeof(*inner_local_kptr));
100 if (!inner_local_kptr)
101 return 2;
102
103 res = bpf_obj_new(typeof(*res));
104 if (!res) {
105 bpf_obj_drop(inner_local_kptr);
106 return 3;
107 }
108 res->key = val;
109
110 inner_local_kptr = bpf_kptr_xchg(&res->stashed_in_local_kptr, inner_local_kptr);
111 if (inner_local_kptr) {
112 /* Should never happen, we just obj_new'd res */
113 bpf_obj_drop(inner_local_kptr);
114 bpf_obj_drop(res);
115 return 4;
116 }
117
118 res = bpf_kptr_xchg(&mapval->node, res);
119 if (res)
120 bpf_obj_drop(res);
121 return 0;
122}
123
124SEC("tc")
125long stash_rb_nodes(void *ctx)
126{
127 return create_and_stash(0, 41) ?: create_and_stash(1, 42);
128}
129
130SEC("tc")
131long stash_plain(void *ctx)
132{
133 struct map_value *mapval;
134 struct plain_local *res;
135 int idx = 0;
136
137 mapval = bpf_map_lookup_elem(&some_nodes, &idx);
138 if (!mapval)
139 return 1;
140
141 res = bpf_obj_new(typeof(*res));
142 if (!res)
143 return 1;
144 res->key = 41;
145
146 res = bpf_kptr_xchg(&mapval->plain, res);
147 if (res)
148 bpf_obj_drop(res);
149 return 0;
150}
151
152SEC("tc")
153long stash_local_with_root(void *ctx)
154{
155 struct local_with_root *res;
156 struct map_value *mapval;
157 struct node_data *n;
158 int idx = 0;
159
160 mapval = bpf_map_lookup_elem(&some_nodes, &idx);
161 if (!mapval)
162 return 1;
163
164 res = bpf_obj_new(typeof(*res));
165 if (!res)
166 return 2;
167 res->key = 41;
168
169 n = bpf_obj_new(typeof(*n));
170 if (!n) {
171 bpf_obj_drop(res);
172 return 3;
173 }
174
175 bpf_spin_lock(&res->l);
176 bpf_rbtree_add(&res->r, &n->node, less);
177 bpf_spin_unlock(&res->l);
178
179 res = bpf_kptr_xchg(&mapval->local_root, res);
180 if (res) {
181 bpf_obj_drop(res);
182 return 4;
183 }
184 return 0;
185}
186
187SEC("tc")
188long unstash_rb_node(void *ctx)
189{
190 struct plain_local *inner_local_kptr = NULL;
191 struct map_value *mapval;
192 struct node_data *res;
193 long retval;
194 int key = 1;
195
196 mapval = bpf_map_lookup_elem(&some_nodes, &key);
197 if (!mapval)
198 return 1;
199
200 res = bpf_kptr_xchg(&mapval->node, NULL);
201 if (res) {
202 inner_local_kptr = bpf_kptr_xchg(&res->stashed_in_local_kptr, inner_local_kptr);
203 if (!inner_local_kptr) {
204 bpf_obj_drop(res);
205 return 1;
206 }
207 bpf_obj_drop(inner_local_kptr);
208
209 retval = res->key;
210 bpf_obj_drop(res);
211 return retval;
212 }
213 return 1;
214}
215
216SEC("tc")
217long stash_test_ref_kfunc(void *ctx)
218{
219 struct prog_test_ref_kfunc *res;
220 struct map_value *mapval;
221 int key = 0;
222
223 mapval = bpf_map_lookup_elem(&some_nodes, &key);
224 if (!mapval)
225 return 1;
226
227 res = bpf_kptr_xchg(&mapval->val, NULL);
228 if (res)
229 bpf_kfunc_call_test_release(res);
230 return 0;
231}
232
233SEC("tc")
234long refcount_acquire_without_unstash(void *ctx)
235{
236 struct refcounted_node *p;
237 struct stash *s;
238 int ret = 0;
239
240 s = bpf_map_lookup_elem(&refcounted_node_stash, &ret);
241 if (!s)
242 return 1;
243
244 if (!s->stashed)
245 /* refcount_acquire failure is expected when no refcounted_node
246 * has been stashed before this program executes
247 */
248 return 2;
249
250 p = bpf_refcount_acquire(s->stashed);
251 if (!p)
252 return 3;
253
254 ret = s->stashed ? s->stashed->data : -1;
255 bpf_obj_drop(p);
256 return ret;
257}
258
259/* Helper for refcount_acquire_without_unstash test */
260SEC("tc")
261long stash_refcounted_node(void *ctx)
262{
263 struct refcounted_node *p;
264 struct stash *s;
265 int key = 0;
266
267 s = bpf_map_lookup_elem(&refcounted_node_stash, &key);
268 if (!s)
269 return 1;
270
271 p = bpf_obj_new(typeof(*p));
272 if (!p)
273 return 2;
274 p->data = 42;
275
276 p = bpf_kptr_xchg(&s->stashed, p);
277 if (p) {
278 bpf_obj_drop(p);
279 return 3;
280 }
281
282 return 0;
283}
284
285char _license[] SEC("license") = "GPL";
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_tracing.h>
6#include <bpf/bpf_helpers.h>
7#include <bpf/bpf_core_read.h>
8#include "../bpf_experimental.h"
9#include "../bpf_testmod/bpf_testmod_kfunc.h"
10
11struct node_data {
12 long key;
13 long data;
14 struct bpf_rb_node node;
15};
16
17struct refcounted_node {
18 long data;
19 struct bpf_rb_node rb_node;
20 struct bpf_refcount refcount;
21};
22
23struct stash {
24 struct bpf_spin_lock l;
25 struct refcounted_node __kptr *stashed;
26};
27
28struct {
29 __uint(type, BPF_MAP_TYPE_ARRAY);
30 __type(key, int);
31 __type(value, struct stash);
32 __uint(max_entries, 10);
33} refcounted_node_stash SEC(".maps");
34
35struct plain_local {
36 long key;
37 long data;
38};
39
40struct local_with_root {
41 long key;
42 struct bpf_spin_lock l;
43 struct bpf_rb_root r __contains(node_data, node);
44};
45
46struct map_value {
47 struct prog_test_ref_kfunc *not_kptr;
48 struct prog_test_ref_kfunc __kptr *val;
49 struct node_data __kptr *node;
50 struct plain_local __kptr *plain;
51 struct local_with_root __kptr *local_root;
52};
53
54/* This is necessary so that LLVM generates BTF for node_data struct
55 * If it's not included, a fwd reference for node_data will be generated but
56 * no struct. Example BTF of "node" field in map_value when not included:
57 *
58 * [10] PTR '(anon)' type_id=35
59 * [34] FWD 'node_data' fwd_kind=struct
60 * [35] TYPE_TAG 'kptr_ref' type_id=34
61 *
62 * (with no node_data struct defined)
63 * Had to do the same w/ bpf_kfunc_call_test_release below
64 */
65struct node_data *just_here_because_btf_bug;
66struct refcounted_node *just_here_because_btf_bug2;
67
68struct {
69 __uint(type, BPF_MAP_TYPE_ARRAY);
70 __type(key, int);
71 __type(value, struct map_value);
72 __uint(max_entries, 2);
73} some_nodes SEC(".maps");
74
75static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
76{
77 struct node_data *node_a;
78 struct node_data *node_b;
79
80 node_a = container_of(a, struct node_data, node);
81 node_b = container_of(b, struct node_data, node);
82
83 return node_a->key < node_b->key;
84}
85
86static int create_and_stash(int idx, int val)
87{
88 struct map_value *mapval;
89 struct node_data *res;
90
91 mapval = bpf_map_lookup_elem(&some_nodes, &idx);
92 if (!mapval)
93 return 1;
94
95 res = bpf_obj_new(typeof(*res));
96 if (!res)
97 return 1;
98 res->key = val;
99
100 res = bpf_kptr_xchg(&mapval->node, res);
101 if (res)
102 bpf_obj_drop(res);
103 return 0;
104}
105
106SEC("tc")
107long stash_rb_nodes(void *ctx)
108{
109 return create_and_stash(0, 41) ?: create_and_stash(1, 42);
110}
111
112SEC("tc")
113long stash_plain(void *ctx)
114{
115 struct map_value *mapval;
116 struct plain_local *res;
117 int idx = 0;
118
119 mapval = bpf_map_lookup_elem(&some_nodes, &idx);
120 if (!mapval)
121 return 1;
122
123 res = bpf_obj_new(typeof(*res));
124 if (!res)
125 return 1;
126 res->key = 41;
127
128 res = bpf_kptr_xchg(&mapval->plain, res);
129 if (res)
130 bpf_obj_drop(res);
131 return 0;
132}
133
134SEC("tc")
135long stash_local_with_root(void *ctx)
136{
137 struct local_with_root *res;
138 struct map_value *mapval;
139 struct node_data *n;
140 int idx = 0;
141
142 mapval = bpf_map_lookup_elem(&some_nodes, &idx);
143 if (!mapval)
144 return 1;
145
146 res = bpf_obj_new(typeof(*res));
147 if (!res)
148 return 2;
149 res->key = 41;
150
151 n = bpf_obj_new(typeof(*n));
152 if (!n) {
153 bpf_obj_drop(res);
154 return 3;
155 }
156
157 bpf_spin_lock(&res->l);
158 bpf_rbtree_add(&res->r, &n->node, less);
159 bpf_spin_unlock(&res->l);
160
161 res = bpf_kptr_xchg(&mapval->local_root, res);
162 if (res) {
163 bpf_obj_drop(res);
164 return 4;
165 }
166 return 0;
167}
168
169SEC("tc")
170long unstash_rb_node(void *ctx)
171{
172 struct map_value *mapval;
173 struct node_data *res;
174 long retval;
175 int key = 1;
176
177 mapval = bpf_map_lookup_elem(&some_nodes, &key);
178 if (!mapval)
179 return 1;
180
181 res = bpf_kptr_xchg(&mapval->node, NULL);
182 if (res) {
183 retval = res->key;
184 bpf_obj_drop(res);
185 return retval;
186 }
187 return 1;
188}
189
190SEC("tc")
191long stash_test_ref_kfunc(void *ctx)
192{
193 struct prog_test_ref_kfunc *res;
194 struct map_value *mapval;
195 int key = 0;
196
197 mapval = bpf_map_lookup_elem(&some_nodes, &key);
198 if (!mapval)
199 return 1;
200
201 res = bpf_kptr_xchg(&mapval->val, NULL);
202 if (res)
203 bpf_kfunc_call_test_release(res);
204 return 0;
205}
206
207SEC("tc")
208long refcount_acquire_without_unstash(void *ctx)
209{
210 struct refcounted_node *p;
211 struct stash *s;
212 int ret = 0;
213
214 s = bpf_map_lookup_elem(&refcounted_node_stash, &ret);
215 if (!s)
216 return 1;
217
218 if (!s->stashed)
219 /* refcount_acquire failure is expected when no refcounted_node
220 * has been stashed before this program executes
221 */
222 return 2;
223
224 p = bpf_refcount_acquire(s->stashed);
225 if (!p)
226 return 3;
227
228 ret = s->stashed ? s->stashed->data : -1;
229 bpf_obj_drop(p);
230 return ret;
231}
232
233/* Helper for refcount_acquire_without_unstash test */
234SEC("tc")
235long stash_refcounted_node(void *ctx)
236{
237 struct refcounted_node *p;
238 struct stash *s;
239 int key = 0;
240
241 s = bpf_map_lookup_elem(&refcounted_node_stash, &key);
242 if (!s)
243 return 1;
244
245 p = bpf_obj_new(typeof(*p));
246 if (!p)
247 return 2;
248 p->data = 42;
249
250 p = bpf_kptr_xchg(&s->stashed, p);
251 if (p) {
252 bpf_obj_drop(p);
253 return 3;
254 }
255
256 return 0;
257}
258
259char _license[] SEC("license") = "GPL";