Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2023. Huawei Technologies Co., Ltd */
3#include <vmlinux.h>
4#include <bpf/bpf_tracing.h>
5#include <bpf/bpf_helpers.h>
6
7#include "bpf_experimental.h"
8#include "bpf_misc.h"
9
10struct generic_map_value {
11 void *data;
12};
13
14char _license[] SEC("license") = "GPL";
15
16const unsigned int data_sizes[] = {16, 32, 64, 96, 128, 192, 256, 512, 1024, 2048, 4096};
17const volatile unsigned int data_btf_ids[ARRAY_SIZE(data_sizes)] = {};
18
19const unsigned int percpu_data_sizes[] = {8, 16, 32, 64, 96, 128, 192, 256, 512};
20const volatile unsigned int percpu_data_btf_ids[ARRAY_SIZE(data_sizes)] = {};
21
22int err = 0;
23u32 pid = 0;
24
25#define DEFINE_ARRAY_WITH_KPTR(_size) \
26 struct bin_data_##_size { \
27 char data[_size - sizeof(void *)]; \
28 }; \
29 /* See Commit 5d8d6634ccc, force btf generation for type bin_data_##_size */ \
30 struct bin_data_##_size *__bin_data_##_size; \
31 struct map_value_##_size { \
32 struct bin_data_##_size __kptr * data; \
33 }; \
34 struct { \
35 __uint(type, BPF_MAP_TYPE_ARRAY); \
36 __type(key, int); \
37 __type(value, struct map_value_##_size); \
38 __uint(max_entries, 128); \
39 } array_##_size SEC(".maps")
40
41#define DEFINE_ARRAY_WITH_PERCPU_KPTR(_size) \
42 struct percpu_bin_data_##_size { \
43 char data[_size]; \
44 }; \
45 struct percpu_bin_data_##_size *__percpu_bin_data_##_size; \
46 struct map_value_percpu_##_size { \
47 struct percpu_bin_data_##_size __percpu_kptr * data; \
48 }; \
49 struct { \
50 __uint(type, BPF_MAP_TYPE_ARRAY); \
51 __type(key, int); \
52 __type(value, struct map_value_percpu_##_size); \
53 __uint(max_entries, 128); \
54 } array_percpu_##_size SEC(".maps")
55
56static __always_inline void batch_alloc(struct bpf_map *map, unsigned int batch, unsigned int idx)
57{
58 struct generic_map_value *value;
59 unsigned int i, key;
60 void *old, *new;
61
62 for (i = 0; i < batch; i++) {
63 key = i;
64 value = bpf_map_lookup_elem(map, &key);
65 if (!value) {
66 err = 1;
67 return;
68 }
69 new = bpf_obj_new_impl(data_btf_ids[idx], NULL);
70 if (!new) {
71 err = 2;
72 return;
73 }
74 old = bpf_kptr_xchg(&value->data, new);
75 if (old) {
76 bpf_obj_drop(old);
77 err = 3;
78 return;
79 }
80 }
81}
82
83static __always_inline void batch_free(struct bpf_map *map, unsigned int batch, unsigned int idx)
84{
85 struct generic_map_value *value;
86 unsigned int i, key;
87 void *old;
88
89 for (i = 0; i < batch; i++) {
90 key = i;
91 value = bpf_map_lookup_elem(map, &key);
92 if (!value) {
93 err = 4;
94 return;
95 }
96 old = bpf_kptr_xchg(&value->data, NULL);
97 if (!old) {
98 err = 5;
99 return;
100 }
101 bpf_obj_drop(old);
102 }
103}
104
105static __always_inline void batch_percpu_alloc(struct bpf_map *map, unsigned int batch,
106 unsigned int idx)
107{
108 struct generic_map_value *value;
109 unsigned int i, key;
110 void *old, *new;
111
112 for (i = 0; i < batch; i++) {
113 key = i;
114 value = bpf_map_lookup_elem(map, &key);
115 if (!value) {
116 err = 1;
117 return;
118 }
119 /* per-cpu allocator may not be able to refill in time */
120 new = bpf_percpu_obj_new_impl(percpu_data_btf_ids[idx], NULL);
121 if (!new)
122 continue;
123
124 old = bpf_kptr_xchg(&value->data, new);
125 if (old) {
126 bpf_percpu_obj_drop(old);
127 err = 2;
128 return;
129 }
130 }
131}
132
133static __always_inline void batch_percpu_free(struct bpf_map *map, unsigned int batch,
134 unsigned int idx)
135{
136 struct generic_map_value *value;
137 unsigned int i, key;
138 void *old;
139
140 for (i = 0; i < batch; i++) {
141 key = i;
142 value = bpf_map_lookup_elem(map, &key);
143 if (!value) {
144 err = 3;
145 return;
146 }
147 old = bpf_kptr_xchg(&value->data, NULL);
148 if (!old)
149 continue;
150 bpf_percpu_obj_drop(old);
151 }
152}
153
154#define CALL_BATCH_ALLOC(size, batch, idx) \
155 batch_alloc((struct bpf_map *)(&array_##size), batch, idx)
156
157#define CALL_BATCH_ALLOC_FREE(size, batch, idx) \
158 do { \
159 batch_alloc((struct bpf_map *)(&array_##size), batch, idx); \
160 batch_free((struct bpf_map *)(&array_##size), batch, idx); \
161 } while (0)
162
163#define CALL_BATCH_PERCPU_ALLOC(size, batch, idx) \
164 batch_percpu_alloc((struct bpf_map *)(&array_percpu_##size), batch, idx)
165
166#define CALL_BATCH_PERCPU_ALLOC_FREE(size, batch, idx) \
167 do { \
168 batch_percpu_alloc((struct bpf_map *)(&array_percpu_##size), batch, idx); \
169 batch_percpu_free((struct bpf_map *)(&array_percpu_##size), batch, idx); \
170 } while (0)
171
172/* kptr doesn't support bin_data_8 which is a zero-sized array */
173DEFINE_ARRAY_WITH_KPTR(16);
174DEFINE_ARRAY_WITH_KPTR(32);
175DEFINE_ARRAY_WITH_KPTR(64);
176DEFINE_ARRAY_WITH_KPTR(96);
177DEFINE_ARRAY_WITH_KPTR(128);
178DEFINE_ARRAY_WITH_KPTR(192);
179DEFINE_ARRAY_WITH_KPTR(256);
180DEFINE_ARRAY_WITH_KPTR(512);
181DEFINE_ARRAY_WITH_KPTR(1024);
182DEFINE_ARRAY_WITH_KPTR(2048);
183DEFINE_ARRAY_WITH_KPTR(4096);
184
185DEFINE_ARRAY_WITH_PERCPU_KPTR(8);
186DEFINE_ARRAY_WITH_PERCPU_KPTR(16);
187DEFINE_ARRAY_WITH_PERCPU_KPTR(32);
188DEFINE_ARRAY_WITH_PERCPU_KPTR(64);
189DEFINE_ARRAY_WITH_PERCPU_KPTR(96);
190DEFINE_ARRAY_WITH_PERCPU_KPTR(128);
191DEFINE_ARRAY_WITH_PERCPU_KPTR(192);
192DEFINE_ARRAY_WITH_PERCPU_KPTR(256);
193DEFINE_ARRAY_WITH_PERCPU_KPTR(512);
194
195SEC("?fentry/" SYS_PREFIX "sys_nanosleep")
196int test_batch_alloc_free(void *ctx)
197{
198 if ((u32)bpf_get_current_pid_tgid() != pid)
199 return 0;
200
201 /* Alloc 128 16-bytes objects in batch to trigger refilling,
202 * then free 128 16-bytes objects in batch to trigger freeing.
203 */
204 CALL_BATCH_ALLOC_FREE(16, 128, 0);
205 CALL_BATCH_ALLOC_FREE(32, 128, 1);
206 CALL_BATCH_ALLOC_FREE(64, 128, 2);
207 CALL_BATCH_ALLOC_FREE(96, 128, 3);
208 CALL_BATCH_ALLOC_FREE(128, 128, 4);
209 CALL_BATCH_ALLOC_FREE(192, 128, 5);
210 CALL_BATCH_ALLOC_FREE(256, 128, 6);
211 CALL_BATCH_ALLOC_FREE(512, 64, 7);
212 CALL_BATCH_ALLOC_FREE(1024, 32, 8);
213 CALL_BATCH_ALLOC_FREE(2048, 16, 9);
214 CALL_BATCH_ALLOC_FREE(4096, 8, 10);
215
216 return 0;
217}
218
219SEC("?fentry/" SYS_PREFIX "sys_nanosleep")
220int test_free_through_map_free(void *ctx)
221{
222 if ((u32)bpf_get_current_pid_tgid() != pid)
223 return 0;
224
225 /* Alloc 128 16-bytes objects in batch to trigger refilling,
226 * then free these objects through map free.
227 */
228 CALL_BATCH_ALLOC(16, 128, 0);
229 CALL_BATCH_ALLOC(32, 128, 1);
230 CALL_BATCH_ALLOC(64, 128, 2);
231 CALL_BATCH_ALLOC(96, 128, 3);
232 CALL_BATCH_ALLOC(128, 128, 4);
233 CALL_BATCH_ALLOC(192, 128, 5);
234 CALL_BATCH_ALLOC(256, 128, 6);
235 CALL_BATCH_ALLOC(512, 64, 7);
236 CALL_BATCH_ALLOC(1024, 32, 8);
237 CALL_BATCH_ALLOC(2048, 16, 9);
238 CALL_BATCH_ALLOC(4096, 8, 10);
239
240 return 0;
241}
242
243SEC("?fentry/" SYS_PREFIX "sys_nanosleep")
244int test_batch_percpu_alloc_free(void *ctx)
245{
246 if ((u32)bpf_get_current_pid_tgid() != pid)
247 return 0;
248
249 /* Alloc 128 8-bytes per-cpu objects in batch to trigger refilling,
250 * then free 128 8-bytes per-cpu objects in batch to trigger freeing.
251 */
252 CALL_BATCH_PERCPU_ALLOC_FREE(8, 128, 0);
253 CALL_BATCH_PERCPU_ALLOC_FREE(16, 128, 1);
254 CALL_BATCH_PERCPU_ALLOC_FREE(32, 128, 2);
255 CALL_BATCH_PERCPU_ALLOC_FREE(64, 128, 3);
256 CALL_BATCH_PERCPU_ALLOC_FREE(96, 128, 4);
257 CALL_BATCH_PERCPU_ALLOC_FREE(128, 128, 5);
258 CALL_BATCH_PERCPU_ALLOC_FREE(192, 128, 6);
259 CALL_BATCH_PERCPU_ALLOC_FREE(256, 128, 7);
260 CALL_BATCH_PERCPU_ALLOC_FREE(512, 64, 8);
261
262 return 0;
263}
264
265SEC("?fentry/" SYS_PREFIX "sys_nanosleep")
266int test_percpu_free_through_map_free(void *ctx)
267{
268 if ((u32)bpf_get_current_pid_tgid() != pid)
269 return 0;
270
271 /* Alloc 128 8-bytes per-cpu objects in batch to trigger refilling,
272 * then free these object through map free.
273 */
274 CALL_BATCH_PERCPU_ALLOC(8, 128, 0);
275 CALL_BATCH_PERCPU_ALLOC(16, 128, 1);
276 CALL_BATCH_PERCPU_ALLOC(32, 128, 2);
277 CALL_BATCH_PERCPU_ALLOC(64, 128, 3);
278 CALL_BATCH_PERCPU_ALLOC(96, 128, 4);
279 CALL_BATCH_PERCPU_ALLOC(128, 128, 5);
280 CALL_BATCH_PERCPU_ALLOC(192, 128, 6);
281 CALL_BATCH_PERCPU_ALLOC(256, 128, 7);
282 CALL_BATCH_PERCPU_ALLOC(512, 64, 8);
283
284 return 0;
285}