Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/* Copyright (c) 2021 Facebook */
3#include <stdio.h>
4#include <stdlib.h>
5#include <string.h>
6#include <errno.h>
7#include <linux/filter.h>
8#include "btf.h"
9#include "bpf.h"
10#include "libbpf.h"
11#include "libbpf_internal.h"
12#include "hashmap.h"
13#include "bpf_gen_internal.h"
14#include "skel_internal.h"
15
16#define MAX_USED_MAPS 64
17#define MAX_USED_PROGS 32
18
19/* The following structure describes the stack layout of the loader program.
20 * In addition R6 contains the pointer to context.
21 * R7 contains the result of the last sys_bpf command (typically error or FD).
22 * R9 contains the result of the last sys_close command.
23 *
24 * Naming convention:
25 * ctx - bpf program context
26 * stack - bpf program stack
27 * blob - bpf_attr-s, strings, insns, map data.
28 * All the bytes that loader prog will use for read/write.
29 */
30struct loader_stack {
31 __u32 btf_fd;
32 __u32 map_fd[MAX_USED_MAPS];
33 __u32 prog_fd[MAX_USED_PROGS];
34 __u32 inner_map_fd;
35};
36
37#define stack_off(field) \
38 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
39
40#define attr_field(attr, field) (attr + offsetof(union bpf_attr, field))
41
42static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
43{
44 size_t off = gen->insn_cur - gen->insn_start;
45 void *insn_start;
46
47 if (gen->error)
48 return gen->error;
49 if (size > INT32_MAX || off + size > INT32_MAX) {
50 gen->error = -ERANGE;
51 return -ERANGE;
52 }
53 insn_start = realloc(gen->insn_start, off + size);
54 if (!insn_start) {
55 gen->error = -ENOMEM;
56 free(gen->insn_start);
57 gen->insn_start = NULL;
58 return -ENOMEM;
59 }
60 gen->insn_start = insn_start;
61 gen->insn_cur = insn_start + off;
62 return 0;
63}
64
65static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
66{
67 size_t off = gen->data_cur - gen->data_start;
68 void *data_start;
69
70 if (gen->error)
71 return gen->error;
72 if (size > INT32_MAX || off + size > INT32_MAX) {
73 gen->error = -ERANGE;
74 return -ERANGE;
75 }
76 data_start = realloc(gen->data_start, off + size);
77 if (!data_start) {
78 gen->error = -ENOMEM;
79 free(gen->data_start);
80 gen->data_start = NULL;
81 return -ENOMEM;
82 }
83 gen->data_start = data_start;
84 gen->data_cur = data_start + off;
85 return 0;
86}
87
88static void emit(struct bpf_gen *gen, struct bpf_insn insn)
89{
90 if (realloc_insn_buf(gen, sizeof(insn)))
91 return;
92 memcpy(gen->insn_cur, &insn, sizeof(insn));
93 gen->insn_cur += sizeof(insn);
94}
95
96static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
97{
98 emit(gen, insn1);
99 emit(gen, insn2);
100}
101
102void bpf_gen__init(struct bpf_gen *gen, int log_level)
103{
104 size_t stack_sz = sizeof(struct loader_stack);
105 int i;
106
107 gen->log_level = log_level;
108 /* save ctx pointer into R6 */
109 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
110
111 /* bzero stack */
112 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
113 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
114 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
115 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
116 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
117
118 /* jump over cleanup code */
119 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
120 /* size of cleanup code below */
121 (stack_sz / 4) * 3 + 2));
122
123 /* remember the label where all error branches will jump to */
124 gen->cleanup_label = gen->insn_cur - gen->insn_start;
125 /* emit cleanup code: close all temp FDs */
126 for (i = 0; i < stack_sz; i += 4) {
127 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
128 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
129 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
130 }
131 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */
132 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
133 emit(gen, BPF_EXIT_INSN());
134}
135
136static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
137{
138 void *prev;
139
140 if (realloc_data_buf(gen, size))
141 return 0;
142 prev = gen->data_cur;
143 memcpy(gen->data_cur, data, size);
144 gen->data_cur += size;
145 return prev - gen->data_start;
146}
147
148static int insn_bytes_to_bpf_size(__u32 sz)
149{
150 switch (sz) {
151 case 8: return BPF_DW;
152 case 4: return BPF_W;
153 case 2: return BPF_H;
154 case 1: return BPF_B;
155 default: return -1;
156 }
157}
158
159/* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */
160static void emit_rel_store(struct bpf_gen *gen, int off, int data)
161{
162 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
163 0, 0, 0, data));
164 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
165 0, 0, 0, off));
166 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
167}
168
169/* *(u64 *)(blob + off) = (u64)(void *)(%sp + stack_off) */
170static void emit_rel_store_sp(struct bpf_gen *gen, int off, int stack_off)
171{
172 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_10));
173 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, stack_off));
174 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
175 0, 0, 0, off));
176 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
177}
178
179static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
180 bool check_non_zero)
181{
182 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
183 if (check_non_zero)
184 /* If value in ctx is zero don't update the blob.
185 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c
186 */
187 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
188 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
189 0, 0, 0, off));
190 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
191}
192
193static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
194{
195 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
196 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
197 0, 0, 0, off));
198 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
199}
200
201static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
202{
203 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
204 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
205}
206
207static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
208{
209 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
210 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
211 0, 0, 0, attr));
212 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
213 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
214 /* remember the result in R7 */
215 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
216}
217
218static bool is_simm16(__s64 value)
219{
220 return value == (__s64)(__s16)value;
221}
222
223static void emit_check_err(struct bpf_gen *gen)
224{
225 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
226
227 /* R7 contains result of last sys_bpf command.
228 * if (R7 < 0) goto cleanup;
229 */
230 if (is_simm16(off)) {
231 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
232 } else {
233 gen->error = -ERANGE;
234 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));
235 }
236}
237
238/* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
239static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
240 const char *fmt, va_list args)
241{
242 char buf[1024];
243 int addr, len, ret;
244
245 if (!gen->log_level)
246 return;
247 ret = vsnprintf(buf, sizeof(buf), fmt, args);
248 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0)
249 /* The special case to accommodate common debug_ret():
250 * to avoid specifying BPF_REG_7 and adding " r=%%d" to
251 * prints explicitly.
252 */
253 strcat(buf, " r=%d");
254 len = strlen(buf) + 1;
255 addr = add_data(gen, buf, len);
256
257 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
258 0, 0, 0, addr));
259 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
260 if (reg1 >= 0)
261 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
262 if (reg2 >= 0)
263 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
264 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
265}
266
267static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
268{
269 va_list args;
270
271 va_start(args, fmt);
272 emit_debug(gen, reg1, reg2, fmt, args);
273 va_end(args);
274}
275
276static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
277{
278 va_list args;
279
280 va_start(args, fmt);
281 emit_debug(gen, BPF_REG_7, -1, fmt, args);
282 va_end(args);
283}
284
285static void __emit_sys_close(struct bpf_gen *gen)
286{
287 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
288 /* 2 is the number of the following insns
289 * * 6 is additional insns in debug_regs
290 */
291 2 + (gen->log_level ? 6 : 0)));
292 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
293 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
294 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
295}
296
297static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
298{
299 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
300 __emit_sys_close(gen);
301}
302
303static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
304{
305 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
306 0, 0, 0, blob_off));
307 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
308 __emit_sys_close(gen);
309}
310
311int bpf_gen__finish(struct bpf_gen *gen)
312{
313 int i;
314
315 emit_sys_close_stack(gen, stack_off(btf_fd));
316 for (i = 0; i < gen->nr_progs; i++)
317 move_stack2ctx(gen,
318 sizeof(struct bpf_loader_ctx) +
319 sizeof(struct bpf_map_desc) * gen->nr_maps +
320 sizeof(struct bpf_prog_desc) * i +
321 offsetof(struct bpf_prog_desc, prog_fd), 4,
322 stack_off(prog_fd[i]));
323 for (i = 0; i < gen->nr_maps; i++)
324 move_stack2ctx(gen,
325 sizeof(struct bpf_loader_ctx) +
326 sizeof(struct bpf_map_desc) * i +
327 offsetof(struct bpf_map_desc, map_fd), 4,
328 stack_off(map_fd[i]));
329 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
330 emit(gen, BPF_EXIT_INSN());
331 pr_debug("gen: finish %d\n", gen->error);
332 if (!gen->error) {
333 struct gen_loader_opts *opts = gen->opts;
334
335 opts->insns = gen->insn_start;
336 opts->insns_sz = gen->insn_cur - gen->insn_start;
337 opts->data = gen->data_start;
338 opts->data_sz = gen->data_cur - gen->data_start;
339 }
340 return gen->error;
341}
342
343void bpf_gen__free(struct bpf_gen *gen)
344{
345 if (!gen)
346 return;
347 free(gen->data_start);
348 free(gen->insn_start);
349 free(gen);
350}
351
352void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
353 __u32 btf_raw_size)
354{
355 int attr_size = offsetofend(union bpf_attr, btf_log_level);
356 int btf_data, btf_load_attr;
357 union bpf_attr attr;
358
359 memset(&attr, 0, attr_size);
360 pr_debug("gen: load_btf: size %d\n", btf_raw_size);
361 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
362
363 attr.btf_size = btf_raw_size;
364 btf_load_attr = add_data(gen, &attr, attr_size);
365
366 /* populate union bpf_attr with user provided log details */
367 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
368 offsetof(struct bpf_loader_ctx, log_level), false);
369 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
370 offsetof(struct bpf_loader_ctx, log_size), false);
371 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
372 offsetof(struct bpf_loader_ctx, log_buf), false);
373 /* populate union bpf_attr with a pointer to the BTF data */
374 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
375 /* emit BTF_LOAD command */
376 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
377 debug_ret(gen, "btf_load size %d", btf_raw_size);
378 emit_check_err(gen);
379 /* remember btf_fd in the stack, if successful */
380 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
381}
382
383void bpf_gen__map_create(struct bpf_gen *gen,
384 struct bpf_create_map_attr *map_attr, int map_idx)
385{
386 int attr_size = offsetofend(union bpf_attr, btf_vmlinux_value_type_id);
387 bool close_inner_map_fd = false;
388 int map_create_attr;
389 union bpf_attr attr;
390
391 memset(&attr, 0, attr_size);
392 attr.map_type = map_attr->map_type;
393 attr.key_size = map_attr->key_size;
394 attr.value_size = map_attr->value_size;
395 attr.map_flags = map_attr->map_flags;
396 memcpy(attr.map_name, map_attr->name,
397 min((unsigned)strlen(map_attr->name), BPF_OBJ_NAME_LEN - 1));
398 attr.numa_node = map_attr->numa_node;
399 attr.map_ifindex = map_attr->map_ifindex;
400 attr.max_entries = map_attr->max_entries;
401 switch (attr.map_type) {
402 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
403 case BPF_MAP_TYPE_CGROUP_ARRAY:
404 case BPF_MAP_TYPE_STACK_TRACE:
405 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
406 case BPF_MAP_TYPE_HASH_OF_MAPS:
407 case BPF_MAP_TYPE_DEVMAP:
408 case BPF_MAP_TYPE_DEVMAP_HASH:
409 case BPF_MAP_TYPE_CPUMAP:
410 case BPF_MAP_TYPE_XSKMAP:
411 case BPF_MAP_TYPE_SOCKMAP:
412 case BPF_MAP_TYPE_SOCKHASH:
413 case BPF_MAP_TYPE_QUEUE:
414 case BPF_MAP_TYPE_STACK:
415 case BPF_MAP_TYPE_RINGBUF:
416 break;
417 default:
418 attr.btf_key_type_id = map_attr->btf_key_type_id;
419 attr.btf_value_type_id = map_attr->btf_value_type_id;
420 }
421
422 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n",
423 attr.map_name, map_idx, map_attr->map_type, attr.btf_value_type_id);
424
425 map_create_attr = add_data(gen, &attr, attr_size);
426 if (attr.btf_value_type_id)
427 /* populate union bpf_attr with btf_fd saved in the stack earlier */
428 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
429 stack_off(btf_fd));
430 switch (attr.map_type) {
431 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
432 case BPF_MAP_TYPE_HASH_OF_MAPS:
433 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
434 stack_off(inner_map_fd));
435 close_inner_map_fd = true;
436 break;
437 default:
438 break;
439 }
440 /* conditionally update max_entries */
441 if (map_idx >= 0)
442 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
443 sizeof(struct bpf_loader_ctx) +
444 sizeof(struct bpf_map_desc) * map_idx +
445 offsetof(struct bpf_map_desc, max_entries),
446 true /* check that max_entries != 0 */);
447 /* emit MAP_CREATE command */
448 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
449 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
450 attr.map_name, map_idx, map_attr->map_type, attr.value_size,
451 attr.btf_value_type_id);
452 emit_check_err(gen);
453 /* remember map_fd in the stack, if successful */
454 if (map_idx < 0) {
455 /* This bpf_gen__map_create() function is called with map_idx >= 0
456 * for all maps that libbpf loading logic tracks.
457 * It's called with -1 to create an inner map.
458 */
459 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
460 stack_off(inner_map_fd)));
461 } else if (map_idx != gen->nr_maps) {
462 gen->error = -EDOM; /* internal bug */
463 return;
464 } else {
465 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
466 stack_off(map_fd[map_idx])));
467 gen->nr_maps++;
468 }
469 if (close_inner_map_fd)
470 emit_sys_close_stack(gen, stack_off(inner_map_fd));
471}
472
473void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
474 enum bpf_attach_type type)
475{
476 const char *prefix;
477 int kind, ret;
478
479 btf_get_kernel_prefix_kind(type, &prefix, &kind);
480 gen->attach_kind = kind;
481 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
482 prefix, attach_name);
483 if (ret == sizeof(gen->attach_target))
484 gen->error = -ENOSPC;
485}
486
487static void emit_find_attach_target(struct bpf_gen *gen)
488{
489 int name, len = strlen(gen->attach_target) + 1;
490
491 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
492 name = add_data(gen, gen->attach_target, len);
493
494 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
495 0, 0, 0, name));
496 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
497 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
498 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
499 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
500 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
501 debug_ret(gen, "find_by_name_kind(%s,%d)",
502 gen->attach_target, gen->attach_kind);
503 emit_check_err(gen);
504 /* if successful, btf_id is in lower 32-bit of R7 and
505 * btf_obj_fd is in upper 32-bit
506 */
507}
508
509void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind,
510 int insn_idx)
511{
512 struct ksym_relo_desc *relo;
513
514 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
515 if (!relo) {
516 gen->error = -ENOMEM;
517 return;
518 }
519 gen->relos = relo;
520 relo += gen->relo_cnt;
521 relo->name = name;
522 relo->kind = kind;
523 relo->insn_idx = insn_idx;
524 gen->relo_cnt++;
525}
526
527static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
528{
529 int name, insn, len = strlen(relo->name) + 1;
530
531 pr_debug("gen: emit_relo: %s at %d\n", relo->name, relo->insn_idx);
532 name = add_data(gen, relo->name, len);
533
534 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
535 0, 0, 0, name));
536 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
537 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
538 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
539 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
540 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
541 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
542 emit_check_err(gen);
543 /* store btf_id into insn[insn_idx].imm */
544 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx +
545 offsetof(struct bpf_insn, imm);
546 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
547 0, 0, 0, insn));
548 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 0));
549 if (relo->kind == BTF_KIND_VAR) {
550 /* store btf_obj_fd into insn[insn_idx + 1].imm */
551 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
552 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
553 sizeof(struct bpf_insn)));
554 }
555}
556
557static void emit_relos(struct bpf_gen *gen, int insns)
558{
559 int i;
560
561 for (i = 0; i < gen->relo_cnt; i++)
562 emit_relo(gen, gen->relos + i, insns);
563}
564
565static void cleanup_relos(struct bpf_gen *gen, int insns)
566{
567 int i, insn;
568
569 for (i = 0; i < gen->relo_cnt; i++) {
570 if (gen->relos[i].kind != BTF_KIND_VAR)
571 continue;
572 /* close fd recorded in insn[insn_idx + 1].imm */
573 insn = insns +
574 sizeof(struct bpf_insn) * (gen->relos[i].insn_idx + 1) +
575 offsetof(struct bpf_insn, imm);
576 emit_sys_close_blob(gen, insn);
577 }
578 if (gen->relo_cnt) {
579 free(gen->relos);
580 gen->relo_cnt = 0;
581 gen->relos = NULL;
582 }
583}
584
585void bpf_gen__prog_load(struct bpf_gen *gen,
586 struct bpf_prog_load_params *load_attr, int prog_idx)
587{
588 int attr_size = offsetofend(union bpf_attr, fd_array);
589 int prog_load_attr, license, insns, func_info, line_info;
590 union bpf_attr attr;
591
592 memset(&attr, 0, attr_size);
593 pr_debug("gen: prog_load: type %d insns_cnt %zd\n",
594 load_attr->prog_type, load_attr->insn_cnt);
595 /* add license string to blob of bytes */
596 license = add_data(gen, load_attr->license, strlen(load_attr->license) + 1);
597 /* add insns to blob of bytes */
598 insns = add_data(gen, load_attr->insns,
599 load_attr->insn_cnt * sizeof(struct bpf_insn));
600
601 attr.prog_type = load_attr->prog_type;
602 attr.expected_attach_type = load_attr->expected_attach_type;
603 attr.attach_btf_id = load_attr->attach_btf_id;
604 attr.prog_ifindex = load_attr->prog_ifindex;
605 attr.kern_version = 0;
606 attr.insn_cnt = (__u32)load_attr->insn_cnt;
607 attr.prog_flags = load_attr->prog_flags;
608
609 attr.func_info_rec_size = load_attr->func_info_rec_size;
610 attr.func_info_cnt = load_attr->func_info_cnt;
611 func_info = add_data(gen, load_attr->func_info,
612 attr.func_info_cnt * attr.func_info_rec_size);
613
614 attr.line_info_rec_size = load_attr->line_info_rec_size;
615 attr.line_info_cnt = load_attr->line_info_cnt;
616 line_info = add_data(gen, load_attr->line_info,
617 attr.line_info_cnt * attr.line_info_rec_size);
618
619 memcpy(attr.prog_name, load_attr->name,
620 min((unsigned)strlen(load_attr->name), BPF_OBJ_NAME_LEN - 1));
621 prog_load_attr = add_data(gen, &attr, attr_size);
622
623 /* populate union bpf_attr with a pointer to license */
624 emit_rel_store(gen, attr_field(prog_load_attr, license), license);
625
626 /* populate union bpf_attr with a pointer to instructions */
627 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns);
628
629 /* populate union bpf_attr with a pointer to func_info */
630 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
631
632 /* populate union bpf_attr with a pointer to line_info */
633 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
634
635 /* populate union bpf_attr fd_array with a pointer to stack where map_fds are saved */
636 emit_rel_store_sp(gen, attr_field(prog_load_attr, fd_array),
637 stack_off(map_fd[0]));
638
639 /* populate union bpf_attr with user provided log details */
640 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
641 offsetof(struct bpf_loader_ctx, log_level), false);
642 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
643 offsetof(struct bpf_loader_ctx, log_size), false);
644 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
645 offsetof(struct bpf_loader_ctx, log_buf), false);
646 /* populate union bpf_attr with btf_fd saved in the stack earlier */
647 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
648 stack_off(btf_fd));
649 if (gen->attach_kind) {
650 emit_find_attach_target(gen);
651 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */
652 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
653 0, 0, 0, prog_load_attr));
654 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
655 offsetof(union bpf_attr, attach_btf_id)));
656 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
657 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
658 offsetof(union bpf_attr, attach_btf_obj_fd)));
659 }
660 emit_relos(gen, insns);
661 /* emit PROG_LOAD command */
662 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
663 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
664 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
665 cleanup_relos(gen, insns);
666 if (gen->attach_kind)
667 emit_sys_close_blob(gen,
668 attr_field(prog_load_attr, attach_btf_obj_fd));
669 emit_check_err(gen);
670 /* remember prog_fd in the stack, if successful */
671 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
672 stack_off(prog_fd[gen->nr_progs])));
673 gen->nr_progs++;
674}
675
676void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
677 __u32 value_size)
678{
679 int attr_size = offsetofend(union bpf_attr, flags);
680 int map_update_attr, value, key;
681 union bpf_attr attr;
682 int zero = 0;
683
684 memset(&attr, 0, attr_size);
685 pr_debug("gen: map_update_elem: idx %d\n", map_idx);
686
687 value = add_data(gen, pvalue, value_size);
688 key = add_data(gen, &zero, sizeof(zero));
689
690 /* if (map_desc[map_idx].initial_value)
691 * copy_from_user(value, initial_value, value_size);
692 */
693 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
694 sizeof(struct bpf_loader_ctx) +
695 sizeof(struct bpf_map_desc) * map_idx +
696 offsetof(struct bpf_map_desc, initial_value)));
697 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 4));
698 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
699 0, 0, 0, value));
700 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
701 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
702
703 map_update_attr = add_data(gen, &attr, attr_size);
704 move_stack2blob(gen, attr_field(map_update_attr, map_fd), 4,
705 stack_off(map_fd[map_idx]));
706 emit_rel_store(gen, attr_field(map_update_attr, key), key);
707 emit_rel_store(gen, attr_field(map_update_attr, value), value);
708 /* emit MAP_UPDATE_ELEM command */
709 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
710 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
711 emit_check_err(gen);
712}
713
714void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
715{
716 int attr_size = offsetofend(union bpf_attr, map_fd);
717 int map_freeze_attr;
718 union bpf_attr attr;
719
720 memset(&attr, 0, attr_size);
721 pr_debug("gen: map_freeze: idx %d\n", map_idx);
722 map_freeze_attr = add_data(gen, &attr, attr_size);
723 move_stack2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
724 stack_off(map_fd[map_idx]));
725 /* emit MAP_FREEZE command */
726 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
727 debug_ret(gen, "map_freeze");
728 emit_check_err(gen);
729}