Loading...
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/* Copyright (c) 2021 Facebook */
3#include <stdio.h>
4#include <stdlib.h>
5#include <string.h>
6#include <errno.h>
7#include <linux/filter.h>
8#include <sys/param.h>
9#include "btf.h"
10#include "bpf.h"
11#include "libbpf.h"
12#include "libbpf_internal.h"
13#include "hashmap.h"
14#include "bpf_gen_internal.h"
15#include "skel_internal.h"
16#include <asm/byteorder.h>
17
18#define MAX_USED_MAPS 64
19#define MAX_USED_PROGS 32
20#define MAX_KFUNC_DESCS 256
21#define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS)
22
23/* The following structure describes the stack layout of the loader program.
24 * In addition R6 contains the pointer to context.
25 * R7 contains the result of the last sys_bpf command (typically error or FD).
26 * R9 contains the result of the last sys_close command.
27 *
28 * Naming convention:
29 * ctx - bpf program context
30 * stack - bpf program stack
31 * blob - bpf_attr-s, strings, insns, map data.
32 * All the bytes that loader prog will use for read/write.
33 */
34struct loader_stack {
35 __u32 btf_fd;
36 __u32 inner_map_fd;
37 __u32 prog_fd[MAX_USED_PROGS];
38};
39
40#define stack_off(field) \
41 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
42
43#define attr_field(attr, field) (attr + offsetof(union bpf_attr, field))
44
45static int blob_fd_array_off(struct bpf_gen *gen, int index)
46{
47 return gen->fd_array + index * sizeof(int);
48}
49
50static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
51{
52 size_t off = gen->insn_cur - gen->insn_start;
53 void *insn_start;
54
55 if (gen->error)
56 return gen->error;
57 if (size > INT32_MAX || off + size > INT32_MAX) {
58 gen->error = -ERANGE;
59 return -ERANGE;
60 }
61 insn_start = realloc(gen->insn_start, off + size);
62 if (!insn_start) {
63 gen->error = -ENOMEM;
64 free(gen->insn_start);
65 gen->insn_start = NULL;
66 return -ENOMEM;
67 }
68 gen->insn_start = insn_start;
69 gen->insn_cur = insn_start + off;
70 return 0;
71}
72
73static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
74{
75 size_t off = gen->data_cur - gen->data_start;
76 void *data_start;
77
78 if (gen->error)
79 return gen->error;
80 if (size > INT32_MAX || off + size > INT32_MAX) {
81 gen->error = -ERANGE;
82 return -ERANGE;
83 }
84 data_start = realloc(gen->data_start, off + size);
85 if (!data_start) {
86 gen->error = -ENOMEM;
87 free(gen->data_start);
88 gen->data_start = NULL;
89 return -ENOMEM;
90 }
91 gen->data_start = data_start;
92 gen->data_cur = data_start + off;
93 return 0;
94}
95
96static void emit(struct bpf_gen *gen, struct bpf_insn insn)
97{
98 if (realloc_insn_buf(gen, sizeof(insn)))
99 return;
100 memcpy(gen->insn_cur, &insn, sizeof(insn));
101 gen->insn_cur += sizeof(insn);
102}
103
104static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
105{
106 emit(gen, insn1);
107 emit(gen, insn2);
108}
109
110static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
111static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
112
113void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
114{
115 size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz;
116 int i;
117
118 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
119 gen->log_level = log_level;
120 /* save ctx pointer into R6 */
121 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
122
123 /* bzero stack */
124 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
125 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
126 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
127 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
128 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
129
130 /* amount of stack actually used, only used to calculate iterations, not stack offset */
131 nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]);
132 /* jump over cleanup code */
133 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
134 /* size of cleanup code below (including map fd cleanup) */
135 (nr_progs_sz / 4) * 3 + 2 +
136 /* 6 insns for emit_sys_close_blob,
137 * 6 insns for debug_regs in emit_sys_close_blob
138 */
139 nr_maps * (6 + (gen->log_level ? 6 : 0))));
140
141 /* remember the label where all error branches will jump to */
142 gen->cleanup_label = gen->insn_cur - gen->insn_start;
143 /* emit cleanup code: close all temp FDs */
144 for (i = 0; i < nr_progs_sz; i += 4) {
145 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
146 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
148 }
149 for (i = 0; i < nr_maps; i++)
150 emit_sys_close_blob(gen, blob_fd_array_off(gen, i));
151 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */
152 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
153 emit(gen, BPF_EXIT_INSN());
154}
155
156static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
157{
158 __u32 size8 = roundup(size, 8);
159 __u64 zero = 0;
160 void *prev;
161
162 if (realloc_data_buf(gen, size8))
163 return 0;
164 prev = gen->data_cur;
165 if (data) {
166 memcpy(gen->data_cur, data, size);
167 memcpy(gen->data_cur + size, &zero, size8 - size);
168 } else {
169 memset(gen->data_cur, 0, size8);
170 }
171 gen->data_cur += size8;
172 return prev - gen->data_start;
173}
174
175/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
176 * to start of fd_array. Caller can decide if it is usable or not.
177 */
178static int add_map_fd(struct bpf_gen *gen)
179{
180 if (gen->nr_maps == MAX_USED_MAPS) {
181 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
182 gen->error = -E2BIG;
183 return 0;
184 }
185 return gen->nr_maps++;
186}
187
188static int add_kfunc_btf_fd(struct bpf_gen *gen)
189{
190 int cur;
191
192 if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
193 cur = add_data(gen, NULL, sizeof(int));
194 return (cur - gen->fd_array) / sizeof(int);
195 }
196 return MAX_USED_MAPS + gen->nr_fd_array++;
197}
198
199static int insn_bytes_to_bpf_size(__u32 sz)
200{
201 switch (sz) {
202 case 8: return BPF_DW;
203 case 4: return BPF_W;
204 case 2: return BPF_H;
205 case 1: return BPF_B;
206 default: return -1;
207 }
208}
209
210/* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */
211static void emit_rel_store(struct bpf_gen *gen, int off, int data)
212{
213 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
214 0, 0, 0, data));
215 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
216 0, 0, 0, off));
217 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
218}
219
220static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
221{
222 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
223 0, 0, 0, blob_off));
224 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
225 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
226 0, 0, 0, off));
227 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
228}
229
230static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
231{
232 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
233 0, 0, 0, blob_off));
234 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
236}
237
238static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
239 bool check_non_zero)
240{
241 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
242 if (check_non_zero)
243 /* If value in ctx is zero don't update the blob.
244 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c
245 */
246 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
247 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
248 0, 0, 0, off));
249 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
250}
251
252static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
253{
254 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
256 0, 0, 0, off));
257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
258}
259
260static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
261{
262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
263 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
264}
265
266static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
267{
268 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
269 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
270 0, 0, 0, attr));
271 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
272 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
273 /* remember the result in R7 */
274 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
275}
276
277static bool is_simm16(__s64 value)
278{
279 return value == (__s64)(__s16)value;
280}
281
282static void emit_check_err(struct bpf_gen *gen)
283{
284 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
285
286 /* R7 contains result of last sys_bpf command.
287 * if (R7 < 0) goto cleanup;
288 */
289 if (is_simm16(off)) {
290 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
291 } else {
292 gen->error = -ERANGE;
293 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));
294 }
295}
296
297/* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
298static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
299 const char *fmt, va_list args)
300{
301 char buf[1024];
302 int addr, len, ret;
303
304 if (!gen->log_level)
305 return;
306 ret = vsnprintf(buf, sizeof(buf), fmt, args);
307 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0)
308 /* The special case to accommodate common debug_ret():
309 * to avoid specifying BPF_REG_7 and adding " r=%%d" to
310 * prints explicitly.
311 */
312 strcat(buf, " r=%d");
313 len = strlen(buf) + 1;
314 addr = add_data(gen, buf, len);
315
316 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
317 0, 0, 0, addr));
318 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
319 if (reg1 >= 0)
320 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
321 if (reg2 >= 0)
322 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
323 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
324}
325
326static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
327{
328 va_list args;
329
330 va_start(args, fmt);
331 emit_debug(gen, reg1, reg2, fmt, args);
332 va_end(args);
333}
334
335static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
336{
337 va_list args;
338
339 va_start(args, fmt);
340 emit_debug(gen, BPF_REG_7, -1, fmt, args);
341 va_end(args);
342}
343
344static void __emit_sys_close(struct bpf_gen *gen)
345{
346 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
347 /* 2 is the number of the following insns
348 * * 6 is additional insns in debug_regs
349 */
350 2 + (gen->log_level ? 6 : 0)));
351 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
352 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
353 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
354}
355
356static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
357{
358 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
359 __emit_sys_close(gen);
360}
361
362static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
363{
364 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
365 0, 0, 0, blob_off));
366 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
367 __emit_sys_close(gen);
368}
369
370int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
371{
372 int i;
373
374 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
375 pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n",
376 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
377 gen->error = -EFAULT;
378 return gen->error;
379 }
380 emit_sys_close_stack(gen, stack_off(btf_fd));
381 for (i = 0; i < gen->nr_progs; i++)
382 move_stack2ctx(gen,
383 sizeof(struct bpf_loader_ctx) +
384 sizeof(struct bpf_map_desc) * gen->nr_maps +
385 sizeof(struct bpf_prog_desc) * i +
386 offsetof(struct bpf_prog_desc, prog_fd), 4,
387 stack_off(prog_fd[i]));
388 for (i = 0; i < gen->nr_maps; i++)
389 move_blob2ctx(gen,
390 sizeof(struct bpf_loader_ctx) +
391 sizeof(struct bpf_map_desc) * i +
392 offsetof(struct bpf_map_desc, map_fd), 4,
393 blob_fd_array_off(gen, i));
394 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
395 emit(gen, BPF_EXIT_INSN());
396 pr_debug("gen: finish %d\n", gen->error);
397 if (!gen->error) {
398 struct gen_loader_opts *opts = gen->opts;
399
400 opts->insns = gen->insn_start;
401 opts->insns_sz = gen->insn_cur - gen->insn_start;
402 opts->data = gen->data_start;
403 opts->data_sz = gen->data_cur - gen->data_start;
404 }
405 return gen->error;
406}
407
408void bpf_gen__free(struct bpf_gen *gen)
409{
410 if (!gen)
411 return;
412 free(gen->data_start);
413 free(gen->insn_start);
414 free(gen);
415}
416
417void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
418 __u32 btf_raw_size)
419{
420 int attr_size = offsetofend(union bpf_attr, btf_log_level);
421 int btf_data, btf_load_attr;
422 union bpf_attr attr;
423
424 memset(&attr, 0, attr_size);
425 pr_debug("gen: load_btf: size %d\n", btf_raw_size);
426 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
427
428 attr.btf_size = btf_raw_size;
429 btf_load_attr = add_data(gen, &attr, attr_size);
430
431 /* populate union bpf_attr with user provided log details */
432 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
433 offsetof(struct bpf_loader_ctx, log_level), false);
434 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
435 offsetof(struct bpf_loader_ctx, log_size), false);
436 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
437 offsetof(struct bpf_loader_ctx, log_buf), false);
438 /* populate union bpf_attr with a pointer to the BTF data */
439 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
440 /* emit BTF_LOAD command */
441 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
442 debug_ret(gen, "btf_load size %d", btf_raw_size);
443 emit_check_err(gen);
444 /* remember btf_fd in the stack, if successful */
445 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
446}
447
448void bpf_gen__map_create(struct bpf_gen *gen,
449 enum bpf_map_type map_type,
450 const char *map_name,
451 __u32 key_size, __u32 value_size, __u32 max_entries,
452 struct bpf_map_create_opts *map_attr, int map_idx)
453{
454 int attr_size = offsetofend(union bpf_attr, map_extra);
455 bool close_inner_map_fd = false;
456 int map_create_attr, idx;
457 union bpf_attr attr;
458
459 memset(&attr, 0, attr_size);
460 attr.map_type = map_type;
461 attr.key_size = key_size;
462 attr.value_size = value_size;
463 attr.map_flags = map_attr->map_flags;
464 attr.map_extra = map_attr->map_extra;
465 if (map_name)
466 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
467 attr.numa_node = map_attr->numa_node;
468 attr.map_ifindex = map_attr->map_ifindex;
469 attr.max_entries = max_entries;
470 attr.btf_key_type_id = map_attr->btf_key_type_id;
471 attr.btf_value_type_id = map_attr->btf_value_type_id;
472
473 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n",
474 attr.map_name, map_idx, map_type, attr.btf_value_type_id);
475
476 map_create_attr = add_data(gen, &attr, attr_size);
477 if (attr.btf_value_type_id)
478 /* populate union bpf_attr with btf_fd saved in the stack earlier */
479 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
480 stack_off(btf_fd));
481 switch (attr.map_type) {
482 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
483 case BPF_MAP_TYPE_HASH_OF_MAPS:
484 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
485 stack_off(inner_map_fd));
486 close_inner_map_fd = true;
487 break;
488 default:
489 break;
490 }
491 /* conditionally update max_entries */
492 if (map_idx >= 0)
493 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
494 sizeof(struct bpf_loader_ctx) +
495 sizeof(struct bpf_map_desc) * map_idx +
496 offsetof(struct bpf_map_desc, max_entries),
497 true /* check that max_entries != 0 */);
498 /* emit MAP_CREATE command */
499 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
500 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
501 attr.map_name, map_idx, map_type, value_size,
502 attr.btf_value_type_id);
503 emit_check_err(gen);
504 /* remember map_fd in the stack, if successful */
505 if (map_idx < 0) {
506 /* This bpf_gen__map_create() function is called with map_idx >= 0
507 * for all maps that libbpf loading logic tracks.
508 * It's called with -1 to create an inner map.
509 */
510 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
511 stack_off(inner_map_fd)));
512 } else if (map_idx != gen->nr_maps) {
513 gen->error = -EDOM; /* internal bug */
514 return;
515 } else {
516 /* add_map_fd does gen->nr_maps++ */
517 idx = add_map_fd(gen);
518 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
519 0, 0, 0, blob_fd_array_off(gen, idx)));
520 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
521 }
522 if (close_inner_map_fd)
523 emit_sys_close_stack(gen, stack_off(inner_map_fd));
524}
525
526void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
527 enum bpf_attach_type type)
528{
529 const char *prefix;
530 int kind, ret;
531
532 btf_get_kernel_prefix_kind(type, &prefix, &kind);
533 gen->attach_kind = kind;
534 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
535 prefix, attach_name);
536 if (ret >= sizeof(gen->attach_target))
537 gen->error = -ENOSPC;
538}
539
540static void emit_find_attach_target(struct bpf_gen *gen)
541{
542 int name, len = strlen(gen->attach_target) + 1;
543
544 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
545 name = add_data(gen, gen->attach_target, len);
546
547 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
548 0, 0, 0, name));
549 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
550 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
551 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
552 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
553 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
554 debug_ret(gen, "find_by_name_kind(%s,%d)",
555 gen->attach_target, gen->attach_kind);
556 emit_check_err(gen);
557 /* if successful, btf_id is in lower 32-bit of R7 and
558 * btf_obj_fd is in upper 32-bit
559 */
560}
561
562void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
563 bool is_typeless, bool is_ld64, int kind, int insn_idx)
564{
565 struct ksym_relo_desc *relo;
566
567 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
568 if (!relo) {
569 gen->error = -ENOMEM;
570 return;
571 }
572 gen->relos = relo;
573 relo += gen->relo_cnt;
574 relo->name = name;
575 relo->is_weak = is_weak;
576 relo->is_typeless = is_typeless;
577 relo->is_ld64 = is_ld64;
578 relo->kind = kind;
579 relo->insn_idx = insn_idx;
580 gen->relo_cnt++;
581}
582
583/* returns existing ksym_desc with ref incremented, or inserts a new one */
584static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
585{
586 struct ksym_desc *kdesc;
587 int i;
588
589 for (i = 0; i < gen->nr_ksyms; i++) {
590 kdesc = &gen->ksyms[i];
591 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 &&
592 !strcmp(kdesc->name, relo->name)) {
593 kdesc->ref++;
594 return kdesc;
595 }
596 }
597 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
598 if (!kdesc) {
599 gen->error = -ENOMEM;
600 return NULL;
601 }
602 gen->ksyms = kdesc;
603 kdesc = &gen->ksyms[gen->nr_ksyms++];
604 kdesc->name = relo->name;
605 kdesc->kind = relo->kind;
606 kdesc->ref = 1;
607 kdesc->off = 0;
608 kdesc->insn = 0;
609 kdesc->is_ld64 = relo->is_ld64;
610 return kdesc;
611}
612
613/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
614 * Returns result in BPF_REG_7
615 */
616static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
617{
618 int name_off, len = strlen(relo->name) + 1;
619
620 name_off = add_data(gen, relo->name, len);
621 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
622 0, 0, 0, name_off));
623 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
624 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
625 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
626 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
627 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
628 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
629}
630
631/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
632 * Returns result in BPF_REG_7
633 * Returns u64 symbol addr in BPF_REG_9
634 */
635static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)
636{
637 int name_off, len = strlen(relo->name) + 1, res_off;
638
639 name_off = add_data(gen, relo->name, len);
640 res_off = add_data(gen, NULL, 8); /* res is u64 */
641 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
642 0, 0, 0, name_off));
643 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
644 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
645 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,
646 0, 0, 0, res_off));
647 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));
648 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));
649 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));
650 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
651 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);
652}
653
654/* Expects:
655 * BPF_REG_8 - pointer to instruction
656 *
657 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new
658 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
659 * this would mean a new BTF fd index for each entry. By pairing symbol name
660 * with index, we get the insn->imm, insn->off pairing that kernel uses for
661 * kfunc_tab, which becomes the effective limit even though all of them may
662 * share same index in fd_array (such that kfunc_btf_tab has 1 element).
663 */
664static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
665{
666 struct ksym_desc *kdesc;
667 int btf_fd_idx;
668
669 kdesc = get_ksym_desc(gen, relo);
670 if (!kdesc)
671 return;
672 /* try to copy from existing bpf_insn */
673 if (kdesc->ref > 1) {
674 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
675 kdesc->insn + offsetof(struct bpf_insn, imm));
676 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
677 kdesc->insn + offsetof(struct bpf_insn, off));
678 goto log;
679 }
680 /* remember insn offset, so we can copy BTF ID and FD later */
681 kdesc->insn = insn;
682 emit_bpf_find_by_name_kind(gen, relo);
683 if (!relo->is_weak)
684 emit_check_err(gen);
685 /* get index in fd_array to store BTF FD at */
686 btf_fd_idx = add_kfunc_btf_fd(gen);
687 if (btf_fd_idx > INT16_MAX) {
688 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
689 btf_fd_idx, relo->name);
690 gen->error = -E2BIG;
691 return;
692 }
693 kdesc->off = btf_fd_idx;
694 /* jump to success case */
695 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
696 /* set value for imm, off as 0 */
697 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
698 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
699 /* skip success case for ret < 0 */
700 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));
701 /* store btf_id into insn[insn_idx].imm */
702 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
703 /* obtain fd in BPF_REG_9 */
704 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
705 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
706 /* load fd_array slot pointer */
707 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
708 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
709 /* store BTF fd in slot, 0 for vmlinux */
710 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
711 /* jump to insn[insn_idx].off store if fd denotes module BTF */
712 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));
713 /* set the default value for off */
714 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
715 /* skip BTF fd store for vmlinux BTF */
716 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
717 /* store index into insn[insn_idx].off */
718 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
719log:
720 if (!gen->log_level)
721 return;
722 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
723 offsetof(struct bpf_insn, imm)));
724 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
725 offsetof(struct bpf_insn, off)));
726 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
727 relo->name, kdesc->ref);
728 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
729 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
730 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
731 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
732 relo->name, kdesc->ref);
733}
734
735static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,
736 int ref)
737{
738 if (!gen->log_level)
739 return;
740 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
741 offsetof(struct bpf_insn, imm)));
742 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
743 offsetof(struct bpf_insn, imm)));
744 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",
745 relo->is_typeless, relo->is_weak, relo->name, ref);
746 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
747 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",
748 relo->is_typeless, relo->is_weak, relo->name, ref);
749}
750
751/* Expects:
752 * BPF_REG_8 - pointer to instruction
753 */
754static void emit_relo_ksym_typeless(struct bpf_gen *gen,
755 struct ksym_relo_desc *relo, int insn)
756{
757 struct ksym_desc *kdesc;
758
759 kdesc = get_ksym_desc(gen, relo);
760 if (!kdesc)
761 return;
762 /* try to copy from existing ldimm64 insn */
763 if (kdesc->ref > 1) {
764 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
765 kdesc->insn + offsetof(struct bpf_insn, imm));
766 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
767 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
768 goto log;
769 }
770 /* remember insn offset, so we can copy ksym addr later */
771 kdesc->insn = insn;
772 /* skip typeless ksym_desc in fd closing loop in cleanup_relos */
773 kdesc->typeless = true;
774 emit_bpf_kallsyms_lookup_name(gen, relo);
775 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));
776 emit_check_err(gen);
777 /* store lower half of addr into insn[insn_idx].imm */
778 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));
779 /* store upper half of addr into insn[insn_idx + 1].imm */
780 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
781 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,
782 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
783log:
784 emit_ksym_relo_log(gen, relo, kdesc->ref);
785}
786
787static __u32 src_reg_mask(void)
788{
789#if defined(__LITTLE_ENDIAN_BITFIELD)
790 return 0x0f; /* src_reg,dst_reg,... */
791#elif defined(__BIG_ENDIAN_BITFIELD)
792 return 0xf0; /* dst_reg,src_reg,... */
793#else
794#error "Unsupported bit endianness, cannot proceed"
795#endif
796}
797
798/* Expects:
799 * BPF_REG_8 - pointer to instruction
800 */
801static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
802{
803 struct ksym_desc *kdesc;
804 __u32 reg_mask;
805
806 kdesc = get_ksym_desc(gen, relo);
807 if (!kdesc)
808 return;
809 /* try to copy from existing ldimm64 insn */
810 if (kdesc->ref > 1) {
811 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
812 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
813 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
814 kdesc->insn + offsetof(struct bpf_insn, imm));
815 /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob
816 * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn
817 */
818 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));
819 goto clear_src_reg;
820 }
821 /* remember insn offset, so we can copy BTF ID and FD later */
822 kdesc->insn = insn;
823 emit_bpf_find_by_name_kind(gen, relo);
824 if (!relo->is_weak)
825 emit_check_err(gen);
826 /* jump to success case */
827 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
828 /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */
829 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
830 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0));
831 /* skip success case for ret < 0 */
832 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
833 /* store btf_id into insn[insn_idx].imm */
834 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
835 /* store btf_obj_fd into insn[insn_idx + 1].imm */
836 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
837 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
838 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
839 /* skip src_reg adjustment */
840 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
841clear_src_reg:
842 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */
843 reg_mask = src_reg_mask();
844 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
845 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
846 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
847
848 emit_ksym_relo_log(gen, relo, kdesc->ref);
849}
850
851void bpf_gen__record_relo_core(struct bpf_gen *gen,
852 const struct bpf_core_relo *core_relo)
853{
854 struct bpf_core_relo *relos;
855
856 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
857 if (!relos) {
858 gen->error = -ENOMEM;
859 return;
860 }
861 gen->core_relos = relos;
862 relos += gen->core_relo_cnt;
863 memcpy(relos, core_relo, sizeof(*relos));
864 gen->core_relo_cnt++;
865}
866
867static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
868{
869 int insn;
870
871 pr_debug("gen: emit_relo (%d): %s at %d %s\n",
872 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call");
873 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
874 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
875 if (relo->is_ld64) {
876 if (relo->is_typeless)
877 emit_relo_ksym_typeless(gen, relo, insn);
878 else
879 emit_relo_ksym_btf(gen, relo, insn);
880 } else {
881 emit_relo_kfunc_btf(gen, relo, insn);
882 }
883}
884
885static void emit_relos(struct bpf_gen *gen, int insns)
886{
887 int i;
888
889 for (i = 0; i < gen->relo_cnt; i++)
890 emit_relo(gen, gen->relos + i, insns);
891}
892
893static void cleanup_core_relo(struct bpf_gen *gen)
894{
895 if (!gen->core_relo_cnt)
896 return;
897 free(gen->core_relos);
898 gen->core_relo_cnt = 0;
899 gen->core_relos = NULL;
900}
901
902static void cleanup_relos(struct bpf_gen *gen, int insns)
903{
904 struct ksym_desc *kdesc;
905 int i, insn;
906
907 for (i = 0; i < gen->nr_ksyms; i++) {
908 kdesc = &gen->ksyms[i];
909 /* only close fds for typed ksyms and kfuncs */
910 if (kdesc->is_ld64 && !kdesc->typeless) {
911 /* close fd recorded in insn[insn_idx + 1].imm */
912 insn = kdesc->insn;
913 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
914 emit_sys_close_blob(gen, insn);
915 } else if (!kdesc->is_ld64) {
916 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off));
917 if (kdesc->off < MAX_FD_ARRAY_SZ)
918 gen->nr_fd_array--;
919 }
920 }
921 if (gen->nr_ksyms) {
922 free(gen->ksyms);
923 gen->nr_ksyms = 0;
924 gen->ksyms = NULL;
925 }
926 if (gen->relo_cnt) {
927 free(gen->relos);
928 gen->relo_cnt = 0;
929 gen->relos = NULL;
930 }
931 cleanup_core_relo(gen);
932}
933
934void bpf_gen__prog_load(struct bpf_gen *gen,
935 enum bpf_prog_type prog_type, const char *prog_name,
936 const char *license, struct bpf_insn *insns, size_t insn_cnt,
937 struct bpf_prog_load_opts *load_attr, int prog_idx)
938{
939 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
940 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
941 union bpf_attr attr;
942
943 memset(&attr, 0, attr_size);
944 pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n",
945 prog_type, insn_cnt, prog_idx);
946 /* add license string to blob of bytes */
947 license_off = add_data(gen, license, strlen(license) + 1);
948 /* add insns to blob of bytes */
949 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
950
951 attr.prog_type = prog_type;
952 attr.expected_attach_type = load_attr->expected_attach_type;
953 attr.attach_btf_id = load_attr->attach_btf_id;
954 attr.prog_ifindex = load_attr->prog_ifindex;
955 attr.kern_version = 0;
956 attr.insn_cnt = (__u32)insn_cnt;
957 attr.prog_flags = load_attr->prog_flags;
958
959 attr.func_info_rec_size = load_attr->func_info_rec_size;
960 attr.func_info_cnt = load_attr->func_info_cnt;
961 func_info = add_data(gen, load_attr->func_info,
962 attr.func_info_cnt * attr.func_info_rec_size);
963
964 attr.line_info_rec_size = load_attr->line_info_rec_size;
965 attr.line_info_cnt = load_attr->line_info_cnt;
966 line_info = add_data(gen, load_attr->line_info,
967 attr.line_info_cnt * attr.line_info_rec_size);
968
969 attr.core_relo_rec_size = sizeof(struct bpf_core_relo);
970 attr.core_relo_cnt = gen->core_relo_cnt;
971 core_relos = add_data(gen, gen->core_relos,
972 attr.core_relo_cnt * attr.core_relo_rec_size);
973
974 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
975 prog_load_attr = add_data(gen, &attr, attr_size);
976
977 /* populate union bpf_attr with a pointer to license */
978 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
979
980 /* populate union bpf_attr with a pointer to instructions */
981 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);
982
983 /* populate union bpf_attr with a pointer to func_info */
984 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
985
986 /* populate union bpf_attr with a pointer to line_info */
987 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
988
989 /* populate union bpf_attr with a pointer to core_relos */
990 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
991
992 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
993 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
994
995 /* populate union bpf_attr with user provided log details */
996 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
997 offsetof(struct bpf_loader_ctx, log_level), false);
998 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
999 offsetof(struct bpf_loader_ctx, log_size), false);
1000 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
1001 offsetof(struct bpf_loader_ctx, log_buf), false);
1002 /* populate union bpf_attr with btf_fd saved in the stack earlier */
1003 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
1004 stack_off(btf_fd));
1005 if (gen->attach_kind) {
1006 emit_find_attach_target(gen);
1007 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */
1008 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
1009 0, 0, 0, prog_load_attr));
1010 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1011 offsetof(union bpf_attr, attach_btf_id)));
1012 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
1013 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1014 offsetof(union bpf_attr, attach_btf_obj_fd)));
1015 }
1016 emit_relos(gen, insns_off);
1017 /* emit PROG_LOAD command */
1018 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
1019 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
1020 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
1021 cleanup_relos(gen, insns_off);
1022 if (gen->attach_kind) {
1023 emit_sys_close_blob(gen,
1024 attr_field(prog_load_attr, attach_btf_obj_fd));
1025 gen->attach_kind = 0;
1026 }
1027 emit_check_err(gen);
1028 /* remember prog_fd in the stack, if successful */
1029 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
1030 stack_off(prog_fd[gen->nr_progs])));
1031 gen->nr_progs++;
1032}
1033
1034void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
1035 __u32 value_size)
1036{
1037 int attr_size = offsetofend(union bpf_attr, flags);
1038 int map_update_attr, value, key;
1039 union bpf_attr attr;
1040 int zero = 0;
1041
1042 memset(&attr, 0, attr_size);
1043 pr_debug("gen: map_update_elem: idx %d\n", map_idx);
1044
1045 value = add_data(gen, pvalue, value_size);
1046 key = add_data(gen, &zero, sizeof(zero));
1047
1048 /* if (map_desc[map_idx].initial_value) {
1049 * if (ctx->flags & BPF_SKEL_KERNEL)
1050 * bpf_probe_read_kernel(value, value_size, initial_value);
1051 * else
1052 * bpf_copy_from_user(value, value_size, initial_value);
1053 * }
1054 */
1055 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
1056 sizeof(struct bpf_loader_ctx) +
1057 sizeof(struct bpf_map_desc) * map_idx +
1058 offsetof(struct bpf_map_desc, initial_value)));
1059 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
1060 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
1061 0, 0, 0, value));
1062 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
1063 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
1064 offsetof(struct bpf_loader_ctx, flags)));
1065 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
1066 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
1067 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
1068 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
1069
1070 map_update_attr = add_data(gen, &attr, attr_size);
1071 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1072 blob_fd_array_off(gen, map_idx));
1073 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1074 emit_rel_store(gen, attr_field(map_update_attr, value), value);
1075 /* emit MAP_UPDATE_ELEM command */
1076 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1077 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
1078 emit_check_err(gen);
1079}
1080
1081void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
1082 int inner_map_idx)
1083{
1084 int attr_size = offsetofend(union bpf_attr, flags);
1085 int map_update_attr, key;
1086 union bpf_attr attr;
1087
1088 memset(&attr, 0, attr_size);
1089 pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n",
1090 outer_map_idx, slot, inner_map_idx);
1091
1092 key = add_data(gen, &slot, sizeof(slot));
1093
1094 map_update_attr = add_data(gen, &attr, attr_size);
1095 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1096 blob_fd_array_off(gen, outer_map_idx));
1097 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1098 emit_rel_store(gen, attr_field(map_update_attr, value),
1099 blob_fd_array_off(gen, inner_map_idx));
1100
1101 /* emit MAP_UPDATE_ELEM command */
1102 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1103 debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
1104 outer_map_idx, slot, inner_map_idx);
1105 emit_check_err(gen);
1106}
1107
1108void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
1109{
1110 int attr_size = offsetofend(union bpf_attr, map_fd);
1111 int map_freeze_attr;
1112 union bpf_attr attr;
1113
1114 memset(&attr, 0, attr_size);
1115 pr_debug("gen: map_freeze: idx %d\n", map_idx);
1116 map_freeze_attr = add_data(gen, &attr, attr_size);
1117 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
1118 blob_fd_array_off(gen, map_idx));
1119 /* emit MAP_FREEZE command */
1120 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
1121 debug_ret(gen, "map_freeze");
1122 emit_check_err(gen);
1123}
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/* Copyright (c) 2021 Facebook */
3#include <stdio.h>
4#include <stdlib.h>
5#include <string.h>
6#include <errno.h>
7#include <linux/filter.h>
8#include <sys/param.h>
9#include "btf.h"
10#include "bpf.h"
11#include "libbpf.h"
12#include "libbpf_internal.h"
13#include "hashmap.h"
14#include "bpf_gen_internal.h"
15#include "skel_internal.h"
16#include <asm/byteorder.h>
17#include "str_error.h"
18
19#define MAX_USED_MAPS 64
20#define MAX_USED_PROGS 32
21#define MAX_KFUNC_DESCS 256
22#define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS)
23
24/* The following structure describes the stack layout of the loader program.
25 * In addition R6 contains the pointer to context.
26 * R7 contains the result of the last sys_bpf command (typically error or FD).
27 * R9 contains the result of the last sys_close command.
28 *
29 * Naming convention:
30 * ctx - bpf program context
31 * stack - bpf program stack
32 * blob - bpf_attr-s, strings, insns, map data.
33 * All the bytes that loader prog will use for read/write.
34 */
35struct loader_stack {
36 __u32 btf_fd;
37 __u32 inner_map_fd;
38 __u32 prog_fd[MAX_USED_PROGS];
39};
40
41#define stack_off(field) \
42 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
43
44#define attr_field(attr, field) (attr + offsetof(union bpf_attr, field))
45
46static int blob_fd_array_off(struct bpf_gen *gen, int index)
47{
48 return gen->fd_array + index * sizeof(int);
49}
50
51static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
52{
53 size_t off = gen->insn_cur - gen->insn_start;
54 void *insn_start;
55
56 if (gen->error)
57 return gen->error;
58 if (size > INT32_MAX || off + size > INT32_MAX) {
59 gen->error = -ERANGE;
60 return -ERANGE;
61 }
62 insn_start = realloc(gen->insn_start, off + size);
63 if (!insn_start) {
64 gen->error = -ENOMEM;
65 free(gen->insn_start);
66 gen->insn_start = NULL;
67 return -ENOMEM;
68 }
69 gen->insn_start = insn_start;
70 gen->insn_cur = insn_start + off;
71 return 0;
72}
73
74static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
75{
76 size_t off = gen->data_cur - gen->data_start;
77 void *data_start;
78
79 if (gen->error)
80 return gen->error;
81 if (size > INT32_MAX || off + size > INT32_MAX) {
82 gen->error = -ERANGE;
83 return -ERANGE;
84 }
85 data_start = realloc(gen->data_start, off + size);
86 if (!data_start) {
87 gen->error = -ENOMEM;
88 free(gen->data_start);
89 gen->data_start = NULL;
90 return -ENOMEM;
91 }
92 gen->data_start = data_start;
93 gen->data_cur = data_start + off;
94 return 0;
95}
96
97static void emit(struct bpf_gen *gen, struct bpf_insn insn)
98{
99 if (realloc_insn_buf(gen, sizeof(insn)))
100 return;
101 memcpy(gen->insn_cur, &insn, sizeof(insn));
102 gen->insn_cur += sizeof(insn);
103}
104
105static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
106{
107 emit(gen, insn1);
108 emit(gen, insn2);
109}
110
111static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
112static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
113
114void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
115{
116 size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz;
117 int i;
118
119 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
120 gen->log_level = log_level;
121 /* save ctx pointer into R6 */
122 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
123
124 /* bzero stack */
125 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
126 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
127 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
128 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
129 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
130
131 /* amount of stack actually used, only used to calculate iterations, not stack offset */
132 nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]);
133 /* jump over cleanup code */
134 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
135 /* size of cleanup code below (including map fd cleanup) */
136 (nr_progs_sz / 4) * 3 + 2 +
137 /* 6 insns for emit_sys_close_blob,
138 * 6 insns for debug_regs in emit_sys_close_blob
139 */
140 nr_maps * (6 + (gen->log_level ? 6 : 0))));
141
142 /* remember the label where all error branches will jump to */
143 gen->cleanup_label = gen->insn_cur - gen->insn_start;
144 /* emit cleanup code: close all temp FDs */
145 for (i = 0; i < nr_progs_sz; i += 4) {
146 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
147 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
148 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
149 }
150 for (i = 0; i < nr_maps; i++)
151 emit_sys_close_blob(gen, blob_fd_array_off(gen, i));
152 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */
153 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
154 emit(gen, BPF_EXIT_INSN());
155}
156
157static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
158{
159 __u32 size8 = roundup(size, 8);
160 __u64 zero = 0;
161 void *prev;
162
163 if (realloc_data_buf(gen, size8))
164 return 0;
165 prev = gen->data_cur;
166 if (data) {
167 memcpy(gen->data_cur, data, size);
168 memcpy(gen->data_cur + size, &zero, size8 - size);
169 } else {
170 memset(gen->data_cur, 0, size8);
171 }
172 gen->data_cur += size8;
173 return prev - gen->data_start;
174}
175
176/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
177 * to start of fd_array. Caller can decide if it is usable or not.
178 */
179static int add_map_fd(struct bpf_gen *gen)
180{
181 if (gen->nr_maps == MAX_USED_MAPS) {
182 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
183 gen->error = -E2BIG;
184 return 0;
185 }
186 return gen->nr_maps++;
187}
188
189static int add_kfunc_btf_fd(struct bpf_gen *gen)
190{
191 int cur;
192
193 if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
194 cur = add_data(gen, NULL, sizeof(int));
195 return (cur - gen->fd_array) / sizeof(int);
196 }
197 return MAX_USED_MAPS + gen->nr_fd_array++;
198}
199
200static int insn_bytes_to_bpf_size(__u32 sz)
201{
202 switch (sz) {
203 case 8: return BPF_DW;
204 case 4: return BPF_W;
205 case 2: return BPF_H;
206 case 1: return BPF_B;
207 default: return -1;
208 }
209}
210
211/* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */
212static void emit_rel_store(struct bpf_gen *gen, int off, int data)
213{
214 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
215 0, 0, 0, data));
216 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
217 0, 0, 0, off));
218 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
219}
220
221static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
222{
223 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
224 0, 0, 0, blob_off));
225 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
226 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
227 0, 0, 0, off));
228 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
229}
230
231static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
232{
233 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
234 0, 0, 0, blob_off));
235 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
236 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
237}
238
239static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
240 bool check_non_zero)
241{
242 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
243 if (check_non_zero)
244 /* If value in ctx is zero don't update the blob.
245 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c
246 */
247 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
248 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
249 0, 0, 0, off));
250 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
251}
252
253static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
254{
255 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
256 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
257 0, 0, 0, off));
258 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
259}
260
261static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
262{
263 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
264 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
265}
266
267static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
268{
269 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
270 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
271 0, 0, 0, attr));
272 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
273 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
274 /* remember the result in R7 */
275 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
276}
277
278static bool is_simm16(__s64 value)
279{
280 return value == (__s64)(__s16)value;
281}
282
283static void emit_check_err(struct bpf_gen *gen)
284{
285 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
286
287 /* R7 contains result of last sys_bpf command.
288 * if (R7 < 0) goto cleanup;
289 */
290 if (is_simm16(off)) {
291 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
292 } else {
293 gen->error = -ERANGE;
294 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));
295 }
296}
297
298/* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
299static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
300 const char *fmt, va_list args)
301{
302 char buf[1024];
303 int addr, len, ret;
304
305 if (!gen->log_level)
306 return;
307 ret = vsnprintf(buf, sizeof(buf), fmt, args);
308 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0)
309 /* The special case to accommodate common debug_ret():
310 * to avoid specifying BPF_REG_7 and adding " r=%%d" to
311 * prints explicitly.
312 */
313 strcat(buf, " r=%d");
314 len = strlen(buf) + 1;
315 addr = add_data(gen, buf, len);
316
317 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
318 0, 0, 0, addr));
319 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
320 if (reg1 >= 0)
321 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
322 if (reg2 >= 0)
323 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
324 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
325}
326
327static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
328{
329 va_list args;
330
331 va_start(args, fmt);
332 emit_debug(gen, reg1, reg2, fmt, args);
333 va_end(args);
334}
335
336static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
337{
338 va_list args;
339
340 va_start(args, fmt);
341 emit_debug(gen, BPF_REG_7, -1, fmt, args);
342 va_end(args);
343}
344
345static void __emit_sys_close(struct bpf_gen *gen)
346{
347 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
348 /* 2 is the number of the following insns
349 * * 6 is additional insns in debug_regs
350 */
351 2 + (gen->log_level ? 6 : 0)));
352 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
353 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
354 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
355}
356
357static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
358{
359 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
360 __emit_sys_close(gen);
361}
362
363static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
364{
365 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
366 0, 0, 0, blob_off));
367 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
368 __emit_sys_close(gen);
369}
370
371int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
372{
373 int i;
374
375 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
376 pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n",
377 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
378 gen->error = -EFAULT;
379 return gen->error;
380 }
381 emit_sys_close_stack(gen, stack_off(btf_fd));
382 for (i = 0; i < gen->nr_progs; i++)
383 move_stack2ctx(gen,
384 sizeof(struct bpf_loader_ctx) +
385 sizeof(struct bpf_map_desc) * gen->nr_maps +
386 sizeof(struct bpf_prog_desc) * i +
387 offsetof(struct bpf_prog_desc, prog_fd), 4,
388 stack_off(prog_fd[i]));
389 for (i = 0; i < gen->nr_maps; i++)
390 move_blob2ctx(gen,
391 sizeof(struct bpf_loader_ctx) +
392 sizeof(struct bpf_map_desc) * i +
393 offsetof(struct bpf_map_desc, map_fd), 4,
394 blob_fd_array_off(gen, i));
395 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
396 emit(gen, BPF_EXIT_INSN());
397 pr_debug("gen: finish %s\n", errstr(gen->error));
398 if (!gen->error) {
399 struct gen_loader_opts *opts = gen->opts;
400
401 opts->insns = gen->insn_start;
402 opts->insns_sz = gen->insn_cur - gen->insn_start;
403 opts->data = gen->data_start;
404 opts->data_sz = gen->data_cur - gen->data_start;
405
406 /* use target endianness for embedded loader */
407 if (gen->swapped_endian) {
408 struct bpf_insn *insn = (struct bpf_insn *)opts->insns;
409 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
410
411 for (i = 0; i < insn_cnt; i++)
412 bpf_insn_bswap(insn++);
413 }
414 }
415 return gen->error;
416}
417
418void bpf_gen__free(struct bpf_gen *gen)
419{
420 if (!gen)
421 return;
422 free(gen->data_start);
423 free(gen->insn_start);
424 free(gen);
425}
426
427/*
428 * Fields of bpf_attr are set to values in native byte-order before being
429 * written to the target-bound data blob, and may need endian conversion.
430 * This macro allows providing the correct value in situ more simply than
431 * writing a separate converter for *all fields* of *all records* included
432 * in union bpf_attr. Note that sizeof(rval) should match the assignment
433 * target to avoid runtime problems.
434 */
435#define tgt_endian(rval) ({ \
436 typeof(rval) _val = (rval); \
437 if (gen->swapped_endian) { \
438 switch (sizeof(_val)) { \
439 case 1: break; \
440 case 2: _val = bswap_16(_val); break; \
441 case 4: _val = bswap_32(_val); break; \
442 case 8: _val = bswap_64(_val); break; \
443 default: pr_warn("unsupported bswap size!\n"); \
444 } \
445 } \
446 _val; \
447})
448
449void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
450 __u32 btf_raw_size)
451{
452 int attr_size = offsetofend(union bpf_attr, btf_log_level);
453 int btf_data, btf_load_attr;
454 union bpf_attr attr;
455
456 memset(&attr, 0, attr_size);
457 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
458
459 attr.btf_size = tgt_endian(btf_raw_size);
460 btf_load_attr = add_data(gen, &attr, attr_size);
461 pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n",
462 btf_data, btf_raw_size, btf_load_attr, attr_size);
463
464 /* populate union bpf_attr with user provided log details */
465 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
466 offsetof(struct bpf_loader_ctx, log_level), false);
467 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
468 offsetof(struct bpf_loader_ctx, log_size), false);
469 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
470 offsetof(struct bpf_loader_ctx, log_buf), false);
471 /* populate union bpf_attr with a pointer to the BTF data */
472 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
473 /* emit BTF_LOAD command */
474 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
475 debug_ret(gen, "btf_load size %d", btf_raw_size);
476 emit_check_err(gen);
477 /* remember btf_fd in the stack, if successful */
478 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
479}
480
481void bpf_gen__map_create(struct bpf_gen *gen,
482 enum bpf_map_type map_type,
483 const char *map_name,
484 __u32 key_size, __u32 value_size, __u32 max_entries,
485 struct bpf_map_create_opts *map_attr, int map_idx)
486{
487 int attr_size = offsetofend(union bpf_attr, map_extra);
488 bool close_inner_map_fd = false;
489 int map_create_attr, idx;
490 union bpf_attr attr;
491
492 memset(&attr, 0, attr_size);
493 attr.map_type = tgt_endian(map_type);
494 attr.key_size = tgt_endian(key_size);
495 attr.value_size = tgt_endian(value_size);
496 attr.map_flags = tgt_endian(map_attr->map_flags);
497 attr.map_extra = tgt_endian(map_attr->map_extra);
498 if (map_name)
499 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
500 attr.numa_node = tgt_endian(map_attr->numa_node);
501 attr.map_ifindex = tgt_endian(map_attr->map_ifindex);
502 attr.max_entries = tgt_endian(max_entries);
503 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id);
504 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id);
505
506 map_create_attr = add_data(gen, &attr, attr_size);
507 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n",
508 map_name, map_idx, map_type, map_attr->btf_value_type_id,
509 map_create_attr, attr_size);
510
511 if (map_attr->btf_value_type_id)
512 /* populate union bpf_attr with btf_fd saved in the stack earlier */
513 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
514 stack_off(btf_fd));
515 switch (map_type) {
516 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
517 case BPF_MAP_TYPE_HASH_OF_MAPS:
518 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
519 stack_off(inner_map_fd));
520 close_inner_map_fd = true;
521 break;
522 default:
523 break;
524 }
525 /* conditionally update max_entries */
526 if (map_idx >= 0)
527 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
528 sizeof(struct bpf_loader_ctx) +
529 sizeof(struct bpf_map_desc) * map_idx +
530 offsetof(struct bpf_map_desc, max_entries),
531 true /* check that max_entries != 0 */);
532 /* emit MAP_CREATE command */
533 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
534 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
535 map_name, map_idx, map_type, value_size,
536 map_attr->btf_value_type_id);
537 emit_check_err(gen);
538 /* remember map_fd in the stack, if successful */
539 if (map_idx < 0) {
540 /* This bpf_gen__map_create() function is called with map_idx >= 0
541 * for all maps that libbpf loading logic tracks.
542 * It's called with -1 to create an inner map.
543 */
544 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
545 stack_off(inner_map_fd)));
546 } else if (map_idx != gen->nr_maps) {
547 gen->error = -EDOM; /* internal bug */
548 return;
549 } else {
550 /* add_map_fd does gen->nr_maps++ */
551 idx = add_map_fd(gen);
552 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
553 0, 0, 0, blob_fd_array_off(gen, idx)));
554 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
555 }
556 if (close_inner_map_fd)
557 emit_sys_close_stack(gen, stack_off(inner_map_fd));
558}
559
560void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
561 enum bpf_attach_type type)
562{
563 const char *prefix;
564 int kind, ret;
565
566 btf_get_kernel_prefix_kind(type, &prefix, &kind);
567 gen->attach_kind = kind;
568 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
569 prefix, attach_name);
570 if (ret >= sizeof(gen->attach_target))
571 gen->error = -ENOSPC;
572}
573
574static void emit_find_attach_target(struct bpf_gen *gen)
575{
576 int name, len = strlen(gen->attach_target) + 1;
577
578 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
579 name = add_data(gen, gen->attach_target, len);
580
581 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
582 0, 0, 0, name));
583 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
584 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
585 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
586 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
587 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
588 debug_ret(gen, "find_by_name_kind(%s,%d)",
589 gen->attach_target, gen->attach_kind);
590 emit_check_err(gen);
591 /* if successful, btf_id is in lower 32-bit of R7 and
592 * btf_obj_fd is in upper 32-bit
593 */
594}
595
596void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
597 bool is_typeless, bool is_ld64, int kind, int insn_idx)
598{
599 struct ksym_relo_desc *relo;
600
601 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
602 if (!relo) {
603 gen->error = -ENOMEM;
604 return;
605 }
606 gen->relos = relo;
607 relo += gen->relo_cnt;
608 relo->name = name;
609 relo->is_weak = is_weak;
610 relo->is_typeless = is_typeless;
611 relo->is_ld64 = is_ld64;
612 relo->kind = kind;
613 relo->insn_idx = insn_idx;
614 gen->relo_cnt++;
615}
616
617/* returns existing ksym_desc with ref incremented, or inserts a new one */
618static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
619{
620 struct ksym_desc *kdesc;
621 int i;
622
623 for (i = 0; i < gen->nr_ksyms; i++) {
624 kdesc = &gen->ksyms[i];
625 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 &&
626 !strcmp(kdesc->name, relo->name)) {
627 kdesc->ref++;
628 return kdesc;
629 }
630 }
631 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
632 if (!kdesc) {
633 gen->error = -ENOMEM;
634 return NULL;
635 }
636 gen->ksyms = kdesc;
637 kdesc = &gen->ksyms[gen->nr_ksyms++];
638 kdesc->name = relo->name;
639 kdesc->kind = relo->kind;
640 kdesc->ref = 1;
641 kdesc->off = 0;
642 kdesc->insn = 0;
643 kdesc->is_ld64 = relo->is_ld64;
644 return kdesc;
645}
646
647/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
648 * Returns result in BPF_REG_7
649 */
650static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
651{
652 int name_off, len = strlen(relo->name) + 1;
653
654 name_off = add_data(gen, relo->name, len);
655 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
656 0, 0, 0, name_off));
657 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
658 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
659 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
660 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
661 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
662 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
663}
664
665/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
666 * Returns result in BPF_REG_7
667 * Returns u64 symbol addr in BPF_REG_9
668 */
669static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)
670{
671 int name_off, len = strlen(relo->name) + 1, res_off;
672
673 name_off = add_data(gen, relo->name, len);
674 res_off = add_data(gen, NULL, 8); /* res is u64 */
675 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
676 0, 0, 0, name_off));
677 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
678 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
679 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,
680 0, 0, 0, res_off));
681 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));
682 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));
683 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));
684 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
685 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);
686}
687
688/* Expects:
689 * BPF_REG_8 - pointer to instruction
690 *
691 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new
692 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
693 * this would mean a new BTF fd index for each entry. By pairing symbol name
694 * with index, we get the insn->imm, insn->off pairing that kernel uses for
695 * kfunc_tab, which becomes the effective limit even though all of them may
696 * share same index in fd_array (such that kfunc_btf_tab has 1 element).
697 */
698static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
699{
700 struct ksym_desc *kdesc;
701 int btf_fd_idx;
702
703 kdesc = get_ksym_desc(gen, relo);
704 if (!kdesc)
705 return;
706 /* try to copy from existing bpf_insn */
707 if (kdesc->ref > 1) {
708 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
709 kdesc->insn + offsetof(struct bpf_insn, imm));
710 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
711 kdesc->insn + offsetof(struct bpf_insn, off));
712 goto log;
713 }
714 /* remember insn offset, so we can copy BTF ID and FD later */
715 kdesc->insn = insn;
716 emit_bpf_find_by_name_kind(gen, relo);
717 if (!relo->is_weak)
718 emit_check_err(gen);
719 /* get index in fd_array to store BTF FD at */
720 btf_fd_idx = add_kfunc_btf_fd(gen);
721 if (btf_fd_idx > INT16_MAX) {
722 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
723 btf_fd_idx, relo->name);
724 gen->error = -E2BIG;
725 return;
726 }
727 kdesc->off = btf_fd_idx;
728 /* jump to success case */
729 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
730 /* set value for imm, off as 0 */
731 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
732 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
733 /* skip success case for ret < 0 */
734 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));
735 /* store btf_id into insn[insn_idx].imm */
736 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
737 /* obtain fd in BPF_REG_9 */
738 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
739 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
740 /* load fd_array slot pointer */
741 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
742 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
743 /* store BTF fd in slot, 0 for vmlinux */
744 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
745 /* jump to insn[insn_idx].off store if fd denotes module BTF */
746 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));
747 /* set the default value for off */
748 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
749 /* skip BTF fd store for vmlinux BTF */
750 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
751 /* store index into insn[insn_idx].off */
752 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
753log:
754 if (!gen->log_level)
755 return;
756 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
757 offsetof(struct bpf_insn, imm)));
758 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
759 offsetof(struct bpf_insn, off)));
760 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
761 relo->name, kdesc->ref);
762 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
763 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
764 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
765 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
766 relo->name, kdesc->ref);
767}
768
769static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,
770 int ref)
771{
772 if (!gen->log_level)
773 return;
774 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
775 offsetof(struct bpf_insn, imm)));
776 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
777 offsetof(struct bpf_insn, imm)));
778 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",
779 relo->is_typeless, relo->is_weak, relo->name, ref);
780 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
781 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",
782 relo->is_typeless, relo->is_weak, relo->name, ref);
783}
784
785/* Expects:
786 * BPF_REG_8 - pointer to instruction
787 */
788static void emit_relo_ksym_typeless(struct bpf_gen *gen,
789 struct ksym_relo_desc *relo, int insn)
790{
791 struct ksym_desc *kdesc;
792
793 kdesc = get_ksym_desc(gen, relo);
794 if (!kdesc)
795 return;
796 /* try to copy from existing ldimm64 insn */
797 if (kdesc->ref > 1) {
798 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
799 kdesc->insn + offsetof(struct bpf_insn, imm));
800 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
801 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
802 goto log;
803 }
804 /* remember insn offset, so we can copy ksym addr later */
805 kdesc->insn = insn;
806 /* skip typeless ksym_desc in fd closing loop in cleanup_relos */
807 kdesc->typeless = true;
808 emit_bpf_kallsyms_lookup_name(gen, relo);
809 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));
810 emit_check_err(gen);
811 /* store lower half of addr into insn[insn_idx].imm */
812 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));
813 /* store upper half of addr into insn[insn_idx + 1].imm */
814 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
815 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,
816 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
817log:
818 emit_ksym_relo_log(gen, relo, kdesc->ref);
819}
820
821static __u32 src_reg_mask(struct bpf_gen *gen)
822{
823#if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */
824 return gen->swapped_endian ? 0xf0 : 0x0f;
825#elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */
826 return gen->swapped_endian ? 0x0f : 0xf0;
827#else
828#error "Unsupported bit endianness, cannot proceed"
829#endif
830}
831
832/* Expects:
833 * BPF_REG_8 - pointer to instruction
834 */
835static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
836{
837 struct ksym_desc *kdesc;
838 __u32 reg_mask;
839
840 kdesc = get_ksym_desc(gen, relo);
841 if (!kdesc)
842 return;
843 /* try to copy from existing ldimm64 insn */
844 if (kdesc->ref > 1) {
845 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
846 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
847 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
848 kdesc->insn + offsetof(struct bpf_insn, imm));
849 /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob
850 * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn
851 */
852 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));
853 goto clear_src_reg;
854 }
855 /* remember insn offset, so we can copy BTF ID and FD later */
856 kdesc->insn = insn;
857 emit_bpf_find_by_name_kind(gen, relo);
858 if (!relo->is_weak)
859 emit_check_err(gen);
860 /* jump to success case */
861 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
862 /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */
863 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
864 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0));
865 /* skip success case for ret < 0 */
866 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
867 /* store btf_id into insn[insn_idx].imm */
868 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
869 /* store btf_obj_fd into insn[insn_idx + 1].imm */
870 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
871 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
872 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
873 /* skip src_reg adjustment */
874 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
875clear_src_reg:
876 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */
877 reg_mask = src_reg_mask(gen);
878 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
879 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
880 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
881
882 emit_ksym_relo_log(gen, relo, kdesc->ref);
883}
884
885void bpf_gen__record_relo_core(struct bpf_gen *gen,
886 const struct bpf_core_relo *core_relo)
887{
888 struct bpf_core_relo *relos;
889
890 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
891 if (!relos) {
892 gen->error = -ENOMEM;
893 return;
894 }
895 gen->core_relos = relos;
896 relos += gen->core_relo_cnt;
897 memcpy(relos, core_relo, sizeof(*relos));
898 gen->core_relo_cnt++;
899}
900
901static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
902{
903 int insn;
904
905 pr_debug("gen: emit_relo (%d): %s at %d %s\n",
906 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call");
907 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
908 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
909 if (relo->is_ld64) {
910 if (relo->is_typeless)
911 emit_relo_ksym_typeless(gen, relo, insn);
912 else
913 emit_relo_ksym_btf(gen, relo, insn);
914 } else {
915 emit_relo_kfunc_btf(gen, relo, insn);
916 }
917}
918
919static void emit_relos(struct bpf_gen *gen, int insns)
920{
921 int i;
922
923 for (i = 0; i < gen->relo_cnt; i++)
924 emit_relo(gen, gen->relos + i, insns);
925}
926
927static void cleanup_core_relo(struct bpf_gen *gen)
928{
929 if (!gen->core_relo_cnt)
930 return;
931 free(gen->core_relos);
932 gen->core_relo_cnt = 0;
933 gen->core_relos = NULL;
934}
935
936static void cleanup_relos(struct bpf_gen *gen, int insns)
937{
938 struct ksym_desc *kdesc;
939 int i, insn;
940
941 for (i = 0; i < gen->nr_ksyms; i++) {
942 kdesc = &gen->ksyms[i];
943 /* only close fds for typed ksyms and kfuncs */
944 if (kdesc->is_ld64 && !kdesc->typeless) {
945 /* close fd recorded in insn[insn_idx + 1].imm */
946 insn = kdesc->insn;
947 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
948 emit_sys_close_blob(gen, insn);
949 } else if (!kdesc->is_ld64) {
950 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off));
951 if (kdesc->off < MAX_FD_ARRAY_SZ)
952 gen->nr_fd_array--;
953 }
954 }
955 if (gen->nr_ksyms) {
956 free(gen->ksyms);
957 gen->nr_ksyms = 0;
958 gen->ksyms = NULL;
959 }
960 if (gen->relo_cnt) {
961 free(gen->relos);
962 gen->relo_cnt = 0;
963 gen->relos = NULL;
964 }
965 cleanup_core_relo(gen);
966}
967
968/* Convert func, line, and core relo info blobs to target endianness */
969static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info,
970 int core_relos, struct bpf_prog_load_opts *load_attr)
971{
972 struct bpf_func_info *fi = gen->data_start + func_info;
973 struct bpf_line_info *li = gen->data_start + line_info;
974 struct bpf_core_relo *cr = gen->data_start + core_relos;
975 int i;
976
977 for (i = 0; i < load_attr->func_info_cnt; i++)
978 bpf_func_info_bswap(fi++);
979
980 for (i = 0; i < load_attr->line_info_cnt; i++)
981 bpf_line_info_bswap(li++);
982
983 for (i = 0; i < gen->core_relo_cnt; i++)
984 bpf_core_relo_bswap(cr++);
985}
986
987void bpf_gen__prog_load(struct bpf_gen *gen,
988 enum bpf_prog_type prog_type, const char *prog_name,
989 const char *license, struct bpf_insn *insns, size_t insn_cnt,
990 struct bpf_prog_load_opts *load_attr, int prog_idx)
991{
992 int func_info_tot_sz = load_attr->func_info_cnt *
993 load_attr->func_info_rec_size;
994 int line_info_tot_sz = load_attr->line_info_cnt *
995 load_attr->line_info_rec_size;
996 int core_relo_tot_sz = gen->core_relo_cnt *
997 sizeof(struct bpf_core_relo);
998 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
999 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
1000 union bpf_attr attr;
1001
1002 memset(&attr, 0, attr_size);
1003 /* add license string to blob of bytes */
1004 license_off = add_data(gen, license, strlen(license) + 1);
1005 /* add insns to blob of bytes */
1006 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
1007 pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n",
1008 prog_idx, prog_type, insns_off, insn_cnt, license_off);
1009
1010 /* convert blob insns to target endianness */
1011 if (gen->swapped_endian) {
1012 struct bpf_insn *insn = gen->data_start + insns_off;
1013 int i;
1014
1015 for (i = 0; i < insn_cnt; i++, insn++)
1016 bpf_insn_bswap(insn);
1017 }
1018
1019 attr.prog_type = tgt_endian(prog_type);
1020 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type);
1021 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id);
1022 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex);
1023 attr.kern_version = 0;
1024 attr.insn_cnt = tgt_endian((__u32)insn_cnt);
1025 attr.prog_flags = tgt_endian(load_attr->prog_flags);
1026
1027 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size);
1028 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt);
1029 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);
1030 pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n",
1031 func_info, load_attr->func_info_cnt,
1032 load_attr->func_info_rec_size);
1033
1034 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size);
1035 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt);
1036 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);
1037 pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n",
1038 line_info, load_attr->line_info_cnt,
1039 load_attr->line_info_rec_size);
1040
1041 attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo));
1042 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);
1043 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);
1044 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n",
1045 core_relos, gen->core_relo_cnt,
1046 sizeof(struct bpf_core_relo));
1047
1048 /* convert all info blobs to target endianness */
1049 if (gen->swapped_endian)
1050 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr);
1051
1052 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
1053 prog_load_attr = add_data(gen, &attr, attr_size);
1054 pr_debug("gen: prog_load: attr: off %d size %d\n",
1055 prog_load_attr, attr_size);
1056
1057 /* populate union bpf_attr with a pointer to license */
1058 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
1059
1060 /* populate union bpf_attr with a pointer to instructions */
1061 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);
1062
1063 /* populate union bpf_attr with a pointer to func_info */
1064 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
1065
1066 /* populate union bpf_attr with a pointer to line_info */
1067 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
1068
1069 /* populate union bpf_attr with a pointer to core_relos */
1070 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
1071
1072 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
1073 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
1074
1075 /* populate union bpf_attr with user provided log details */
1076 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
1077 offsetof(struct bpf_loader_ctx, log_level), false);
1078 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
1079 offsetof(struct bpf_loader_ctx, log_size), false);
1080 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
1081 offsetof(struct bpf_loader_ctx, log_buf), false);
1082 /* populate union bpf_attr with btf_fd saved in the stack earlier */
1083 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
1084 stack_off(btf_fd));
1085 if (gen->attach_kind) {
1086 emit_find_attach_target(gen);
1087 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */
1088 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
1089 0, 0, 0, prog_load_attr));
1090 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1091 offsetof(union bpf_attr, attach_btf_id)));
1092 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
1093 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1094 offsetof(union bpf_attr, attach_btf_obj_fd)));
1095 }
1096 emit_relos(gen, insns_off);
1097 /* emit PROG_LOAD command */
1098 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
1099 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
1100 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
1101 cleanup_relos(gen, insns_off);
1102 if (gen->attach_kind) {
1103 emit_sys_close_blob(gen,
1104 attr_field(prog_load_attr, attach_btf_obj_fd));
1105 gen->attach_kind = 0;
1106 }
1107 emit_check_err(gen);
1108 /* remember prog_fd in the stack, if successful */
1109 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
1110 stack_off(prog_fd[gen->nr_progs])));
1111 gen->nr_progs++;
1112}
1113
1114void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
1115 __u32 value_size)
1116{
1117 int attr_size = offsetofend(union bpf_attr, flags);
1118 int map_update_attr, value, key;
1119 union bpf_attr attr;
1120 int zero = 0;
1121
1122 memset(&attr, 0, attr_size);
1123
1124 value = add_data(gen, pvalue, value_size);
1125 key = add_data(gen, &zero, sizeof(zero));
1126
1127 /* if (map_desc[map_idx].initial_value) {
1128 * if (ctx->flags & BPF_SKEL_KERNEL)
1129 * bpf_probe_read_kernel(value, value_size, initial_value);
1130 * else
1131 * bpf_copy_from_user(value, value_size, initial_value);
1132 * }
1133 */
1134 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
1135 sizeof(struct bpf_loader_ctx) +
1136 sizeof(struct bpf_map_desc) * map_idx +
1137 offsetof(struct bpf_map_desc, initial_value)));
1138 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
1139 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
1140 0, 0, 0, value));
1141 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
1142 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
1143 offsetof(struct bpf_loader_ctx, flags)));
1144 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
1145 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
1146 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
1147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
1148
1149 map_update_attr = add_data(gen, &attr, attr_size);
1150 pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n",
1151 map_idx, value, value_size, map_update_attr, attr_size);
1152 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1153 blob_fd_array_off(gen, map_idx));
1154 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1155 emit_rel_store(gen, attr_field(map_update_attr, value), value);
1156 /* emit MAP_UPDATE_ELEM command */
1157 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1158 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
1159 emit_check_err(gen);
1160}
1161
1162void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
1163 int inner_map_idx)
1164{
1165 int attr_size = offsetofend(union bpf_attr, flags);
1166 int map_update_attr, key;
1167 union bpf_attr attr;
1168 int tgt_slot;
1169
1170 memset(&attr, 0, attr_size);
1171
1172 tgt_slot = tgt_endian(slot);
1173 key = add_data(gen, &tgt_slot, sizeof(tgt_slot));
1174
1175 map_update_attr = add_data(gen, &attr, attr_size);
1176 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n",
1177 outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size);
1178 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1179 blob_fd_array_off(gen, outer_map_idx));
1180 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1181 emit_rel_store(gen, attr_field(map_update_attr, value),
1182 blob_fd_array_off(gen, inner_map_idx));
1183
1184 /* emit MAP_UPDATE_ELEM command */
1185 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1186 debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
1187 outer_map_idx, slot, inner_map_idx);
1188 emit_check_err(gen);
1189}
1190
1191void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
1192{
1193 int attr_size = offsetofend(union bpf_attr, map_fd);
1194 int map_freeze_attr;
1195 union bpf_attr attr;
1196
1197 memset(&attr, 0, attr_size);
1198 map_freeze_attr = add_data(gen, &attr, attr_size);
1199 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n",
1200 map_idx, map_freeze_attr, attr_size);
1201 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
1202 blob_fd_array_off(gen, map_idx));
1203 /* emit MAP_FREEZE command */
1204 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
1205 debug_ret(gen, "map_freeze");
1206 emit_check_err(gen);
1207}