1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) 2019 Facebook
  3#include <linux/sched.h>
  4#include <linux/ptrace.h>
  5#include <stdint.h>
  6#include <stddef.h>
  7#include <stdbool.h>
  8#include <linux/bpf.h>
  9#include <bpf/bpf_helpers.h>
 10
 11#define FUNCTION_NAME_LEN 64
 12#define FILE_NAME_LEN 128
 13#define TASK_COMM_LEN 16
 14
 15typedef struct {
 16	int PyThreadState_frame;
 17	int PyThreadState_thread;
 18	int PyFrameObject_back;
 19	int PyFrameObject_code;
 20	int PyFrameObject_lineno;
 21	int PyCodeObject_filename;
 22	int PyCodeObject_name;
 23	int String_data;
 24	int String_size;
 25} OffsetConfig;
 26
 27typedef struct {
 28	uintptr_t current_state_addr;
 29	uintptr_t tls_key_addr;
 30	OffsetConfig offsets;
 31	bool use_tls;
 32} PidData;
 33
 34typedef struct {
 35	uint32_t success;
 36} Stats;
 37
 38typedef struct {
 39	char name[FUNCTION_NAME_LEN];
 40	char file[FILE_NAME_LEN];
 41} Symbol;
 42
 43typedef struct {
 44	uint32_t pid;
 45	uint32_t tid;
 46	char comm[TASK_COMM_LEN];
 47	int32_t kernel_stack_id;
 48	int32_t user_stack_id;
 49	bool thread_current;
 50	bool pthread_match;
 51	bool stack_complete;
 52	int16_t stack_len;
 53	int32_t stack[STACK_MAX_LEN];
 54
 55	int has_meta;
 56	int metadata;
 57	char dummy_safeguard;
 58} Event;
 59
 60
 61typedef int pid_t;
 62
 63typedef struct {
 64	void* f_back; // PyFrameObject.f_back, previous frame
 65	void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject
 66	void* co_filename; // PyCodeObject.co_filename
 67	void* co_name; // PyCodeObject.co_name
 68} FrameData;
 69
 70static __always_inline void *get_thread_state(void *tls_base, PidData *pidData)
 71{
 72	void* thread_state;
 73	int key;
 74
 75	bpf_probe_read_user(&key, sizeof(key), (void*)(long)pidData->tls_key_addr);
 76	bpf_probe_read_user(&thread_state, sizeof(thread_state),
 77			    tls_base + 0x310 + key * 0x10 + 0x08);
 78	return thread_state;
 79}
 80
 81static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData,
 82					   FrameData *frame, Symbol *symbol)
 83{
 84	// read data from PyFrameObject
 85	bpf_probe_read_user(&frame->f_back,
 86			    sizeof(frame->f_back),
 87			    frame_ptr + pidData->offsets.PyFrameObject_back);
 88	bpf_probe_read_user(&frame->f_code,
 89			    sizeof(frame->f_code),
 90			    frame_ptr + pidData->offsets.PyFrameObject_code);
 91
 92	// read data from PyCodeObject
 93	if (!frame->f_code)
 94		return false;
 95	bpf_probe_read_user(&frame->co_filename,
 96			    sizeof(frame->co_filename),
 97			    frame->f_code + pidData->offsets.PyCodeObject_filename);
 98	bpf_probe_read_user(&frame->co_name,
 99			    sizeof(frame->co_name),
100			    frame->f_code + pidData->offsets.PyCodeObject_name);
101	// read actual names into symbol
102	if (frame->co_filename)
103		bpf_probe_read_user_str(&symbol->file,
104					sizeof(symbol->file),
105					frame->co_filename +
106					pidData->offsets.String_data);
107	if (frame->co_name)
108		bpf_probe_read_user_str(&symbol->name,
109					sizeof(symbol->name),
110					frame->co_name +
111					pidData->offsets.String_data);
112	return true;
113}
114
115struct {
116	__uint(type, BPF_MAP_TYPE_HASH);
117	__uint(max_entries, 1);
118	__type(key, int);
119	__type(value, PidData);
120} pidmap SEC(".maps");
121
122struct {
123	__uint(type, BPF_MAP_TYPE_HASH);
124	__uint(max_entries, 1);
125	__type(key, int);
126	__type(value, Event);
127} eventmap SEC(".maps");
128
129struct {
130	__uint(type, BPF_MAP_TYPE_HASH);
131	__uint(max_entries, 1);
132	__type(key, Symbol);
133	__type(value, int);
134} symbolmap SEC(".maps");
135
136struct {
137	__uint(type, BPF_MAP_TYPE_ARRAY);
138	__uint(max_entries, 1);
139	__type(key, int);
140	__type(value, Stats);
141} statsmap SEC(".maps");
142
143struct {
144	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
145	__uint(max_entries, 32);
146	__uint(key_size, sizeof(int));
147	__uint(value_size, sizeof(int));
148} perfmap SEC(".maps");
149
150struct {
151	__uint(type, BPF_MAP_TYPE_STACK_TRACE);
152	__uint(max_entries, 1000);
153	__uint(key_size, sizeof(int));
154	__uint(value_size, sizeof(long long) * 127);
155} stackmap SEC(".maps");
156
157#ifdef GLOBAL_FUNC
158__attribute__((noinline))
159#else
160static __always_inline
161#endif
162int __on_event(struct bpf_raw_tracepoint_args *ctx)
163{
164	uint64_t pid_tgid = bpf_get_current_pid_tgid();
165	pid_t pid = (pid_t)(pid_tgid >> 32);
166	PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid);
167	if (!pidData)
168		return 0;
169
170	int zero = 0;
171	Event* event = bpf_map_lookup_elem(&eventmap, &zero);
172	if (!event)
173		return 0;
174
175	event->pid = pid;
176
177	event->tid = (pid_t)pid_tgid;
178	bpf_get_current_comm(&event->comm, sizeof(event->comm));
179
180	event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK);
181	event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0);
182
183	void* thread_state_current = (void*)0;
184	bpf_probe_read_user(&thread_state_current,
185			    sizeof(thread_state_current),
186			    (void*)(long)pidData->current_state_addr);
187
188	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
189	void* tls_base = (void*)task;
190
191	void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData)
192		: thread_state_current;
193	event->thread_current = thread_state == thread_state_current;
194
195	if (pidData->use_tls) {
196		uint64_t pthread_created;
197		uint64_t pthread_self;
198		bpf_probe_read_user(&pthread_self, sizeof(pthread_self),
199				    tls_base + 0x10);
200
201		bpf_probe_read_user(&pthread_created,
202				    sizeof(pthread_created),
203				    thread_state +
204				    pidData->offsets.PyThreadState_thread);
205		event->pthread_match = pthread_created == pthread_self;
206	} else {
207		event->pthread_match = 1;
208	}
209
210	if (event->pthread_match || !pidData->use_tls) {
211		void* frame_ptr;
212		FrameData frame;
213		Symbol sym = {};
214		int cur_cpu = bpf_get_smp_processor_id();
215
216		bpf_probe_read_user(&frame_ptr,
217				    sizeof(frame_ptr),
218				    thread_state +
219				    pidData->offsets.PyThreadState_frame);
220
221		int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
222		if (symbol_counter == NULL)
223			return 0;
224#ifdef NO_UNROLL
225#pragma clang loop unroll(disable)
226#else
227#pragma clang loop unroll(full)
228#endif
229		/* Unwind python stack */
230		for (int i = 0; i < STACK_MAX_LEN; ++i) {
231			if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {
232				int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
233				int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
234				if (!symbol_id) {
235					bpf_map_update_elem(&symbolmap, &sym, &zero, 0);
236					symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
237					if (!symbol_id)
238						return 0;
239				}
240				if (*symbol_id == new_symbol_id)
241					(*symbol_counter)++;
242				event->stack[i] = *symbol_id;
243				event->stack_len = i + 1;
244				frame_ptr = frame.f_back;
245			}
246		}
247		event->stack_complete = frame_ptr == NULL;
248	} else {
249		event->stack_complete = 1;
250	}
251
252	Stats* stats = bpf_map_lookup_elem(&statsmap, &zero);
253	if (stats)
254		stats->success++;
255
256	event->has_meta = 0;
257	bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata));
258	return 0;
259}
260
261SEC("raw_tracepoint/kfree_skb")
262int on_event(struct bpf_raw_tracepoint_args* ctx)
263{
264	int i, ret = 0;
265	ret |= __on_event(ctx);
266	ret |= __on_event(ctx);
267	ret |= __on_event(ctx);
268	ret |= __on_event(ctx);
269	ret |= __on_event(ctx);
270	return ret;
271}
272
273char _license[] SEC("license") = "GPL";