Loading...
1// SPDX-License-Identifier: GPL-2.0
2
3#define pr_fmt(fmt) "rethook: " fmt
4
5#include <linux/bug.h>
6#include <linux/kallsyms.h>
7#include <linux/kprobes.h>
8#include <linux/preempt.h>
9#include <linux/rethook.h>
10#include <linux/slab.h>
11
12/* Return hook list (shadow stack by list) */
13
14/*
15 * This function is called from delayed_put_task_struct() when a task is
16 * dead and cleaned up to recycle any kretprobe instances associated with
17 * this task. These left over instances represent probed functions that
18 * have been called but will never return.
19 */
20void rethook_flush_task(struct task_struct *tk)
21{
22 struct rethook_node *rhn;
23 struct llist_node *node;
24
25 node = __llist_del_all(&tk->rethooks);
26 while (node) {
27 rhn = container_of(node, struct rethook_node, llist);
28 node = node->next;
29 preempt_disable();
30 rethook_recycle(rhn);
31 preempt_enable();
32 }
33}
34
35static void rethook_free_rcu(struct rcu_head *head)
36{
37 struct rethook *rh = container_of(head, struct rethook, rcu);
38 objpool_fini(&rh->pool);
39}
40
41/**
42 * rethook_stop() - Stop using a rethook.
43 * @rh: the struct rethook to stop.
44 *
45 * Stop using a rethook to prepare for freeing it. If you want to wait for
46 * all running rethook handler before calling rethook_free(), you need to
47 * call this first and wait RCU, and call rethook_free().
48 */
49void rethook_stop(struct rethook *rh)
50{
51 rcu_assign_pointer(rh->handler, NULL);
52}
53
54/**
55 * rethook_free() - Free struct rethook.
56 * @rh: the struct rethook to be freed.
57 *
58 * Free the rethook. Before calling this function, user must ensure the
59 * @rh::data is cleaned if needed (or, the handler can access it after
60 * calling this function.) This function will set the @rh to be freed
61 * after all rethook_node are freed (not soon). And the caller must
62 * not touch @rh after calling this.
63 */
64void rethook_free(struct rethook *rh)
65{
66 rethook_stop(rh);
67
68 call_rcu(&rh->rcu, rethook_free_rcu);
69}
70
71static int rethook_init_node(void *nod, void *context)
72{
73 struct rethook_node *node = nod;
74
75 node->rethook = context;
76 return 0;
77}
78
79static int rethook_fini_pool(struct objpool_head *head, void *context)
80{
81 kfree(context);
82 return 0;
83}
84
85static inline rethook_handler_t rethook_get_handler(struct rethook *rh)
86{
87 return (rethook_handler_t)rcu_dereference_check(rh->handler,
88 rcu_read_lock_any_held());
89}
90
91/**
92 * rethook_alloc() - Allocate struct rethook.
93 * @data: a data to pass the @handler when hooking the return.
94 * @handler: the return hook callback function, must NOT be NULL
95 * @size: node size: rethook node and additional data
96 * @num: number of rethook nodes to be preallocated
97 *
98 * Allocate and initialize a new rethook with @data and @handler.
99 * Return pointer of new rethook, or error codes for failures.
100 *
101 * Note that @handler == NULL means this rethook is going to be freed.
102 */
103struct rethook *rethook_alloc(void *data, rethook_handler_t handler,
104 int size, int num)
105{
106 struct rethook *rh;
107
108 if (!handler || num <= 0 || size < sizeof(struct rethook_node))
109 return ERR_PTR(-EINVAL);
110
111 rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
112 if (!rh)
113 return ERR_PTR(-ENOMEM);
114
115 rh->data = data;
116 rcu_assign_pointer(rh->handler, handler);
117
118 /* initialize the objpool for rethook nodes */
119 if (objpool_init(&rh->pool, num, size, GFP_KERNEL, rh,
120 rethook_init_node, rethook_fini_pool)) {
121 kfree(rh);
122 return ERR_PTR(-ENOMEM);
123 }
124 return rh;
125}
126
127static void free_rethook_node_rcu(struct rcu_head *head)
128{
129 struct rethook_node *node = container_of(head, struct rethook_node, rcu);
130 struct rethook *rh = node->rethook;
131
132 objpool_drop(node, &rh->pool);
133}
134
135/**
136 * rethook_recycle() - return the node to rethook.
137 * @node: The struct rethook_node to be returned.
138 *
139 * Return back the @node to @node::rethook. If the @node::rethook is already
140 * marked as freed, this will free the @node.
141 */
142void rethook_recycle(struct rethook_node *node)
143{
144 rethook_handler_t handler;
145
146 handler = rethook_get_handler(node->rethook);
147 if (likely(handler))
148 objpool_push(node, &node->rethook->pool);
149 else
150 call_rcu(&node->rcu, free_rethook_node_rcu);
151}
152NOKPROBE_SYMBOL(rethook_recycle);
153
154/**
155 * rethook_try_get() - get an unused rethook node.
156 * @rh: The struct rethook which pools the nodes.
157 *
158 * Get an unused rethook node from @rh. If the node pool is empty, this
159 * will return NULL. Caller must disable preemption.
160 */
161struct rethook_node *rethook_try_get(struct rethook *rh)
162{
163 rethook_handler_t handler = rethook_get_handler(rh);
164
165 /* Check whether @rh is going to be freed. */
166 if (unlikely(!handler))
167 return NULL;
168
169 /*
170 * This expects the caller will set up a rethook on a function entry.
171 * When the function returns, the rethook will eventually be reclaimed
172 * or released in the rethook_recycle() with call_rcu().
173 * This means the caller must be run in the RCU-availabe context.
174 */
175 if (unlikely(!rcu_is_watching()))
176 return NULL;
177
178 return (struct rethook_node *)objpool_pop(&rh->pool);
179}
180NOKPROBE_SYMBOL(rethook_try_get);
181
182/**
183 * rethook_hook() - Hook the current function return.
184 * @node: The struct rethook node to hook the function return.
185 * @regs: The struct pt_regs for the function entry.
186 * @mcount: True if this is called from mcount(ftrace) context.
187 *
188 * Hook the current running function return. This must be called when the
189 * function entry (or at least @regs must be the registers of the function
190 * entry.) @mcount is used for identifying the context. If this is called
191 * from ftrace (mcount) callback, @mcount must be set true. If this is called
192 * from the real function entry (e.g. kprobes) @mcount must be set false.
193 * This is because the way to hook the function return depends on the context.
194 */
195void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
196{
197 arch_rethook_prepare(node, regs, mcount);
198 __llist_add(&node->llist, ¤t->rethooks);
199}
200NOKPROBE_SYMBOL(rethook_hook);
201
202/* This assumes the 'tsk' is the current task or is not running. */
203static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
204 struct llist_node **cur)
205{
206 struct rethook_node *rh = NULL;
207 struct llist_node *node = *cur;
208
209 if (!node)
210 node = tsk->rethooks.first;
211 else
212 node = node->next;
213
214 while (node) {
215 rh = container_of(node, struct rethook_node, llist);
216 if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
217 *cur = node;
218 return rh->ret_addr;
219 }
220 node = node->next;
221 }
222 return 0;
223}
224NOKPROBE_SYMBOL(__rethook_find_ret_addr);
225
226/**
227 * rethook_find_ret_addr -- Find correct return address modified by rethook
228 * @tsk: Target task
229 * @frame: A frame pointer
230 * @cur: a storage of the loop cursor llist_node pointer for next call
231 *
232 * Find the correct return address modified by a rethook on @tsk in unsigned
233 * long type.
234 * The @tsk must be 'current' or a task which is not running. @frame is a hint
235 * to get the currect return address - which is compared with the
236 * rethook::frame field. The @cur is a loop cursor for searching the
237 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
238 * first call, but '@cur' itself must NOT NULL.
239 *
240 * Returns found address value or zero if not found.
241 */
242unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
243 struct llist_node **cur)
244{
245 struct rethook_node *rhn = NULL;
246 unsigned long ret;
247
248 if (WARN_ON_ONCE(!cur))
249 return 0;
250
251 if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
252 return 0;
253
254 do {
255 ret = __rethook_find_ret_addr(tsk, cur);
256 if (!ret)
257 break;
258 rhn = container_of(*cur, struct rethook_node, llist);
259 } while (rhn->frame != frame);
260
261 return ret;
262}
263NOKPROBE_SYMBOL(rethook_find_ret_addr);
264
265void __weak arch_rethook_fixup_return(struct pt_regs *regs,
266 unsigned long correct_ret_addr)
267{
268 /*
269 * Do nothing by default. If the architecture which uses a
270 * frame pointer to record real return address on the stack,
271 * it should fill this function to fixup the return address
272 * so that stacktrace works from the rethook handler.
273 */
274}
275
276/* This function will be called from each arch-defined trampoline. */
277unsigned long rethook_trampoline_handler(struct pt_regs *regs,
278 unsigned long frame)
279{
280 struct llist_node *first, *node = NULL;
281 unsigned long correct_ret_addr;
282 rethook_handler_t handler;
283 struct rethook_node *rhn;
284
285 correct_ret_addr = __rethook_find_ret_addr(current, &node);
286 if (!correct_ret_addr) {
287 pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
288 BUG_ON(1);
289 }
290
291 instruction_pointer_set(regs, correct_ret_addr);
292
293 /*
294 * These loops must be protected from rethook_free_rcu() because those
295 * are accessing 'rhn->rethook'.
296 */
297 preempt_disable_notrace();
298
299 /*
300 * Run the handler on the shadow stack. Do not unlink the list here because
301 * stackdump inside the handlers needs to decode it.
302 */
303 first = current->rethooks.first;
304 while (first) {
305 rhn = container_of(first, struct rethook_node, llist);
306 if (WARN_ON_ONCE(rhn->frame != frame))
307 break;
308 handler = rethook_get_handler(rhn->rethook);
309 if (handler)
310 handler(rhn, rhn->rethook->data,
311 correct_ret_addr, regs);
312
313 if (first == node)
314 break;
315 first = first->next;
316 }
317
318 /* Fixup registers for returning to correct address. */
319 arch_rethook_fixup_return(regs, correct_ret_addr);
320
321 /* Unlink used shadow stack */
322 first = current->rethooks.first;
323 current->rethooks.first = node->next;
324 node->next = NULL;
325
326 while (first) {
327 rhn = container_of(first, struct rethook_node, llist);
328 first = first->next;
329 rethook_recycle(rhn);
330 }
331 preempt_enable_notrace();
332
333 return correct_ret_addr;
334}
335NOKPROBE_SYMBOL(rethook_trampoline_handler);
1// SPDX-License-Identifier: GPL-2.0
2
3#define pr_fmt(fmt) "rethook: " fmt
4
5#include <linux/bug.h>
6#include <linux/kallsyms.h>
7#include <linux/kprobes.h>
8#include <linux/preempt.h>
9#include <linux/rethook.h>
10#include <linux/slab.h>
11#include <linux/sort.h>
12
13/* Return hook list (shadow stack by list) */
14
15/*
16 * This function is called from delayed_put_task_struct() when a task is
17 * dead and cleaned up to recycle any kretprobe instances associated with
18 * this task. These left over instances represent probed functions that
19 * have been called but will never return.
20 */
21void rethook_flush_task(struct task_struct *tk)
22{
23 struct rethook_node *rhn;
24 struct llist_node *node;
25
26 node = __llist_del_all(&tk->rethooks);
27 while (node) {
28 rhn = container_of(node, struct rethook_node, llist);
29 node = node->next;
30 preempt_disable();
31 rethook_recycle(rhn);
32 preempt_enable();
33 }
34}
35
36static void rethook_free_rcu(struct rcu_head *head)
37{
38 struct rethook *rh = container_of(head, struct rethook, rcu);
39 struct rethook_node *rhn;
40 struct freelist_node *node;
41 int count = 1;
42
43 node = rh->pool.head;
44 while (node) {
45 rhn = container_of(node, struct rethook_node, freelist);
46 node = node->next;
47 kfree(rhn);
48 count++;
49 }
50
51 /* The rh->ref is the number of pooled node + 1 */
52 if (refcount_sub_and_test(count, &rh->ref))
53 kfree(rh);
54}
55
56/**
57 * rethook_free() - Free struct rethook.
58 * @rh: the struct rethook to be freed.
59 *
60 * Free the rethook. Before calling this function, user must ensure the
61 * @rh::data is cleaned if needed (or, the handler can access it after
62 * calling this function.) This function will set the @rh to be freed
63 * after all rethook_node are freed (not soon). And the caller must
64 * not touch @rh after calling this.
65 */
66void rethook_free(struct rethook *rh)
67{
68 WRITE_ONCE(rh->handler, NULL);
69
70 call_rcu(&rh->rcu, rethook_free_rcu);
71}
72
73/**
74 * rethook_alloc() - Allocate struct rethook.
75 * @data: a data to pass the @handler when hooking the return.
76 * @handler: the return hook callback function.
77 *
78 * Allocate and initialize a new rethook with @data and @handler.
79 * Return NULL if memory allocation fails or @handler is NULL.
80 * Note that @handler == NULL means this rethook is going to be freed.
81 */
82struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
83{
84 struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
85
86 if (!rh || !handler) {
87 kfree(rh);
88 return NULL;
89 }
90
91 rh->data = data;
92 rh->handler = handler;
93 rh->pool.head = NULL;
94 refcount_set(&rh->ref, 1);
95
96 return rh;
97}
98
99/**
100 * rethook_add_node() - Add a new node to the rethook.
101 * @rh: the struct rethook.
102 * @node: the struct rethook_node to be added.
103 *
104 * Add @node to @rh. User must allocate @node (as a part of user's
105 * data structure.) The @node fields are initialized in this function.
106 */
107void rethook_add_node(struct rethook *rh, struct rethook_node *node)
108{
109 node->rethook = rh;
110 freelist_add(&node->freelist, &rh->pool);
111 refcount_inc(&rh->ref);
112}
113
114static void free_rethook_node_rcu(struct rcu_head *head)
115{
116 struct rethook_node *node = container_of(head, struct rethook_node, rcu);
117
118 if (refcount_dec_and_test(&node->rethook->ref))
119 kfree(node->rethook);
120 kfree(node);
121}
122
123/**
124 * rethook_recycle() - return the node to rethook.
125 * @node: The struct rethook_node to be returned.
126 *
127 * Return back the @node to @node::rethook. If the @node::rethook is already
128 * marked as freed, this will free the @node.
129 */
130void rethook_recycle(struct rethook_node *node)
131{
132 lockdep_assert_preemption_disabled();
133
134 if (likely(READ_ONCE(node->rethook->handler)))
135 freelist_add(&node->freelist, &node->rethook->pool);
136 else
137 call_rcu(&node->rcu, free_rethook_node_rcu);
138}
139NOKPROBE_SYMBOL(rethook_recycle);
140
141/**
142 * rethook_try_get() - get an unused rethook node.
143 * @rh: The struct rethook which pools the nodes.
144 *
145 * Get an unused rethook node from @rh. If the node pool is empty, this
146 * will return NULL. Caller must disable preemption.
147 */
148struct rethook_node *rethook_try_get(struct rethook *rh)
149{
150 rethook_handler_t handler = READ_ONCE(rh->handler);
151 struct freelist_node *fn;
152
153 lockdep_assert_preemption_disabled();
154
155 /* Check whether @rh is going to be freed. */
156 if (unlikely(!handler))
157 return NULL;
158
159 /*
160 * This expects the caller will set up a rethook on a function entry.
161 * When the function returns, the rethook will eventually be reclaimed
162 * or released in the rethook_recycle() with call_rcu().
163 * This means the caller must be run in the RCU-availabe context.
164 */
165 if (unlikely(!rcu_is_watching()))
166 return NULL;
167
168 fn = freelist_try_get(&rh->pool);
169 if (!fn)
170 return NULL;
171
172 return container_of(fn, struct rethook_node, freelist);
173}
174NOKPROBE_SYMBOL(rethook_try_get);
175
176/**
177 * rethook_hook() - Hook the current function return.
178 * @node: The struct rethook node to hook the function return.
179 * @regs: The struct pt_regs for the function entry.
180 * @mcount: True if this is called from mcount(ftrace) context.
181 *
182 * Hook the current running function return. This must be called when the
183 * function entry (or at least @regs must be the registers of the function
184 * entry.) @mcount is used for identifying the context. If this is called
185 * from ftrace (mcount) callback, @mcount must be set true. If this is called
186 * from the real function entry (e.g. kprobes) @mcount must be set false.
187 * This is because the way to hook the function return depends on the context.
188 */
189void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
190{
191 arch_rethook_prepare(node, regs, mcount);
192 __llist_add(&node->llist, ¤t->rethooks);
193}
194NOKPROBE_SYMBOL(rethook_hook);
195
196/* This assumes the 'tsk' is the current task or is not running. */
197static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
198 struct llist_node **cur)
199{
200 struct rethook_node *rh = NULL;
201 struct llist_node *node = *cur;
202
203 if (!node)
204 node = tsk->rethooks.first;
205 else
206 node = node->next;
207
208 while (node) {
209 rh = container_of(node, struct rethook_node, llist);
210 if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
211 *cur = node;
212 return rh->ret_addr;
213 }
214 node = node->next;
215 }
216 return 0;
217}
218NOKPROBE_SYMBOL(__rethook_find_ret_addr);
219
220/**
221 * rethook_find_ret_addr -- Find correct return address modified by rethook
222 * @tsk: Target task
223 * @frame: A frame pointer
224 * @cur: a storage of the loop cursor llist_node pointer for next call
225 *
226 * Find the correct return address modified by a rethook on @tsk in unsigned
227 * long type.
228 * The @tsk must be 'current' or a task which is not running. @frame is a hint
229 * to get the currect return address - which is compared with the
230 * rethook::frame field. The @cur is a loop cursor for searching the
231 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
232 * first call, but '@cur' itself must NOT NULL.
233 *
234 * Returns found address value or zero if not found.
235 */
236unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
237 struct llist_node **cur)
238{
239 struct rethook_node *rhn = NULL;
240 unsigned long ret;
241
242 if (WARN_ON_ONCE(!cur))
243 return 0;
244
245 if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
246 return 0;
247
248 do {
249 ret = __rethook_find_ret_addr(tsk, cur);
250 if (!ret)
251 break;
252 rhn = container_of(*cur, struct rethook_node, llist);
253 } while (rhn->frame != frame);
254
255 return ret;
256}
257NOKPROBE_SYMBOL(rethook_find_ret_addr);
258
259void __weak arch_rethook_fixup_return(struct pt_regs *regs,
260 unsigned long correct_ret_addr)
261{
262 /*
263 * Do nothing by default. If the architecture which uses a
264 * frame pointer to record real return address on the stack,
265 * it should fill this function to fixup the return address
266 * so that stacktrace works from the rethook handler.
267 */
268}
269
270/* This function will be called from each arch-defined trampoline. */
271unsigned long rethook_trampoline_handler(struct pt_regs *regs,
272 unsigned long frame)
273{
274 struct llist_node *first, *node = NULL;
275 unsigned long correct_ret_addr;
276 rethook_handler_t handler;
277 struct rethook_node *rhn;
278
279 correct_ret_addr = __rethook_find_ret_addr(current, &node);
280 if (!correct_ret_addr) {
281 pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
282 BUG_ON(1);
283 }
284
285 instruction_pointer_set(regs, correct_ret_addr);
286
287 /*
288 * These loops must be protected from rethook_free_rcu() because those
289 * are accessing 'rhn->rethook'.
290 */
291 preempt_disable();
292
293 /*
294 * Run the handler on the shadow stack. Do not unlink the list here because
295 * stackdump inside the handlers needs to decode it.
296 */
297 first = current->rethooks.first;
298 while (first) {
299 rhn = container_of(first, struct rethook_node, llist);
300 if (WARN_ON_ONCE(rhn->frame != frame))
301 break;
302 handler = READ_ONCE(rhn->rethook->handler);
303 if (handler)
304 handler(rhn, rhn->rethook->data, regs);
305
306 if (first == node)
307 break;
308 first = first->next;
309 }
310
311 /* Fixup registers for returning to correct address. */
312 arch_rethook_fixup_return(regs, correct_ret_addr);
313
314 /* Unlink used shadow stack */
315 first = current->rethooks.first;
316 current->rethooks.first = node->next;
317 node->next = NULL;
318
319 while (first) {
320 rhn = container_of(first, struct rethook_node, llist);
321 first = first->next;
322 rethook_recycle(rhn);
323 }
324 preempt_enable();
325
326 return correct_ret_addr;
327}
328NOKPROBE_SYMBOL(rethook_trampoline_handler);