1// SPDX-License-Identifier: GPL-2.0
  2
  3#define pr_fmt(fmt) "rethook: " fmt
  4
  5#include <linux/bug.h>
  6#include <linux/kallsyms.h>
  7#include <linux/kprobes.h>
  8#include <linux/preempt.h>
  9#include <linux/rethook.h>
 10#include <linux/slab.h>
 11#include <linux/sort.h>
 12
 13/* Return hook list (shadow stack by list) */
 14
 15/*
 16 * This function is called from delayed_put_task_struct() when a task is
 17 * dead and cleaned up to recycle any kretprobe instances associated with
 18 * this task. These left over instances represent probed functions that
 19 * have been called but will never return.
 20 */
 21void rethook_flush_task(struct task_struct *tk)
 22{
 23	struct rethook_node *rhn;
 24	struct llist_node *node;
 25
 26	node = __llist_del_all(&tk->rethooks);
 27	while (node) {
 28		rhn = container_of(node, struct rethook_node, llist);
 29		node = node->next;
 30		preempt_disable();
 31		rethook_recycle(rhn);
 32		preempt_enable();
 33	}
 34}
 35
 36static void rethook_free_rcu(struct rcu_head *head)
 37{
 38	struct rethook *rh = container_of(head, struct rethook, rcu);
 39	struct rethook_node *rhn;
 40	struct freelist_node *node;
 41	int count = 1;
 42
 43	node = rh->pool.head;
 44	while (node) {
 45		rhn = container_of(node, struct rethook_node, freelist);
 46		node = node->next;
 47		kfree(rhn);
 48		count++;
 49	}
 50
 51	/* The rh->ref is the number of pooled node + 1 */
 52	if (refcount_sub_and_test(count, &rh->ref))
 53		kfree(rh);
 54}
 55
 56/**
 57 * rethook_free() - Free struct rethook.
 58 * @rh: the struct rethook to be freed.
 59 *
 60 * Free the rethook. Before calling this function, user must ensure the
 61 * @rh::data is cleaned if needed (or, the handler can access it after
 62 * calling this function.) This function will set the @rh to be freed
 63 * after all rethook_node are freed (not soon). And the caller must
 64 * not touch @rh after calling this.
 65 */
 66void rethook_free(struct rethook *rh)
 67{
 68	WRITE_ONCE(rh->handler, NULL);
 69
 70	call_rcu(&rh->rcu, rethook_free_rcu);
 71}
 72
 73/**
 74 * rethook_alloc() - Allocate struct rethook.
 75 * @data: a data to pass the @handler when hooking the return.
 76 * @handler: the return hook callback function.
 77 *
 78 * Allocate and initialize a new rethook with @data and @handler.
 79 * Return NULL if memory allocation fails or @handler is NULL.
 80 * Note that @handler == NULL means this rethook is going to be freed.
 81 */
 82struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
 83{
 84	struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
 85
 86	if (!rh || !handler) {
 87		kfree(rh);
 88		return NULL;
 89	}
 90
 91	rh->data = data;
 92	rh->handler = handler;
 93	rh->pool.head = NULL;
 94	refcount_set(&rh->ref, 1);
 95
 96	return rh;
 97}
 98
 99/**
100 * rethook_add_node() - Add a new node to the rethook.
101 * @rh: the struct rethook.
102 * @node: the struct rethook_node to be added.
103 *
104 * Add @node to @rh. User must allocate @node (as a part of user's
105 * data structure.) The @node fields are initialized in this function.
106 */
107void rethook_add_node(struct rethook *rh, struct rethook_node *node)
108{
109	node->rethook = rh;
110	freelist_add(&node->freelist, &rh->pool);
111	refcount_inc(&rh->ref);
112}
113
114static void free_rethook_node_rcu(struct rcu_head *head)
115{
116	struct rethook_node *node = container_of(head, struct rethook_node, rcu);
117
118	if (refcount_dec_and_test(&node->rethook->ref))
119		kfree(node->rethook);
120	kfree(node);
121}
122
123/**
124 * rethook_recycle() - return the node to rethook.
125 * @node: The struct rethook_node to be returned.
126 *
127 * Return back the @node to @node::rethook. If the @node::rethook is already
128 * marked as freed, this will free the @node.
129 */
130void rethook_recycle(struct rethook_node *node)
131{
132	lockdep_assert_preemption_disabled();
133
134	if (likely(READ_ONCE(node->rethook->handler)))
135		freelist_add(&node->freelist, &node->rethook->pool);
136	else
137		call_rcu(&node->rcu, free_rethook_node_rcu);
138}
139NOKPROBE_SYMBOL(rethook_recycle);
140
141/**
142 * rethook_try_get() - get an unused rethook node.
143 * @rh: The struct rethook which pools the nodes.
144 *
145 * Get an unused rethook node from @rh. If the node pool is empty, this
146 * will return NULL. Caller must disable preemption.
147 */
148struct rethook_node *rethook_try_get(struct rethook *rh)
149{
150	rethook_handler_t handler = READ_ONCE(rh->handler);
151	struct freelist_node *fn;
152
153	lockdep_assert_preemption_disabled();
154
155	/* Check whether @rh is going to be freed. */
156	if (unlikely(!handler))
157		return NULL;
158
159	/*
160	 * This expects the caller will set up a rethook on a function entry.
161	 * When the function returns, the rethook will eventually be reclaimed
162	 * or released in the rethook_recycle() with call_rcu().
163	 * This means the caller must be run in the RCU-availabe context.
164	 */
165	if (unlikely(!rcu_is_watching()))
166		return NULL;
167
168	fn = freelist_try_get(&rh->pool);
169	if (!fn)
170		return NULL;
171
172	return container_of(fn, struct rethook_node, freelist);
173}
174NOKPROBE_SYMBOL(rethook_try_get);
175
176/**
177 * rethook_hook() - Hook the current function return.
178 * @node: The struct rethook node to hook the function return.
179 * @regs: The struct pt_regs for the function entry.
180 * @mcount: True if this is called from mcount(ftrace) context.
181 *
182 * Hook the current running function return. This must be called when the
183 * function entry (or at least @regs must be the registers of the function
184 * entry.) @mcount is used for identifying the context. If this is called
185 * from ftrace (mcount) callback, @mcount must be set true. If this is called
186 * from the real function entry (e.g. kprobes) @mcount must be set false.
187 * This is because the way to hook the function return depends on the context.
188 */
189void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
190{
191	arch_rethook_prepare(node, regs, mcount);
192	__llist_add(&node->llist, &current->rethooks);
193}
194NOKPROBE_SYMBOL(rethook_hook);
195
196/* This assumes the 'tsk' is the current task or is not running. */
197static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
198					     struct llist_node **cur)
199{
200	struct rethook_node *rh = NULL;
201	struct llist_node *node = *cur;
202
203	if (!node)
204		node = tsk->rethooks.first;
205	else
206		node = node->next;
207
208	while (node) {
209		rh = container_of(node, struct rethook_node, llist);
210		if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
211			*cur = node;
212			return rh->ret_addr;
213		}
214		node = node->next;
215	}
216	return 0;
217}
218NOKPROBE_SYMBOL(__rethook_find_ret_addr);
219
220/**
221 * rethook_find_ret_addr -- Find correct return address modified by rethook
222 * @tsk: Target task
223 * @frame: A frame pointer
224 * @cur: a storage of the loop cursor llist_node pointer for next call
225 *
226 * Find the correct return address modified by a rethook on @tsk in unsigned
227 * long type.
228 * The @tsk must be 'current' or a task which is not running. @frame is a hint
229 * to get the currect return address - which is compared with the
230 * rethook::frame field. The @cur is a loop cursor for searching the
231 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
232 * first call, but '@cur' itself must NOT NULL.
233 *
234 * Returns found address value or zero if not found.
235 */
236unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
237				    struct llist_node **cur)
238{
239	struct rethook_node *rhn = NULL;
240	unsigned long ret;
241
242	if (WARN_ON_ONCE(!cur))
243		return 0;
244
245	if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
246		return 0;
247
248	do {
249		ret = __rethook_find_ret_addr(tsk, cur);
250		if (!ret)
251			break;
252		rhn = container_of(*cur, struct rethook_node, llist);
253	} while (rhn->frame != frame);
254
255	return ret;
256}
257NOKPROBE_SYMBOL(rethook_find_ret_addr);
258
259void __weak arch_rethook_fixup_return(struct pt_regs *regs,
260				      unsigned long correct_ret_addr)
261{
262	/*
263	 * Do nothing by default. If the architecture which uses a
264	 * frame pointer to record real return address on the stack,
265	 * it should fill this function to fixup the return address
266	 * so that stacktrace works from the rethook handler.
267	 */
268}
269
270/* This function will be called from each arch-defined trampoline. */
271unsigned long rethook_trampoline_handler(struct pt_regs *regs,
272					 unsigned long frame)
273{
274	struct llist_node *first, *node = NULL;
275	unsigned long correct_ret_addr;
276	rethook_handler_t handler;
277	struct rethook_node *rhn;
278
279	correct_ret_addr = __rethook_find_ret_addr(current, &node);
280	if (!correct_ret_addr) {
281		pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
282		BUG_ON(1);
283	}
284
285	instruction_pointer_set(regs, correct_ret_addr);
286
287	/*
288	 * These loops must be protected from rethook_free_rcu() because those
289	 * are accessing 'rhn->rethook'.
290	 */
291	preempt_disable();
292
293	/*
294	 * Run the handler on the shadow stack. Do not unlink the list here because
295	 * stackdump inside the handlers needs to decode it.
296	 */
297	first = current->rethooks.first;
298	while (first) {
299		rhn = container_of(first, struct rethook_node, llist);
300		if (WARN_ON_ONCE(rhn->frame != frame))
301			break;
302		handler = READ_ONCE(rhn->rethook->handler);
303		if (handler)
304			handler(rhn, rhn->rethook->data, regs);
305
306		if (first == node)
307			break;
308		first = first->next;
309	}
310
311	/* Fixup registers for returning to correct address. */
312	arch_rethook_fixup_return(regs, correct_ret_addr);
313
314	/* Unlink used shadow stack */
315	first = current->rethooks.first;
316	current->rethooks.first = node->next;
317	node->next = NULL;
318
319	while (first) {
320		rhn = container_of(first, struct rethook_node, llist);
321		first = first->next;
322		rethook_recycle(rhn);
323	}
324	preempt_enable();
325
326	return correct_ret_addr;
327}
328NOKPROBE_SYMBOL(rethook_trampoline_handler);