Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * KFENCE reporting.
4 *
5 * Copyright (C) 2020, Google LLC.
6 */
7
8#include <linux/stdarg.h>
9
10#include <linux/kernel.h>
11#include <linux/lockdep.h>
12#include <linux/math.h>
13#include <linux/printk.h>
14#include <linux/sched/debug.h>
15#include <linux/seq_file.h>
16#include <linux/stacktrace.h>
17#include <linux/string.h>
18#include <trace/events/error_report.h>
19
20#include <asm/kfence.h>
21
22#include "kfence.h"
23
24/* May be overridden by <asm/kfence.h>. */
25#ifndef ARCH_FUNC_PREFIX
26#define ARCH_FUNC_PREFIX ""
27#endif
28
29extern bool no_hash_pointers;
30
31/* Helper function to either print to a seq_file or to console. */
32__printf(2, 3)
33static void seq_con_printf(struct seq_file *seq, const char *fmt, ...)
34{
35 va_list args;
36
37 va_start(args, fmt);
38 if (seq)
39 seq_vprintf(seq, fmt, args);
40 else
41 vprintk(fmt, args);
42 va_end(args);
43}
44
45/*
46 * Get the number of stack entries to skip to get out of MM internals. @type is
47 * optional, and if set to NULL, assumes an allocation or free stack.
48 */
49static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries,
50 const enum kfence_error_type *type)
51{
52 char buf[64];
53 int skipnr, fallback = 0;
54
55 if (type) {
56 /* Depending on error type, find different stack entries. */
57 switch (*type) {
58 case KFENCE_ERROR_UAF:
59 case KFENCE_ERROR_OOB:
60 case KFENCE_ERROR_INVALID:
61 /*
62 * kfence_handle_page_fault() may be called with pt_regs
63 * set to NULL; in that case we'll simply show the full
64 * stack trace.
65 */
66 return 0;
67 case KFENCE_ERROR_CORRUPTION:
68 case KFENCE_ERROR_INVALID_FREE:
69 break;
70 }
71 }
72
73 for (skipnr = 0; skipnr < num_entries; skipnr++) {
74 int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]);
75
76 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") ||
77 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") ||
78 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||
79 !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) {
80 /*
81 * In case of tail calls from any of the below to any of
82 * the above, optimized by the compiler such that the
83 * stack trace would omit the initial entry point below.
84 */
85 fallback = skipnr + 1;
86 }
87
88 /*
89 * The below list should only include the initial entry points
90 * into the slab allocators. Includes the *_bulk() variants by
91 * checking prefixes.
92 */
93 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||
94 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||
95 str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||
96 str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))
97 goto found;
98 }
99 if (fallback < num_entries)
100 return fallback;
101found:
102 skipnr++;
103 return skipnr < num_entries ? skipnr : 0;
104}
105
106static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta,
107 bool show_alloc)
108{
109 const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;
110 u64 ts_sec = track->ts_nsec;
111 unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC);
112
113 /* Timestamp matches printk timestamp format. */
114 seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus:\n",
115 show_alloc ? "allocated" : "freed", track->pid,
116 track->cpu, (unsigned long)ts_sec, rem_nsec / 1000);
117
118 if (track->num_stack_entries) {
119 /* Skip allocation/free internals stack. */
120 int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
121
122 /* stack_trace_seq_print() does not exist; open code our own. */
123 for (; i < track->num_stack_entries; i++)
124 seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]);
125 } else {
126 seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation");
127 }
128}
129
130void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta)
131{
132 const int size = abs(meta->size);
133 const unsigned long start = meta->addr;
134 const struct kmem_cache *const cache = meta->cache;
135
136 lockdep_assert_held(&meta->lock);
137
138 if (meta->state == KFENCE_OBJECT_UNUSED) {
139 seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata);
140 return;
141 }
142
143 seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n",
144 meta - kfence_metadata, (void *)start, (void *)(start + size - 1),
145 size, (cache && cache->name) ? cache->name : "<destroyed>");
146
147 kfence_print_stack(seq, meta, true);
148
149 if (meta->state == KFENCE_OBJECT_FREED) {
150 seq_con_printf(seq, "\n");
151 kfence_print_stack(seq, meta, false);
152 }
153}
154
155/*
156 * Show bytes at @addr that are different from the expected canary values, up to
157 * @max_bytes.
158 */
159static void print_diff_canary(unsigned long address, size_t bytes_to_show,
160 const struct kfence_metadata *meta)
161{
162 const unsigned long show_until_addr = address + bytes_to_show;
163 const u8 *cur, *end;
164
165 /* Do not show contents of object nor read into following guard page. */
166 end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr)
167 : min(show_until_addr, PAGE_ALIGN(address)));
168
169 pr_cont("[");
170 for (cur = (const u8 *)address; cur < end; cur++) {
171 if (*cur == KFENCE_CANARY_PATTERN(cur))
172 pr_cont(" .");
173 else if (no_hash_pointers)
174 pr_cont(" 0x%02x", *cur);
175 else /* Do not leak kernel memory in non-debug builds. */
176 pr_cont(" !");
177 }
178 pr_cont(" ]");
179}
180
181static const char *get_access_type(bool is_write)
182{
183 return is_write ? "write" : "read";
184}
185
186void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
187 const struct kfence_metadata *meta, enum kfence_error_type type)
188{
189 unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 };
190 const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1;
191 int num_stack_entries;
192 int skipnr = 0;
193
194 if (regs) {
195 num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0);
196 } else {
197 num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1);
198 skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type);
199 }
200
201 /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */
202 if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta))
203 return;
204
205 if (meta)
206 lockdep_assert_held(&meta->lock);
207 /*
208 * Because we may generate reports in printk-unfriendly parts of the
209 * kernel, such as scheduler code, the use of printk() could deadlock.
210 * Until such time that all printing code here is safe in all parts of
211 * the kernel, accept the risk, and just get our message out (given the
212 * system might already behave unpredictably due to the memory error).
213 * As such, also disable lockdep to hide warnings, and avoid disabling
214 * lockdep for the rest of the kernel.
215 */
216 lockdep_off();
217
218 pr_err("==================================================================\n");
219 /* Print report header. */
220 switch (type) {
221 case KFENCE_ERROR_OOB: {
222 const bool left_of_object = address < meta->addr;
223
224 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write),
225 (void *)stack_entries[skipnr]);
226 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n",
227 get_access_type(is_write), (void *)address,
228 left_of_object ? meta->addr - address : address - meta->addr,
229 left_of_object ? "left" : "right", object_index);
230 break;
231 }
232 case KFENCE_ERROR_UAF:
233 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write),
234 (void *)stack_entries[skipnr]);
235 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n",
236 get_access_type(is_write), (void *)address, object_index);
237 break;
238 case KFENCE_ERROR_CORRUPTION:
239 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]);
240 pr_err("Corrupted memory at 0x%p ", (void *)address);
241 print_diff_canary(address, 16, meta);
242 pr_cont(" (in kfence-#%td):\n", object_index);
243 break;
244 case KFENCE_ERROR_INVALID:
245 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write),
246 (void *)stack_entries[skipnr]);
247 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write),
248 (void *)address);
249 break;
250 case KFENCE_ERROR_INVALID_FREE:
251 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]);
252 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address,
253 object_index);
254 break;
255 }
256
257 /* Print stack trace and object info. */
258 stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0);
259
260 if (meta) {
261 pr_err("\n");
262 kfence_print_object(NULL, meta);
263 }
264
265 /* Print report footer. */
266 pr_err("\n");
267 if (no_hash_pointers && regs)
268 show_regs(regs);
269 else
270 dump_stack_print_info(KERN_ERR);
271 trace_error_report_end(ERROR_DETECTOR_KFENCE, address);
272 pr_err("==================================================================\n");
273
274 lockdep_on();
275
276 check_panic_on_warn("KFENCE");
277
278 /* We encountered a memory safety error, taint the kernel! */
279 add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK);
280}
281
282#ifdef CONFIG_PRINTK
283static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack)
284{
285 int i, j;
286
287 i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
288 for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j)
289 kp_stack[j] = (void *)track->stack_entries[i];
290 if (j < KS_ADDRS_COUNT)
291 kp_stack[j] = NULL;
292}
293
294bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
295{
296 struct kfence_metadata *meta = addr_to_metadata((unsigned long)object);
297 unsigned long flags;
298
299 if (!meta)
300 return false;
301
302 /*
303 * If state is UNUSED at least show the pointer requested; the rest
304 * would be garbage data.
305 */
306 kpp->kp_ptr = object;
307
308 /* Requesting info an a never-used object is almost certainly a bug. */
309 if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED))
310 return true;
311
312 raw_spin_lock_irqsave(&meta->lock, flags);
313
314 kpp->kp_slab = slab;
315 kpp->kp_slab_cache = meta->cache;
316 kpp->kp_objp = (void *)meta->addr;
317 kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack);
318 if (meta->state == KFENCE_OBJECT_FREED)
319 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack);
320 /* get_stack_skipnr() ensures the first entry is outside allocator. */
321 kpp->kp_ret = kpp->kp_stack[0];
322
323 raw_spin_unlock_irqrestore(&meta->lock, flags);
324
325 return true;
326}
327#endif