1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Kernel Electric-Fence (KFENCE). For more info please see
  4 * Documentation/dev-tools/kfence.rst.
  5 *
  6 * Copyright (C) 2020, Google LLC.
  7 */
  8
  9#ifndef MM_KFENCE_KFENCE_H
 10#define MM_KFENCE_KFENCE_H
 11
 12#include <linux/mm.h>
 13#include <linux/slab.h>
 14#include <linux/spinlock.h>
 15#include <linux/types.h>
 16
 17#include "../slab.h" /* for struct kmem_cache */
 18
 19/*
 20 * Get the canary byte pattern for @addr. Use a pattern that varies based on the
 21 * lower 3 bits of the address, to detect memory corruptions with higher
 22 * probability, where similar constants are used.
 23 */
 24#define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))
 25
 26/*
 27 * Define a continuous 8-byte canary starting from a multiple of 8. The canary
 28 * of each byte is only related to the lowest three bits of its address, so the
 29 * canary of every 8 bytes is the same. 64-bit memory can be filled and checked
 30 * at a time instead of byte by byte to improve performance.
 31 */
 32#define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(le64_to_cpu(0x0706050403020100)))
 33
 34/* Maximum stack depth for reports. */
 35#define KFENCE_STACK_DEPTH 64
 36
 37/* KFENCE object states. */
 38enum kfence_object_state {
 39	KFENCE_OBJECT_UNUSED,		/* Object is unused. */
 40	KFENCE_OBJECT_ALLOCATED,	/* Object is currently allocated. */
 41	KFENCE_OBJECT_FREED,		/* Object was allocated, and then freed. */
 42};
 43
 44/* Alloc/free tracking information. */
 45struct kfence_track {
 46	pid_t pid;
 47	int cpu;
 48	u64 ts_nsec;
 49	int num_stack_entries;
 50	unsigned long stack_entries[KFENCE_STACK_DEPTH];
 51};
 52
 53/* KFENCE metadata per guarded allocation. */
 54struct kfence_metadata {
 55	struct list_head list;		/* Freelist node; access under kfence_freelist_lock. */
 56	struct rcu_head rcu_head;	/* For delayed freeing. */
 57
 58	/*
 59	 * Lock protecting below data; to ensure consistency of the below data,
 60	 * since the following may execute concurrently: __kfence_alloc(),
 61	 * __kfence_free(), kfence_handle_page_fault(). However, note that we
 62	 * cannot grab the same metadata off the freelist twice, and multiple
 63	 * __kfence_alloc() cannot run concurrently on the same metadata.
 64	 */
 65	raw_spinlock_t lock;
 66
 67	/* The current state of the object; see above. */
 68	enum kfence_object_state state;
 69
 70	/*
 71	 * Allocated object address; cannot be calculated from size, because of
 72	 * alignment requirements.
 73	 *
 74	 * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
 75	 */
 76	unsigned long addr;
 77
 78	/*
 79	 * The size of the original allocation.
 80	 */
 81	size_t size;
 82
 83	/*
 84	 * The kmem_cache cache of the last allocation; NULL if never allocated
 85	 * or the cache has already been destroyed.
 86	 */
 87	struct kmem_cache *cache;
 88
 89	/*
 90	 * In case of an invalid access, the page that was unprotected; we
 91	 * optimistically only store one address.
 92	 */
 93	unsigned long unprotected_page;
 94
 95	/* Allocation and free stack information. */
 96	struct kfence_track alloc_track;
 97	struct kfence_track free_track;
 98	/* For updating alloc_covered on frees. */
 99	u32 alloc_stack_hash;
100#ifdef CONFIG_MEMCG
101	struct obj_cgroup *objcg;
102#endif
103};
104
105#define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \
106					CONFIG_KFENCE_NUM_OBJECTS)
107
108extern struct kfence_metadata *kfence_metadata;
109
110static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
111{
112	long index;
113
114	/* The checks do not affect performance; only called from slow-paths. */
115
116	if (!is_kfence_address((void *)addr))
117		return NULL;
118
119	/*
120	 * May be an invalid index if called with an address at the edge of
121	 * __kfence_pool, in which case we would report an "invalid access"
122	 * error.
123	 */
124	index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
125	if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
126		return NULL;
127
128	return &kfence_metadata[index];
129}
130
131/* KFENCE error types for report generation. */
132enum kfence_error_type {
133	KFENCE_ERROR_OOB,		/* Detected a out-of-bounds access. */
134	KFENCE_ERROR_UAF,		/* Detected a use-after-free access. */
135	KFENCE_ERROR_CORRUPTION,	/* Detected a memory corruption on free. */
136	KFENCE_ERROR_INVALID,		/* Invalid access of unknown type. */
137	KFENCE_ERROR_INVALID_FREE,	/* Invalid free. */
138};
139
140void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
141			 const struct kfence_metadata *meta, enum kfence_error_type type);
142
143void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta);
144
145#endif /* MM_KFENCE_KFENCE_H */