Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * shadow.c - Shadow Variables
4 *
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
8 */
9
10/**
11 * DOC: Shadow variable API concurrency notes:
12 *
13 * The shadow variable API provides a simple relationship between an
14 * <obj, id> pair and a pointer value. It is the responsibility of the
15 * caller to provide any mutual exclusion required of the shadow data.
16 *
17 * Once a shadow variable is attached to its parent object via the
18 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19 * call to klp_shadow_get() may then return the shadow variable's data
20 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
21 * accordingly.
22 *
23 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24 * variable structures. Their implementation does not call kmalloc
25 * inside any spinlocks, but API callers should pass GFP flags according
26 * to their specific needs.
27 *
28 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29 * concurrent klp_shadow_free() and klp_shadow_get() operations.
30 */
31
32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34#include <linux/hashtable.h>
35#include <linux/slab.h>
36#include <linux/livepatch.h>
37
38static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39
40/*
41 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42 * the shadow variables it references.
43 */
44static DEFINE_SPINLOCK(klp_shadow_lock);
45
46/**
47 * struct klp_shadow - shadow variable structure
48 * @node: klp_shadow_hash hash table node
49 * @rcu_head: RCU is used to safely free this structure
50 * @obj: pointer to parent object
51 * @id: data identifier
52 * @data: data area
53 */
54struct klp_shadow {
55 struct hlist_node node;
56 struct rcu_head rcu_head;
57 void *obj;
58 unsigned long id;
59 char data[];
60};
61
62/**
63 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64 * @shadow: shadow variable to match
65 * @obj: pointer to parent object
66 * @id: data identifier
67 *
68 * Return: true if the shadow variable matches.
69 */
70static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 unsigned long id)
72{
73 return shadow->obj == obj && shadow->id == id;
74}
75
76/**
77 * klp_shadow_get() - retrieve a shadow variable data pointer
78 * @obj: pointer to parent object
79 * @id: data identifier
80 *
81 * Return: the shadow variable data element, NULL on failure.
82 */
83void *klp_shadow_get(void *obj, unsigned long id)
84{
85 struct klp_shadow *shadow;
86
87 rcu_read_lock();
88
89 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 (unsigned long)obj) {
91
92 if (klp_shadow_match(shadow, obj, id)) {
93 rcu_read_unlock();
94 return shadow->data;
95 }
96 }
97
98 rcu_read_unlock();
99
100 return NULL;
101}
102EXPORT_SYMBOL_GPL(klp_shadow_get);
103
104static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
105 size_t size, gfp_t gfp_flags,
106 klp_shadow_ctor_t ctor, void *ctor_data,
107 bool warn_on_exist)
108{
109 struct klp_shadow *new_shadow;
110 void *shadow_data;
111 unsigned long flags;
112
113 /* Check if the shadow variable already exists */
114 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data)
116 goto exists;
117
118 /*
119 * Allocate a new shadow variable. Fill it with zeroes by default.
120 * More complex setting can be done by @ctor function. But it is
121 * called only when the buffer is really used (under klp_shadow_lock).
122 */
123 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 if (!new_shadow)
125 return NULL;
126
127 /* Look for <obj, id> again under the lock */
128 spin_lock_irqsave(&klp_shadow_lock, flags);
129 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) {
131 /*
132 * Shadow variable was found, throw away speculative
133 * allocation.
134 */
135 spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 kfree(new_shadow);
137 goto exists;
138 }
139
140 new_shadow->obj = obj;
141 new_shadow->id = id;
142
143 if (ctor) {
144 int err;
145
146 err = ctor(obj, new_shadow->data, ctor_data);
147 if (err) {
148 spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 kfree(new_shadow);
150 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
151 obj, id, err);
152 return NULL;
153 }
154 }
155
156 /* No <obj, id> found, so attach the newly allocated one */
157 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 (unsigned long)new_shadow->obj);
159 spin_unlock_irqrestore(&klp_shadow_lock, flags);
160
161 return new_shadow->data;
162
163exists:
164 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 return NULL;
167 }
168
169 return shadow_data;
170}
171
172/**
173 * klp_shadow_alloc() - allocate and add a new shadow variable
174 * @obj: pointer to parent object
175 * @id: data identifier
176 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initialize the shadow data (optional)
179 * @ctor_data: pointer to any data needed by @ctor (optional)
180 *
181 * Allocates @size bytes for new shadow variable data using @gfp_flags.
182 * The data are zeroed by default. They are further initialized by @ctor
183 * function if it is not NULL. The new shadow variable is then added
184 * to the global hashtable.
185 *
186 * If an existing <obj, id> shadow variable can be found, this routine will
187 * issue a WARN, exit early and return NULL.
188 *
189 * This function guarantees that the constructor function is called only when
190 * the variable did not exist before. The cost is that @ctor is called
191 * in atomic context under a spin lock.
192 *
193 * Return: the shadow variable data element, NULL on duplicate or
194 * failure.
195 */
196void *klp_shadow_alloc(void *obj, unsigned long id,
197 size_t size, gfp_t gfp_flags,
198 klp_shadow_ctor_t ctor, void *ctor_data)
199{
200 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
201 ctor, ctor_data, true);
202}
203EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204
205/**
206 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207 * @obj: pointer to parent object
208 * @id: data identifier
209 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initialize the shadow data (optional)
212 * @ctor_data: pointer to any data needed by @ctor (optional)
213 *
214 * Returns a pointer to existing shadow data if an <obj, id> shadow
215 * variable is already present. Otherwise, it creates a new shadow
216 * variable like klp_shadow_alloc().
217 *
218 * This function guarantees that only one shadow variable exists with the given
219 * @id for the given @obj. It also guarantees that the constructor function
220 * will be called only when the variable did not exist before. The cost is
221 * that @ctor is called in atomic context under a spin lock.
222 *
223 * Return: the shadow variable data element, NULL on failure.
224 */
225void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
226 size_t size, gfp_t gfp_flags,
227 klp_shadow_ctor_t ctor, void *ctor_data)
228{
229 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
230 ctor, ctor_data, false);
231}
232EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233
234static void klp_shadow_free_struct(struct klp_shadow *shadow,
235 klp_shadow_dtor_t dtor)
236{
237 hash_del_rcu(&shadow->node);
238 if (dtor)
239 dtor(shadow->obj, shadow->data);
240 kfree_rcu(shadow, rcu_head);
241}
242
243/**
244 * klp_shadow_free() - detach and free a <obj, id> shadow variable
245 * @obj: pointer to parent object
246 * @id: data identifier
247 * @dtor: custom callback that can be used to unregister the variable
248 * and/or free data that the shadow variable points to (optional)
249 *
250 * This function releases the memory for this <obj, id> shadow variable
251 * instance, callers should stop referencing it accordingly.
252 */
253void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
254{
255 struct klp_shadow *shadow;
256 unsigned long flags;
257
258 spin_lock_irqsave(&klp_shadow_lock, flags);
259
260 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash, shadow, node,
262 (unsigned long)obj) {
263
264 if (klp_shadow_match(shadow, obj, id)) {
265 klp_shadow_free_struct(shadow, dtor);
266 break;
267 }
268 }
269
270 spin_unlock_irqrestore(&klp_shadow_lock, flags);
271}
272EXPORT_SYMBOL_GPL(klp_shadow_free);
273
274/**
275 * klp_shadow_free_all() - detach and free all <_, id> shadow variables
276 * @id: data identifier
277 * @dtor: custom callback that can be used to unregister the variable
278 * and/or free data that the shadow variable points to (optional)
279 *
280 * This function releases the memory for all <_, id> shadow variable
281 * instances, callers should stop referencing them accordingly.
282 */
283void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
284{
285 struct klp_shadow *shadow;
286 unsigned long flags;
287 int i;
288
289 spin_lock_irqsave(&klp_shadow_lock, flags);
290
291 /* Delete all <_, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shadow, node) {
293 if (klp_shadow_match(shadow, shadow->obj, id))
294 klp_shadow_free_struct(shadow, dtor);
295 }
296
297 spin_unlock_irqrestore(&klp_shadow_lock, flags);
298}
299EXPORT_SYMBOL_GPL(klp_shadow_free_all);
1/*
2 * shadow.c - Shadow Variables
3 *
4 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
5 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
6 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version 2
11 * of the License, or (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
22/**
23 * DOC: Shadow variable API concurrency notes:
24 *
25 * The shadow variable API provides a simple relationship between an
26 * <obj, id> pair and a pointer value. It is the responsibility of the
27 * caller to provide any mutual exclusion required of the shadow data.
28 *
29 * Once a shadow variable is attached to its parent object via the
30 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
31 * call to klp_shadow_get() may then return the shadow variable's data
32 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
33 * accordingly.
34 *
35 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
36 * variable structures. Their implementation does not call kmalloc
37 * inside any spinlocks, but API callers should pass GFP flags according
38 * to their specific needs.
39 *
40 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
41 * concurrent klp_shadow_free() and klp_shadow_get() operations.
42 */
43
44#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
45
46#include <linux/hashtable.h>
47#include <linux/slab.h>
48#include <linux/livepatch.h>
49
50static DEFINE_HASHTABLE(klp_shadow_hash, 12);
51
52/*
53 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
54 * the shadow variables it references.
55 */
56static DEFINE_SPINLOCK(klp_shadow_lock);
57
58/**
59 * struct klp_shadow - shadow variable structure
60 * @node: klp_shadow_hash hash table node
61 * @rcu_head: RCU is used to safely free this structure
62 * @obj: pointer to parent object
63 * @id: data identifier
64 * @data: data area
65 */
66struct klp_shadow {
67 struct hlist_node node;
68 struct rcu_head rcu_head;
69 void *obj;
70 unsigned long id;
71 char data[];
72};
73
74/**
75 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
76 * @shadow: shadow variable to match
77 * @obj: pointer to parent object
78 * @id: data identifier
79 *
80 * Return: true if the shadow variable matches.
81 */
82static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
83 unsigned long id)
84{
85 return shadow->obj == obj && shadow->id == id;
86}
87
88/**
89 * klp_shadow_get() - retrieve a shadow variable data pointer
90 * @obj: pointer to parent object
91 * @id: data identifier
92 *
93 * Return: the shadow variable data element, NULL on failure.
94 */
95void *klp_shadow_get(void *obj, unsigned long id)
96{
97 struct klp_shadow *shadow;
98
99 rcu_read_lock();
100
101 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
102 (unsigned long)obj) {
103
104 if (klp_shadow_match(shadow, obj, id)) {
105 rcu_read_unlock();
106 return shadow->data;
107 }
108 }
109
110 rcu_read_unlock();
111
112 return NULL;
113}
114EXPORT_SYMBOL_GPL(klp_shadow_get);
115
116static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
117 size_t size, gfp_t gfp_flags,
118 klp_shadow_ctor_t ctor, void *ctor_data,
119 bool warn_on_exist)
120{
121 struct klp_shadow *new_shadow;
122 void *shadow_data;
123 unsigned long flags;
124
125 /* Check if the shadow variable already exists */
126 shadow_data = klp_shadow_get(obj, id);
127 if (shadow_data)
128 goto exists;
129
130 /*
131 * Allocate a new shadow variable. Fill it with zeroes by default.
132 * More complex setting can be done by @ctor function. But it is
133 * called only when the buffer is really used (under klp_shadow_lock).
134 */
135 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
136 if (!new_shadow)
137 return NULL;
138
139 /* Look for <obj, id> again under the lock */
140 spin_lock_irqsave(&klp_shadow_lock, flags);
141 shadow_data = klp_shadow_get(obj, id);
142 if (unlikely(shadow_data)) {
143 /*
144 * Shadow variable was found, throw away speculative
145 * allocation.
146 */
147 spin_unlock_irqrestore(&klp_shadow_lock, flags);
148 kfree(new_shadow);
149 goto exists;
150 }
151
152 new_shadow->obj = obj;
153 new_shadow->id = id;
154
155 if (ctor) {
156 int err;
157
158 err = ctor(obj, new_shadow->data, ctor_data);
159 if (err) {
160 spin_unlock_irqrestore(&klp_shadow_lock, flags);
161 kfree(new_shadow);
162 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
163 obj, id, err);
164 return NULL;
165 }
166 }
167
168 /* No <obj, id> found, so attach the newly allocated one */
169 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
170 (unsigned long)new_shadow->obj);
171 spin_unlock_irqrestore(&klp_shadow_lock, flags);
172
173 return new_shadow->data;
174
175exists:
176 if (warn_on_exist) {
177 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
178 return NULL;
179 }
180
181 return shadow_data;
182}
183
184/**
185 * klp_shadow_alloc() - allocate and add a new shadow variable
186 * @obj: pointer to parent object
187 * @id: data identifier
188 * @size: size of attached data
189 * @gfp_flags: GFP mask for allocation
190 * @ctor: custom constructor to initialize the shadow data (optional)
191 * @ctor_data: pointer to any data needed by @ctor (optional)
192 *
193 * Allocates @size bytes for new shadow variable data using @gfp_flags.
194 * The data are zeroed by default. They are further initialized by @ctor
195 * function if it is not NULL. The new shadow variable is then added
196 * to the global hashtable.
197 *
198 * If an existing <obj, id> shadow variable can be found, this routine will
199 * issue a WARN, exit early and return NULL.
200 *
201 * This function guarantees that the constructor function is called only when
202 * the variable did not exist before. The cost is that @ctor is called
203 * in atomic context under a spin lock.
204 *
205 * Return: the shadow variable data element, NULL on duplicate or
206 * failure.
207 */
208void *klp_shadow_alloc(void *obj, unsigned long id,
209 size_t size, gfp_t gfp_flags,
210 klp_shadow_ctor_t ctor, void *ctor_data)
211{
212 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
213 ctor, ctor_data, true);
214}
215EXPORT_SYMBOL_GPL(klp_shadow_alloc);
216
217/**
218 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
219 * @obj: pointer to parent object
220 * @id: data identifier
221 * @size: size of attached data
222 * @gfp_flags: GFP mask for allocation
223 * @ctor: custom constructor to initialize the shadow data (optional)
224 * @ctor_data: pointer to any data needed by @ctor (optional)
225 *
226 * Returns a pointer to existing shadow data if an <obj, id> shadow
227 * variable is already present. Otherwise, it creates a new shadow
228 * variable like klp_shadow_alloc().
229 *
230 * This function guarantees that only one shadow variable exists with the given
231 * @id for the given @obj. It also guarantees that the constructor function
232 * will be called only when the variable did not exist before. The cost is
233 * that @ctor is called in atomic context under a spin lock.
234 *
235 * Return: the shadow variable data element, NULL on failure.
236 */
237void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
238 size_t size, gfp_t gfp_flags,
239 klp_shadow_ctor_t ctor, void *ctor_data)
240{
241 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
242 ctor, ctor_data, false);
243}
244EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
245
246static void klp_shadow_free_struct(struct klp_shadow *shadow,
247 klp_shadow_dtor_t dtor)
248{
249 hash_del_rcu(&shadow->node);
250 if (dtor)
251 dtor(shadow->obj, shadow->data);
252 kfree_rcu(shadow, rcu_head);
253}
254
255/**
256 * klp_shadow_free() - detach and free a <obj, id> shadow variable
257 * @obj: pointer to parent object
258 * @id: data identifier
259 * @dtor: custom callback that can be used to unregister the variable
260 * and/or free data that the shadow variable points to (optional)
261 *
262 * This function releases the memory for this <obj, id> shadow variable
263 * instance, callers should stop referencing it accordingly.
264 */
265void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
266{
267 struct klp_shadow *shadow;
268 unsigned long flags;
269
270 spin_lock_irqsave(&klp_shadow_lock, flags);
271
272 /* Delete <obj, id> from hash */
273 hash_for_each_possible(klp_shadow_hash, shadow, node,
274 (unsigned long)obj) {
275
276 if (klp_shadow_match(shadow, obj, id)) {
277 klp_shadow_free_struct(shadow, dtor);
278 break;
279 }
280 }
281
282 spin_unlock_irqrestore(&klp_shadow_lock, flags);
283}
284EXPORT_SYMBOL_GPL(klp_shadow_free);
285
286/**
287 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
288 * @id: data identifier
289 * @dtor: custom callback that can be used to unregister the variable
290 * and/or free data that the shadow variable points to (optional)
291 *
292 * This function releases the memory for all <*, id> shadow variable
293 * instances, callers should stop referencing them accordingly.
294 */
295void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
296{
297 struct klp_shadow *shadow;
298 unsigned long flags;
299 int i;
300
301 spin_lock_irqsave(&klp_shadow_lock, flags);
302
303 /* Delete all <*, id> from hash */
304 hash_for_each(klp_shadow_hash, i, shadow, node) {
305 if (klp_shadow_match(shadow, shadow->obj, id))
306 klp_shadow_free_struct(shadow, dtor);
307 }
308
309 spin_unlock_irqrestore(&klp_shadow_lock, flags);
310}
311EXPORT_SYMBOL_GPL(klp_shadow_free_all);