Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8#include <linux/reboot.h>
9
10/*
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
13 * and the like.
14 */
15BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16
17/*
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
20 */
21
22static int notifier_chain_register(struct notifier_block **nl,
23 struct notifier_block *n)
24{
25 while ((*nl) != NULL) {
26 WARN_ONCE(((*nl) == n), "double register detected");
27 if (n->priority > (*nl)->priority)
28 break;
29 nl = &((*nl)->next);
30 }
31 n->next = *nl;
32 rcu_assign_pointer(*nl, n);
33 return 0;
34}
35
36static int notifier_chain_cond_register(struct notifier_block **nl,
37 struct notifier_block *n)
38{
39 while ((*nl) != NULL) {
40 if ((*nl) == n)
41 return 0;
42 if (n->priority > (*nl)->priority)
43 break;
44 nl = &((*nl)->next);
45 }
46 n->next = *nl;
47 rcu_assign_pointer(*nl, n);
48 return 0;
49}
50
51static int notifier_chain_unregister(struct notifier_block **nl,
52 struct notifier_block *n)
53{
54 while ((*nl) != NULL) {
55 if ((*nl) == n) {
56 rcu_assign_pointer(*nl, n->next);
57 return 0;
58 }
59 nl = &((*nl)->next);
60 }
61 return -ENOENT;
62}
63
64/**
65 * notifier_call_chain - Informs the registered notifiers about an event.
66 * @nl: Pointer to head of the blocking notifier chain
67 * @val: Value passed unmodified to notifier function
68 * @v: Pointer passed unmodified to notifier function
69 * @nr_to_call: Number of notifier functions to be called. Don't care
70 * value of this parameter is -1.
71 * @nr_calls: Records the number of notifications sent. Don't care
72 * value of this field is NULL.
73 * @returns: notifier_call_chain returns the value returned by the
74 * last notifier function called.
75 */
76static int notifier_call_chain(struct notifier_block **nl,
77 unsigned long val, void *v,
78 int nr_to_call, int *nr_calls)
79{
80 int ret = NOTIFY_DONE;
81 struct notifier_block *nb, *next_nb;
82
83 nb = rcu_dereference_raw(*nl);
84
85 while (nb && nr_to_call) {
86 next_nb = rcu_dereference_raw(nb->next);
87
88#ifdef CONFIG_DEBUG_NOTIFIERS
89 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
90 WARN(1, "Invalid notifier called!");
91 nb = next_nb;
92 continue;
93 }
94#endif
95 ret = nb->notifier_call(nb, val, v);
96
97 if (nr_calls)
98 (*nr_calls)++;
99
100 if (ret & NOTIFY_STOP_MASK)
101 break;
102 nb = next_nb;
103 nr_to_call--;
104 }
105 return ret;
106}
107NOKPROBE_SYMBOL(notifier_call_chain);
108
109/*
110 * Atomic notifier chain routines. Registration and unregistration
111 * use a spinlock, and call_chain is synchronized by RCU (no locks).
112 */
113
114/**
115 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
116 * @nh: Pointer to head of the atomic notifier chain
117 * @n: New entry in notifier chain
118 *
119 * Adds a notifier to an atomic notifier chain.
120 *
121 * Currently always returns zero.
122 */
123int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
124 struct notifier_block *n)
125{
126 unsigned long flags;
127 int ret;
128
129 spin_lock_irqsave(&nh->lock, flags);
130 ret = notifier_chain_register(&nh->head, n);
131 spin_unlock_irqrestore(&nh->lock, flags);
132 return ret;
133}
134EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
135
136/**
137 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
138 * @nh: Pointer to head of the atomic notifier chain
139 * @n: Entry to remove from notifier chain
140 *
141 * Removes a notifier from an atomic notifier chain.
142 *
143 * Returns zero on success or %-ENOENT on failure.
144 */
145int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
146 struct notifier_block *n)
147{
148 unsigned long flags;
149 int ret;
150
151 spin_lock_irqsave(&nh->lock, flags);
152 ret = notifier_chain_unregister(&nh->head, n);
153 spin_unlock_irqrestore(&nh->lock, flags);
154 synchronize_rcu();
155 return ret;
156}
157EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
158
159/**
160 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
161 * @nh: Pointer to head of the atomic notifier chain
162 * @val: Value passed unmodified to notifier function
163 * @v: Pointer passed unmodified to notifier function
164 * @nr_to_call: See the comment for notifier_call_chain.
165 * @nr_calls: See the comment for notifier_call_chain.
166 *
167 * Calls each function in a notifier chain in turn. The functions
168 * run in an atomic context, so they must not block.
169 * This routine uses RCU to synchronize with changes to the chain.
170 *
171 * If the return value of the notifier can be and'ed
172 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
173 * will return immediately, with the return value of
174 * the notifier function which halted execution.
175 * Otherwise the return value is the return value
176 * of the last notifier function called.
177 */
178int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
179 unsigned long val, void *v,
180 int nr_to_call, int *nr_calls)
181{
182 int ret;
183
184 rcu_read_lock();
185 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
186 rcu_read_unlock();
187 return ret;
188}
189EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
190NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
191
192int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
193 unsigned long val, void *v)
194{
195 return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
196}
197EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
198NOKPROBE_SYMBOL(atomic_notifier_call_chain);
199
200/*
201 * Blocking notifier chain routines. All access to the chain is
202 * synchronized by an rwsem.
203 */
204
205/**
206 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
207 * @nh: Pointer to head of the blocking notifier chain
208 * @n: New entry in notifier chain
209 *
210 * Adds a notifier to a blocking notifier chain.
211 * Must be called in process context.
212 *
213 * Currently always returns zero.
214 */
215int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
216 struct notifier_block *n)
217{
218 int ret;
219
220 /*
221 * This code gets used during boot-up, when task switching is
222 * not yet working and interrupts must remain disabled. At
223 * such times we must not call down_write().
224 */
225 if (unlikely(system_state == SYSTEM_BOOTING))
226 return notifier_chain_register(&nh->head, n);
227
228 down_write(&nh->rwsem);
229 ret = notifier_chain_register(&nh->head, n);
230 up_write(&nh->rwsem);
231 return ret;
232}
233EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
234
235/**
236 * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain
237 * @nh: Pointer to head of the blocking notifier chain
238 * @n: New entry in notifier chain
239 *
240 * Adds a notifier to a blocking notifier chain, only if not already
241 * present in the chain.
242 * Must be called in process context.
243 *
244 * Currently always returns zero.
245 */
246int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh,
247 struct notifier_block *n)
248{
249 int ret;
250
251 down_write(&nh->rwsem);
252 ret = notifier_chain_cond_register(&nh->head, n);
253 up_write(&nh->rwsem);
254 return ret;
255}
256EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register);
257
258/**
259 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
260 * @nh: Pointer to head of the blocking notifier chain
261 * @n: Entry to remove from notifier chain
262 *
263 * Removes a notifier from a blocking notifier chain.
264 * Must be called from process context.
265 *
266 * Returns zero on success or %-ENOENT on failure.
267 */
268int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
269 struct notifier_block *n)
270{
271 int ret;
272
273 /*
274 * This code gets used during boot-up, when task switching is
275 * not yet working and interrupts must remain disabled. At
276 * such times we must not call down_write().
277 */
278 if (unlikely(system_state == SYSTEM_BOOTING))
279 return notifier_chain_unregister(&nh->head, n);
280
281 down_write(&nh->rwsem);
282 ret = notifier_chain_unregister(&nh->head, n);
283 up_write(&nh->rwsem);
284 return ret;
285}
286EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
287
288/**
289 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
290 * @nh: Pointer to head of the blocking notifier chain
291 * @val: Value passed unmodified to notifier function
292 * @v: Pointer passed unmodified to notifier function
293 * @nr_to_call: See comment for notifier_call_chain.
294 * @nr_calls: See comment for notifier_call_chain.
295 *
296 * Calls each function in a notifier chain in turn. The functions
297 * run in a process context, so they are allowed to block.
298 *
299 * If the return value of the notifier can be and'ed
300 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
301 * will return immediately, with the return value of
302 * the notifier function which halted execution.
303 * Otherwise the return value is the return value
304 * of the last notifier function called.
305 */
306int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
307 unsigned long val, void *v,
308 int nr_to_call, int *nr_calls)
309{
310 int ret = NOTIFY_DONE;
311
312 /*
313 * We check the head outside the lock, but if this access is
314 * racy then it does not matter what the result of the test
315 * is, we re-check the list after having taken the lock anyway:
316 */
317 if (rcu_access_pointer(nh->head)) {
318 down_read(&nh->rwsem);
319 ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
320 nr_calls);
321 up_read(&nh->rwsem);
322 }
323 return ret;
324}
325EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain);
326
327int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
328 unsigned long val, void *v)
329{
330 return __blocking_notifier_call_chain(nh, val, v, -1, NULL);
331}
332EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
333
334/*
335 * Raw notifier chain routines. There is no protection;
336 * the caller must provide it. Use at your own risk!
337 */
338
339/**
340 * raw_notifier_chain_register - Add notifier to a raw notifier chain
341 * @nh: Pointer to head of the raw notifier chain
342 * @n: New entry in notifier chain
343 *
344 * Adds a notifier to a raw notifier chain.
345 * All locking must be provided by the caller.
346 *
347 * Currently always returns zero.
348 */
349int raw_notifier_chain_register(struct raw_notifier_head *nh,
350 struct notifier_block *n)
351{
352 return notifier_chain_register(&nh->head, n);
353}
354EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
355
356/**
357 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
358 * @nh: Pointer to head of the raw notifier chain
359 * @n: Entry to remove from notifier chain
360 *
361 * Removes a notifier from a raw notifier chain.
362 * All locking must be provided by the caller.
363 *
364 * Returns zero on success or %-ENOENT on failure.
365 */
366int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
367 struct notifier_block *n)
368{
369 return notifier_chain_unregister(&nh->head, n);
370}
371EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
372
373/**
374 * __raw_notifier_call_chain - Call functions in a raw notifier chain
375 * @nh: Pointer to head of the raw notifier chain
376 * @val: Value passed unmodified to notifier function
377 * @v: Pointer passed unmodified to notifier function
378 * @nr_to_call: See comment for notifier_call_chain.
379 * @nr_calls: See comment for notifier_call_chain
380 *
381 * Calls each function in a notifier chain in turn. The functions
382 * run in an undefined context.
383 * All locking must be provided by the caller.
384 *
385 * If the return value of the notifier can be and'ed
386 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
387 * will return immediately, with the return value of
388 * the notifier function which halted execution.
389 * Otherwise the return value is the return value
390 * of the last notifier function called.
391 */
392int __raw_notifier_call_chain(struct raw_notifier_head *nh,
393 unsigned long val, void *v,
394 int nr_to_call, int *nr_calls)
395{
396 return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
397}
398EXPORT_SYMBOL_GPL(__raw_notifier_call_chain);
399
400int raw_notifier_call_chain(struct raw_notifier_head *nh,
401 unsigned long val, void *v)
402{
403 return __raw_notifier_call_chain(nh, val, v, -1, NULL);
404}
405EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
406
407#ifdef CONFIG_SRCU
408/*
409 * SRCU notifier chain routines. Registration and unregistration
410 * use a mutex, and call_chain is synchronized by SRCU (no locks).
411 */
412
413/**
414 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
415 * @nh: Pointer to head of the SRCU notifier chain
416 * @n: New entry in notifier chain
417 *
418 * Adds a notifier to an SRCU notifier chain.
419 * Must be called in process context.
420 *
421 * Currently always returns zero.
422 */
423int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
424 struct notifier_block *n)
425{
426 int ret;
427
428 /*
429 * This code gets used during boot-up, when task switching is
430 * not yet working and interrupts must remain disabled. At
431 * such times we must not call mutex_lock().
432 */
433 if (unlikely(system_state == SYSTEM_BOOTING))
434 return notifier_chain_register(&nh->head, n);
435
436 mutex_lock(&nh->mutex);
437 ret = notifier_chain_register(&nh->head, n);
438 mutex_unlock(&nh->mutex);
439 return ret;
440}
441EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
442
443/**
444 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
445 * @nh: Pointer to head of the SRCU notifier chain
446 * @n: Entry to remove from notifier chain
447 *
448 * Removes a notifier from an SRCU notifier chain.
449 * Must be called from process context.
450 *
451 * Returns zero on success or %-ENOENT on failure.
452 */
453int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
454 struct notifier_block *n)
455{
456 int ret;
457
458 /*
459 * This code gets used during boot-up, when task switching is
460 * not yet working and interrupts must remain disabled. At
461 * such times we must not call mutex_lock().
462 */
463 if (unlikely(system_state == SYSTEM_BOOTING))
464 return notifier_chain_unregister(&nh->head, n);
465
466 mutex_lock(&nh->mutex);
467 ret = notifier_chain_unregister(&nh->head, n);
468 mutex_unlock(&nh->mutex);
469 synchronize_srcu(&nh->srcu);
470 return ret;
471}
472EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
473
474/**
475 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
476 * @nh: Pointer to head of the SRCU notifier chain
477 * @val: Value passed unmodified to notifier function
478 * @v: Pointer passed unmodified to notifier function
479 * @nr_to_call: See comment for notifier_call_chain.
480 * @nr_calls: See comment for notifier_call_chain
481 *
482 * Calls each function in a notifier chain in turn. The functions
483 * run in a process context, so they are allowed to block.
484 *
485 * If the return value of the notifier can be and'ed
486 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
487 * will return immediately, with the return value of
488 * the notifier function which halted execution.
489 * Otherwise the return value is the return value
490 * of the last notifier function called.
491 */
492int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
493 unsigned long val, void *v,
494 int nr_to_call, int *nr_calls)
495{
496 int ret;
497 int idx;
498
499 idx = srcu_read_lock(&nh->srcu);
500 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
501 srcu_read_unlock(&nh->srcu, idx);
502 return ret;
503}
504EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain);
505
506int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
507 unsigned long val, void *v)
508{
509 return __srcu_notifier_call_chain(nh, val, v, -1, NULL);
510}
511EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
512
513/**
514 * srcu_init_notifier_head - Initialize an SRCU notifier head
515 * @nh: Pointer to head of the srcu notifier chain
516 *
517 * Unlike other sorts of notifier heads, SRCU notifier heads require
518 * dynamic initialization. Be sure to call this routine before
519 * calling any of the other SRCU notifier routines for this head.
520 *
521 * If an SRCU notifier head is deallocated, it must first be cleaned
522 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
523 * per-cpu data (used by the SRCU mechanism) will leak.
524 */
525void srcu_init_notifier_head(struct srcu_notifier_head *nh)
526{
527 mutex_init(&nh->mutex);
528 if (init_srcu_struct(&nh->srcu) < 0)
529 BUG();
530 nh->head = NULL;
531}
532EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
533
534#endif /* CONFIG_SRCU */
535
536static ATOMIC_NOTIFIER_HEAD(die_chain);
537
538int notrace notify_die(enum die_val val, const char *str,
539 struct pt_regs *regs, long err, int trap, int sig)
540{
541 struct die_args args = {
542 .regs = regs,
543 .str = str,
544 .err = err,
545 .trapnr = trap,
546 .signr = sig,
547
548 };
549 RCU_LOCKDEP_WARN(!rcu_is_watching(),
550 "notify_die called but RCU thinks we're quiescent");
551 return atomic_notifier_call_chain(&die_chain, val, &args);
552}
553NOKPROBE_SYMBOL(notify_die);
554
555int register_die_notifier(struct notifier_block *nb)
556{
557 vmalloc_sync_all();
558 return atomic_notifier_chain_register(&die_chain, nb);
559}
560EXPORT_SYMBOL_GPL(register_die_notifier);
561
562int unregister_die_notifier(struct notifier_block *nb)
563{
564 return atomic_notifier_chain_unregister(&die_chain, nb);
565}
566EXPORT_SYMBOL_GPL(unregister_die_notifier);
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8#include <linux/reboot.h>
9
10/*
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
13 * and the like.
14 */
15BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16
17/*
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
20 */
21
22static int notifier_chain_register(struct notifier_block **nl,
23 struct notifier_block *n)
24{
25 while ((*nl) != NULL) {
26 if (unlikely((*nl) == n)) {
27 WARN(1, "double register detected");
28 return 0;
29 }
30 if (n->priority > (*nl)->priority)
31 break;
32 nl = &((*nl)->next);
33 }
34 n->next = *nl;
35 rcu_assign_pointer(*nl, n);
36 return 0;
37}
38
39static int notifier_chain_unregister(struct notifier_block **nl,
40 struct notifier_block *n)
41{
42 while ((*nl) != NULL) {
43 if ((*nl) == n) {
44 rcu_assign_pointer(*nl, n->next);
45 return 0;
46 }
47 nl = &((*nl)->next);
48 }
49 return -ENOENT;
50}
51
52/**
53 * notifier_call_chain - Informs the registered notifiers about an event.
54 * @nl: Pointer to head of the blocking notifier chain
55 * @val: Value passed unmodified to notifier function
56 * @v: Pointer passed unmodified to notifier function
57 * @nr_to_call: Number of notifier functions to be called. Don't care
58 * value of this parameter is -1.
59 * @nr_calls: Records the number of notifications sent. Don't care
60 * value of this field is NULL.
61 * @returns: notifier_call_chain returns the value returned by the
62 * last notifier function called.
63 */
64static int notifier_call_chain(struct notifier_block **nl,
65 unsigned long val, void *v,
66 int nr_to_call, int *nr_calls)
67{
68 int ret = NOTIFY_DONE;
69 struct notifier_block *nb, *next_nb;
70
71 nb = rcu_dereference_raw(*nl);
72
73 while (nb && nr_to_call) {
74 next_nb = rcu_dereference_raw(nb->next);
75
76#ifdef CONFIG_DEBUG_NOTIFIERS
77 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
78 WARN(1, "Invalid notifier called!");
79 nb = next_nb;
80 continue;
81 }
82#endif
83 ret = nb->notifier_call(nb, val, v);
84
85 if (nr_calls)
86 (*nr_calls)++;
87
88 if (ret & NOTIFY_STOP_MASK)
89 break;
90 nb = next_nb;
91 nr_to_call--;
92 }
93 return ret;
94}
95NOKPROBE_SYMBOL(notifier_call_chain);
96
97/**
98 * notifier_call_chain_robust - Inform the registered notifiers about an event
99 * and rollback on error.
100 * @nl: Pointer to head of the blocking notifier chain
101 * @val_up: Value passed unmodified to the notifier function
102 * @val_down: Value passed unmodified to the notifier function when recovering
103 * from an error on @val_up
104 * @v Pointer passed unmodified to the notifier function
105 *
106 * NOTE: It is important the @nl chain doesn't change between the two
107 * invocations of notifier_call_chain() such that we visit the
108 * exact same notifier callbacks; this rules out any RCU usage.
109 *
110 * Returns: the return value of the @val_up call.
111 */
112static int notifier_call_chain_robust(struct notifier_block **nl,
113 unsigned long val_up, unsigned long val_down,
114 void *v)
115{
116 int ret, nr = 0;
117
118 ret = notifier_call_chain(nl, val_up, v, -1, &nr);
119 if (ret & NOTIFY_STOP_MASK)
120 notifier_call_chain(nl, val_down, v, nr-1, NULL);
121
122 return ret;
123}
124
125/*
126 * Atomic notifier chain routines. Registration and unregistration
127 * use a spinlock, and call_chain is synchronized by RCU (no locks).
128 */
129
130/**
131 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
132 * @nh: Pointer to head of the atomic notifier chain
133 * @n: New entry in notifier chain
134 *
135 * Adds a notifier to an atomic notifier chain.
136 *
137 * Currently always returns zero.
138 */
139int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
140 struct notifier_block *n)
141{
142 unsigned long flags;
143 int ret;
144
145 spin_lock_irqsave(&nh->lock, flags);
146 ret = notifier_chain_register(&nh->head, n);
147 spin_unlock_irqrestore(&nh->lock, flags);
148 return ret;
149}
150EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
151
152/**
153 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
154 * @nh: Pointer to head of the atomic notifier chain
155 * @n: Entry to remove from notifier chain
156 *
157 * Removes a notifier from an atomic notifier chain.
158 *
159 * Returns zero on success or %-ENOENT on failure.
160 */
161int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
162 struct notifier_block *n)
163{
164 unsigned long flags;
165 int ret;
166
167 spin_lock_irqsave(&nh->lock, flags);
168 ret = notifier_chain_unregister(&nh->head, n);
169 spin_unlock_irqrestore(&nh->lock, flags);
170 synchronize_rcu();
171 return ret;
172}
173EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
174
175int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
176 unsigned long val_up, unsigned long val_down, void *v)
177{
178 unsigned long flags;
179 int ret;
180
181 /*
182 * Musn't use RCU; because then the notifier list can
183 * change between the up and down traversal.
184 */
185 spin_lock_irqsave(&nh->lock, flags);
186 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
187 spin_unlock_irqrestore(&nh->lock, flags);
188
189 return ret;
190}
191EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust);
192NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust);
193
194/**
195 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
196 * @nh: Pointer to head of the atomic notifier chain
197 * @val: Value passed unmodified to notifier function
198 * @v: Pointer passed unmodified to notifier function
199 *
200 * Calls each function in a notifier chain in turn. The functions
201 * run in an atomic context, so they must not block.
202 * This routine uses RCU to synchronize with changes to the chain.
203 *
204 * If the return value of the notifier can be and'ed
205 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
206 * will return immediately, with the return value of
207 * the notifier function which halted execution.
208 * Otherwise the return value is the return value
209 * of the last notifier function called.
210 */
211int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
212 unsigned long val, void *v)
213{
214 int ret;
215
216 rcu_read_lock();
217 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
218 rcu_read_unlock();
219
220 return ret;
221}
222EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
223NOKPROBE_SYMBOL(atomic_notifier_call_chain);
224
225/*
226 * Blocking notifier chain routines. All access to the chain is
227 * synchronized by an rwsem.
228 */
229
230/**
231 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
232 * @nh: Pointer to head of the blocking notifier chain
233 * @n: New entry in notifier chain
234 *
235 * Adds a notifier to a blocking notifier chain.
236 * Must be called in process context.
237 *
238 * Currently always returns zero.
239 */
240int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
241 struct notifier_block *n)
242{
243 int ret;
244
245 /*
246 * This code gets used during boot-up, when task switching is
247 * not yet working and interrupts must remain disabled. At
248 * such times we must not call down_write().
249 */
250 if (unlikely(system_state == SYSTEM_BOOTING))
251 return notifier_chain_register(&nh->head, n);
252
253 down_write(&nh->rwsem);
254 ret = notifier_chain_register(&nh->head, n);
255 up_write(&nh->rwsem);
256 return ret;
257}
258EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
259
260/**
261 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
262 * @nh: Pointer to head of the blocking notifier chain
263 * @n: Entry to remove from notifier chain
264 *
265 * Removes a notifier from a blocking notifier chain.
266 * Must be called from process context.
267 *
268 * Returns zero on success or %-ENOENT on failure.
269 */
270int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
271 struct notifier_block *n)
272{
273 int ret;
274
275 /*
276 * This code gets used during boot-up, when task switching is
277 * not yet working and interrupts must remain disabled. At
278 * such times we must not call down_write().
279 */
280 if (unlikely(system_state == SYSTEM_BOOTING))
281 return notifier_chain_unregister(&nh->head, n);
282
283 down_write(&nh->rwsem);
284 ret = notifier_chain_unregister(&nh->head, n);
285 up_write(&nh->rwsem);
286 return ret;
287}
288EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
289
290int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
291 unsigned long val_up, unsigned long val_down, void *v)
292{
293 int ret = NOTIFY_DONE;
294
295 /*
296 * We check the head outside the lock, but if this access is
297 * racy then it does not matter what the result of the test
298 * is, we re-check the list after having taken the lock anyway:
299 */
300 if (rcu_access_pointer(nh->head)) {
301 down_read(&nh->rwsem);
302 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
303 up_read(&nh->rwsem);
304 }
305 return ret;
306}
307EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
308
309/**
310 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
311 * @nh: Pointer to head of the blocking notifier chain
312 * @val: Value passed unmodified to notifier function
313 * @v: Pointer passed unmodified to notifier function
314 *
315 * Calls each function in a notifier chain in turn. The functions
316 * run in a process context, so they are allowed to block.
317 *
318 * If the return value of the notifier can be and'ed
319 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
320 * will return immediately, with the return value of
321 * the notifier function which halted execution.
322 * Otherwise the return value is the return value
323 * of the last notifier function called.
324 */
325int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
326 unsigned long val, void *v)
327{
328 int ret = NOTIFY_DONE;
329
330 /*
331 * We check the head outside the lock, but if this access is
332 * racy then it does not matter what the result of the test
333 * is, we re-check the list after having taken the lock anyway:
334 */
335 if (rcu_access_pointer(nh->head)) {
336 down_read(&nh->rwsem);
337 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
338 up_read(&nh->rwsem);
339 }
340 return ret;
341}
342EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
343
344/*
345 * Raw notifier chain routines. There is no protection;
346 * the caller must provide it. Use at your own risk!
347 */
348
349/**
350 * raw_notifier_chain_register - Add notifier to a raw notifier chain
351 * @nh: Pointer to head of the raw notifier chain
352 * @n: New entry in notifier chain
353 *
354 * Adds a notifier to a raw notifier chain.
355 * All locking must be provided by the caller.
356 *
357 * Currently always returns zero.
358 */
359int raw_notifier_chain_register(struct raw_notifier_head *nh,
360 struct notifier_block *n)
361{
362 return notifier_chain_register(&nh->head, n);
363}
364EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
365
366/**
367 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
368 * @nh: Pointer to head of the raw notifier chain
369 * @n: Entry to remove from notifier chain
370 *
371 * Removes a notifier from a raw notifier chain.
372 * All locking must be provided by the caller.
373 *
374 * Returns zero on success or %-ENOENT on failure.
375 */
376int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
377 struct notifier_block *n)
378{
379 return notifier_chain_unregister(&nh->head, n);
380}
381EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
382
383int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
384 unsigned long val_up, unsigned long val_down, void *v)
385{
386 return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
387}
388EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
389
390/**
391 * raw_notifier_call_chain - Call functions in a raw notifier chain
392 * @nh: Pointer to head of the raw notifier chain
393 * @val: Value passed unmodified to notifier function
394 * @v: Pointer passed unmodified to notifier function
395 *
396 * Calls each function in a notifier chain in turn. The functions
397 * run in an undefined context.
398 * All locking must be provided by the caller.
399 *
400 * If the return value of the notifier can be and'ed
401 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
402 * will return immediately, with the return value of
403 * the notifier function which halted execution.
404 * Otherwise the return value is the return value
405 * of the last notifier function called.
406 */
407int raw_notifier_call_chain(struct raw_notifier_head *nh,
408 unsigned long val, void *v)
409{
410 return notifier_call_chain(&nh->head, val, v, -1, NULL);
411}
412EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
413
414#ifdef CONFIG_SRCU
415/*
416 * SRCU notifier chain routines. Registration and unregistration
417 * use a mutex, and call_chain is synchronized by SRCU (no locks).
418 */
419
420/**
421 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
422 * @nh: Pointer to head of the SRCU notifier chain
423 * @n: New entry in notifier chain
424 *
425 * Adds a notifier to an SRCU notifier chain.
426 * Must be called in process context.
427 *
428 * Currently always returns zero.
429 */
430int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
431 struct notifier_block *n)
432{
433 int ret;
434
435 /*
436 * This code gets used during boot-up, when task switching is
437 * not yet working and interrupts must remain disabled. At
438 * such times we must not call mutex_lock().
439 */
440 if (unlikely(system_state == SYSTEM_BOOTING))
441 return notifier_chain_register(&nh->head, n);
442
443 mutex_lock(&nh->mutex);
444 ret = notifier_chain_register(&nh->head, n);
445 mutex_unlock(&nh->mutex);
446 return ret;
447}
448EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
449
450/**
451 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
452 * @nh: Pointer to head of the SRCU notifier chain
453 * @n: Entry to remove from notifier chain
454 *
455 * Removes a notifier from an SRCU notifier chain.
456 * Must be called from process context.
457 *
458 * Returns zero on success or %-ENOENT on failure.
459 */
460int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
461 struct notifier_block *n)
462{
463 int ret;
464
465 /*
466 * This code gets used during boot-up, when task switching is
467 * not yet working and interrupts must remain disabled. At
468 * such times we must not call mutex_lock().
469 */
470 if (unlikely(system_state == SYSTEM_BOOTING))
471 return notifier_chain_unregister(&nh->head, n);
472
473 mutex_lock(&nh->mutex);
474 ret = notifier_chain_unregister(&nh->head, n);
475 mutex_unlock(&nh->mutex);
476 synchronize_srcu(&nh->srcu);
477 return ret;
478}
479EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
480
481/**
482 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
483 * @nh: Pointer to head of the SRCU notifier chain
484 * @val: Value passed unmodified to notifier function
485 * @v: Pointer passed unmodified to notifier function
486 *
487 * Calls each function in a notifier chain in turn. The functions
488 * run in a process context, so they are allowed to block.
489 *
490 * If the return value of the notifier can be and'ed
491 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
492 * will return immediately, with the return value of
493 * the notifier function which halted execution.
494 * Otherwise the return value is the return value
495 * of the last notifier function called.
496 */
497int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
498 unsigned long val, void *v)
499{
500 int ret;
501 int idx;
502
503 idx = srcu_read_lock(&nh->srcu);
504 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
505 srcu_read_unlock(&nh->srcu, idx);
506 return ret;
507}
508EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
509
510/**
511 * srcu_init_notifier_head - Initialize an SRCU notifier head
512 * @nh: Pointer to head of the srcu notifier chain
513 *
514 * Unlike other sorts of notifier heads, SRCU notifier heads require
515 * dynamic initialization. Be sure to call this routine before
516 * calling any of the other SRCU notifier routines for this head.
517 *
518 * If an SRCU notifier head is deallocated, it must first be cleaned
519 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
520 * per-cpu data (used by the SRCU mechanism) will leak.
521 */
522void srcu_init_notifier_head(struct srcu_notifier_head *nh)
523{
524 mutex_init(&nh->mutex);
525 if (init_srcu_struct(&nh->srcu) < 0)
526 BUG();
527 nh->head = NULL;
528}
529EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
530
531#endif /* CONFIG_SRCU */
532
533static ATOMIC_NOTIFIER_HEAD(die_chain);
534
535int notrace notify_die(enum die_val val, const char *str,
536 struct pt_regs *regs, long err, int trap, int sig)
537{
538 struct die_args args = {
539 .regs = regs,
540 .str = str,
541 .err = err,
542 .trapnr = trap,
543 .signr = sig,
544
545 };
546 RCU_LOCKDEP_WARN(!rcu_is_watching(),
547 "notify_die called but RCU thinks we're quiescent");
548 return atomic_notifier_call_chain(&die_chain, val, &args);
549}
550NOKPROBE_SYMBOL(notify_die);
551
552int register_die_notifier(struct notifier_block *nb)
553{
554 return atomic_notifier_chain_register(&die_chain, nb);
555}
556EXPORT_SYMBOL_GPL(register_die_notifier);
557
558int unregister_die_notifier(struct notifier_block *nb)
559{
560 return atomic_notifier_chain_unregister(&die_chain, nb);
561}
562EXPORT_SYMBOL_GPL(unregister_die_notifier);