Loading...
1#include <linux/kdebug.h>
2#include <linux/kprobes.h>
3#include <linux/export.h>
4#include <linux/notifier.h>
5#include <linux/rcupdate.h>
6#include <linux/vmalloc.h>
7#include <linux/reboot.h>
8
9/*
10 * Notifier list for kernel code which wants to be called
11 * at shutdown. This is used to stop any idling DMA operations
12 * and the like.
13 */
14BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
15
16/*
17 * Notifier chain core routines. The exported routines below
18 * are layered on top of these, with appropriate locking added.
19 */
20
21static int notifier_chain_register(struct notifier_block **nl,
22 struct notifier_block *n)
23{
24 while ((*nl) != NULL) {
25 if (n->priority > (*nl)->priority)
26 break;
27 nl = &((*nl)->next);
28 }
29 n->next = *nl;
30 rcu_assign_pointer(*nl, n);
31 return 0;
32}
33
34static int notifier_chain_cond_register(struct notifier_block **nl,
35 struct notifier_block *n)
36{
37 while ((*nl) != NULL) {
38 if ((*nl) == n)
39 return 0;
40 if (n->priority > (*nl)->priority)
41 break;
42 nl = &((*nl)->next);
43 }
44 n->next = *nl;
45 rcu_assign_pointer(*nl, n);
46 return 0;
47}
48
49static int notifier_chain_unregister(struct notifier_block **nl,
50 struct notifier_block *n)
51{
52 while ((*nl) != NULL) {
53 if ((*nl) == n) {
54 rcu_assign_pointer(*nl, n->next);
55 return 0;
56 }
57 nl = &((*nl)->next);
58 }
59 return -ENOENT;
60}
61
62/**
63 * notifier_call_chain - Informs the registered notifiers about an event.
64 * @nl: Pointer to head of the blocking notifier chain
65 * @val: Value passed unmodified to notifier function
66 * @v: Pointer passed unmodified to notifier function
67 * @nr_to_call: Number of notifier functions to be called. Don't care
68 * value of this parameter is -1.
69 * @nr_calls: Records the number of notifications sent. Don't care
70 * value of this field is NULL.
71 * @returns: notifier_call_chain returns the value returned by the
72 * last notifier function called.
73 */
74static int notifier_call_chain(struct notifier_block **nl,
75 unsigned long val, void *v,
76 int nr_to_call, int *nr_calls)
77{
78 int ret = NOTIFY_DONE;
79 struct notifier_block *nb, *next_nb;
80
81 nb = rcu_dereference_raw(*nl);
82
83 while (nb && nr_to_call) {
84 next_nb = rcu_dereference_raw(nb->next);
85
86#ifdef CONFIG_DEBUG_NOTIFIERS
87 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
88 WARN(1, "Invalid notifier called!");
89 nb = next_nb;
90 continue;
91 }
92#endif
93 ret = nb->notifier_call(nb, val, v);
94
95 if (nr_calls)
96 (*nr_calls)++;
97
98 if (ret & NOTIFY_STOP_MASK)
99 break;
100 nb = next_nb;
101 nr_to_call--;
102 }
103 return ret;
104}
105NOKPROBE_SYMBOL(notifier_call_chain);
106
107/*
108 * Atomic notifier chain routines. Registration and unregistration
109 * use a spinlock, and call_chain is synchronized by RCU (no locks).
110 */
111
112/**
113 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
114 * @nh: Pointer to head of the atomic notifier chain
115 * @n: New entry in notifier chain
116 *
117 * Adds a notifier to an atomic notifier chain.
118 *
119 * Currently always returns zero.
120 */
121int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
122 struct notifier_block *n)
123{
124 unsigned long flags;
125 int ret;
126
127 spin_lock_irqsave(&nh->lock, flags);
128 ret = notifier_chain_register(&nh->head, n);
129 spin_unlock_irqrestore(&nh->lock, flags);
130 return ret;
131}
132EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
133
134/**
135 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
136 * @nh: Pointer to head of the atomic notifier chain
137 * @n: Entry to remove from notifier chain
138 *
139 * Removes a notifier from an atomic notifier chain.
140 *
141 * Returns zero on success or %-ENOENT on failure.
142 */
143int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
144 struct notifier_block *n)
145{
146 unsigned long flags;
147 int ret;
148
149 spin_lock_irqsave(&nh->lock, flags);
150 ret = notifier_chain_unregister(&nh->head, n);
151 spin_unlock_irqrestore(&nh->lock, flags);
152 synchronize_rcu();
153 return ret;
154}
155EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
156
157/**
158 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
159 * @nh: Pointer to head of the atomic notifier chain
160 * @val: Value passed unmodified to notifier function
161 * @v: Pointer passed unmodified to notifier function
162 * @nr_to_call: See the comment for notifier_call_chain.
163 * @nr_calls: See the comment for notifier_call_chain.
164 *
165 * Calls each function in a notifier chain in turn. The functions
166 * run in an atomic context, so they must not block.
167 * This routine uses RCU to synchronize with changes to the chain.
168 *
169 * If the return value of the notifier can be and'ed
170 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
171 * will return immediately, with the return value of
172 * the notifier function which halted execution.
173 * Otherwise the return value is the return value
174 * of the last notifier function called.
175 */
176int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
177 unsigned long val, void *v,
178 int nr_to_call, int *nr_calls)
179{
180 int ret;
181
182 rcu_read_lock();
183 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
184 rcu_read_unlock();
185 return ret;
186}
187EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
188NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
189
190int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
191 unsigned long val, void *v)
192{
193 return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
194}
195EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
196NOKPROBE_SYMBOL(atomic_notifier_call_chain);
197
198/*
199 * Blocking notifier chain routines. All access to the chain is
200 * synchronized by an rwsem.
201 */
202
203/**
204 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
205 * @nh: Pointer to head of the blocking notifier chain
206 * @n: New entry in notifier chain
207 *
208 * Adds a notifier to a blocking notifier chain.
209 * Must be called in process context.
210 *
211 * Currently always returns zero.
212 */
213int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
214 struct notifier_block *n)
215{
216 int ret;
217
218 /*
219 * This code gets used during boot-up, when task switching is
220 * not yet working and interrupts must remain disabled. At
221 * such times we must not call down_write().
222 */
223 if (unlikely(system_state == SYSTEM_BOOTING))
224 return notifier_chain_register(&nh->head, n);
225
226 down_write(&nh->rwsem);
227 ret = notifier_chain_register(&nh->head, n);
228 up_write(&nh->rwsem);
229 return ret;
230}
231EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
232
233/**
234 * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain
235 * @nh: Pointer to head of the blocking notifier chain
236 * @n: New entry in notifier chain
237 *
238 * Adds a notifier to a blocking notifier chain, only if not already
239 * present in the chain.
240 * Must be called in process context.
241 *
242 * Currently always returns zero.
243 */
244int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh,
245 struct notifier_block *n)
246{
247 int ret;
248
249 down_write(&nh->rwsem);
250 ret = notifier_chain_cond_register(&nh->head, n);
251 up_write(&nh->rwsem);
252 return ret;
253}
254EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register);
255
256/**
257 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
258 * @nh: Pointer to head of the blocking notifier chain
259 * @n: Entry to remove from notifier chain
260 *
261 * Removes a notifier from a blocking notifier chain.
262 * Must be called from process context.
263 *
264 * Returns zero on success or %-ENOENT on failure.
265 */
266int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
267 struct notifier_block *n)
268{
269 int ret;
270
271 /*
272 * This code gets used during boot-up, when task switching is
273 * not yet working and interrupts must remain disabled. At
274 * such times we must not call down_write().
275 */
276 if (unlikely(system_state == SYSTEM_BOOTING))
277 return notifier_chain_unregister(&nh->head, n);
278
279 down_write(&nh->rwsem);
280 ret = notifier_chain_unregister(&nh->head, n);
281 up_write(&nh->rwsem);
282 return ret;
283}
284EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
285
286/**
287 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
288 * @nh: Pointer to head of the blocking notifier chain
289 * @val: Value passed unmodified to notifier function
290 * @v: Pointer passed unmodified to notifier function
291 * @nr_to_call: See comment for notifier_call_chain.
292 * @nr_calls: See comment for notifier_call_chain.
293 *
294 * Calls each function in a notifier chain in turn. The functions
295 * run in a process context, so they are allowed to block.
296 *
297 * If the return value of the notifier can be and'ed
298 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
299 * will return immediately, with the return value of
300 * the notifier function which halted execution.
301 * Otherwise the return value is the return value
302 * of the last notifier function called.
303 */
304int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
305 unsigned long val, void *v,
306 int nr_to_call, int *nr_calls)
307{
308 int ret = NOTIFY_DONE;
309
310 /*
311 * We check the head outside the lock, but if this access is
312 * racy then it does not matter what the result of the test
313 * is, we re-check the list after having taken the lock anyway:
314 */
315 if (rcu_access_pointer(nh->head)) {
316 down_read(&nh->rwsem);
317 ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
318 nr_calls);
319 up_read(&nh->rwsem);
320 }
321 return ret;
322}
323EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain);
324
325int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
326 unsigned long val, void *v)
327{
328 return __blocking_notifier_call_chain(nh, val, v, -1, NULL);
329}
330EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
331
332/*
333 * Raw notifier chain routines. There is no protection;
334 * the caller must provide it. Use at your own risk!
335 */
336
337/**
338 * raw_notifier_chain_register - Add notifier to a raw notifier chain
339 * @nh: Pointer to head of the raw notifier chain
340 * @n: New entry in notifier chain
341 *
342 * Adds a notifier to a raw notifier chain.
343 * All locking must be provided by the caller.
344 *
345 * Currently always returns zero.
346 */
347int raw_notifier_chain_register(struct raw_notifier_head *nh,
348 struct notifier_block *n)
349{
350 return notifier_chain_register(&nh->head, n);
351}
352EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
353
354/**
355 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
356 * @nh: Pointer to head of the raw notifier chain
357 * @n: Entry to remove from notifier chain
358 *
359 * Removes a notifier from a raw notifier chain.
360 * All locking must be provided by the caller.
361 *
362 * Returns zero on success or %-ENOENT on failure.
363 */
364int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
365 struct notifier_block *n)
366{
367 return notifier_chain_unregister(&nh->head, n);
368}
369EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
370
371/**
372 * __raw_notifier_call_chain - Call functions in a raw notifier chain
373 * @nh: Pointer to head of the raw notifier chain
374 * @val: Value passed unmodified to notifier function
375 * @v: Pointer passed unmodified to notifier function
376 * @nr_to_call: See comment for notifier_call_chain.
377 * @nr_calls: See comment for notifier_call_chain
378 *
379 * Calls each function in a notifier chain in turn. The functions
380 * run in an undefined context.
381 * All locking must be provided by the caller.
382 *
383 * If the return value of the notifier can be and'ed
384 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
385 * will return immediately, with the return value of
386 * the notifier function which halted execution.
387 * Otherwise the return value is the return value
388 * of the last notifier function called.
389 */
390int __raw_notifier_call_chain(struct raw_notifier_head *nh,
391 unsigned long val, void *v,
392 int nr_to_call, int *nr_calls)
393{
394 return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
395}
396EXPORT_SYMBOL_GPL(__raw_notifier_call_chain);
397
398int raw_notifier_call_chain(struct raw_notifier_head *nh,
399 unsigned long val, void *v)
400{
401 return __raw_notifier_call_chain(nh, val, v, -1, NULL);
402}
403EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
404
405#ifdef CONFIG_SRCU
406/*
407 * SRCU notifier chain routines. Registration and unregistration
408 * use a mutex, and call_chain is synchronized by SRCU (no locks).
409 */
410
411/**
412 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
413 * @nh: Pointer to head of the SRCU notifier chain
414 * @n: New entry in notifier chain
415 *
416 * Adds a notifier to an SRCU notifier chain.
417 * Must be called in process context.
418 *
419 * Currently always returns zero.
420 */
421int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
422 struct notifier_block *n)
423{
424 int ret;
425
426 /*
427 * This code gets used during boot-up, when task switching is
428 * not yet working and interrupts must remain disabled. At
429 * such times we must not call mutex_lock().
430 */
431 if (unlikely(system_state == SYSTEM_BOOTING))
432 return notifier_chain_register(&nh->head, n);
433
434 mutex_lock(&nh->mutex);
435 ret = notifier_chain_register(&nh->head, n);
436 mutex_unlock(&nh->mutex);
437 return ret;
438}
439EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
440
441/**
442 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
443 * @nh: Pointer to head of the SRCU notifier chain
444 * @n: Entry to remove from notifier chain
445 *
446 * Removes a notifier from an SRCU notifier chain.
447 * Must be called from process context.
448 *
449 * Returns zero on success or %-ENOENT on failure.
450 */
451int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
452 struct notifier_block *n)
453{
454 int ret;
455
456 /*
457 * This code gets used during boot-up, when task switching is
458 * not yet working and interrupts must remain disabled. At
459 * such times we must not call mutex_lock().
460 */
461 if (unlikely(system_state == SYSTEM_BOOTING))
462 return notifier_chain_unregister(&nh->head, n);
463
464 mutex_lock(&nh->mutex);
465 ret = notifier_chain_unregister(&nh->head, n);
466 mutex_unlock(&nh->mutex);
467 synchronize_srcu(&nh->srcu);
468 return ret;
469}
470EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
471
472/**
473 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
474 * @nh: Pointer to head of the SRCU notifier chain
475 * @val: Value passed unmodified to notifier function
476 * @v: Pointer passed unmodified to notifier function
477 * @nr_to_call: See comment for notifier_call_chain.
478 * @nr_calls: See comment for notifier_call_chain
479 *
480 * Calls each function in a notifier chain in turn. The functions
481 * run in a process context, so they are allowed to block.
482 *
483 * If the return value of the notifier can be and'ed
484 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
485 * will return immediately, with the return value of
486 * the notifier function which halted execution.
487 * Otherwise the return value is the return value
488 * of the last notifier function called.
489 */
490int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
491 unsigned long val, void *v,
492 int nr_to_call, int *nr_calls)
493{
494 int ret;
495 int idx;
496
497 idx = srcu_read_lock(&nh->srcu);
498 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
499 srcu_read_unlock(&nh->srcu, idx);
500 return ret;
501}
502EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain);
503
504int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
505 unsigned long val, void *v)
506{
507 return __srcu_notifier_call_chain(nh, val, v, -1, NULL);
508}
509EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
510
511/**
512 * srcu_init_notifier_head - Initialize an SRCU notifier head
513 * @nh: Pointer to head of the srcu notifier chain
514 *
515 * Unlike other sorts of notifier heads, SRCU notifier heads require
516 * dynamic initialization. Be sure to call this routine before
517 * calling any of the other SRCU notifier routines for this head.
518 *
519 * If an SRCU notifier head is deallocated, it must first be cleaned
520 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
521 * per-cpu data (used by the SRCU mechanism) will leak.
522 */
523void srcu_init_notifier_head(struct srcu_notifier_head *nh)
524{
525 mutex_init(&nh->mutex);
526 if (init_srcu_struct(&nh->srcu) < 0)
527 BUG();
528 nh->head = NULL;
529}
530EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
531
532#endif /* CONFIG_SRCU */
533
534static ATOMIC_NOTIFIER_HEAD(die_chain);
535
536int notrace notify_die(enum die_val val, const char *str,
537 struct pt_regs *regs, long err, int trap, int sig)
538{
539 struct die_args args = {
540 .regs = regs,
541 .str = str,
542 .err = err,
543 .trapnr = trap,
544 .signr = sig,
545
546 };
547 RCU_LOCKDEP_WARN(!rcu_is_watching(),
548 "notify_die called but RCU thinks we're quiescent");
549 return atomic_notifier_call_chain(&die_chain, val, &args);
550}
551NOKPROBE_SYMBOL(notify_die);
552
553int register_die_notifier(struct notifier_block *nb)
554{
555 vmalloc_sync_all();
556 return atomic_notifier_chain_register(&die_chain, nb);
557}
558EXPORT_SYMBOL_GPL(register_die_notifier);
559
560int unregister_die_notifier(struct notifier_block *nb)
561{
562 return atomic_notifier_chain_unregister(&die_chain, nb);
563}
564EXPORT_SYMBOL_GPL(unregister_die_notifier);
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8
9#define CREATE_TRACE_POINTS
10#include <trace/events/notifier.h>
11
12/*
13 * Notifier chain core routines. The exported routines below
14 * are layered on top of these, with appropriate locking added.
15 */
16
17static int notifier_chain_register(struct notifier_block **nl,
18 struct notifier_block *n,
19 bool unique_priority)
20{
21 while ((*nl) != NULL) {
22 if (unlikely((*nl) == n)) {
23 WARN(1, "notifier callback %ps already registered",
24 n->notifier_call);
25 return -EEXIST;
26 }
27 if (n->priority > (*nl)->priority)
28 break;
29 if (n->priority == (*nl)->priority && unique_priority)
30 return -EBUSY;
31 nl = &((*nl)->next);
32 }
33 n->next = *nl;
34 rcu_assign_pointer(*nl, n);
35 trace_notifier_register((void *)n->notifier_call);
36 return 0;
37}
38
39static int notifier_chain_unregister(struct notifier_block **nl,
40 struct notifier_block *n)
41{
42 while ((*nl) != NULL) {
43 if ((*nl) == n) {
44 rcu_assign_pointer(*nl, n->next);
45 trace_notifier_unregister((void *)n->notifier_call);
46 return 0;
47 }
48 nl = &((*nl)->next);
49 }
50 return -ENOENT;
51}
52
53/**
54 * notifier_call_chain - Informs the registered notifiers about an event.
55 * @nl: Pointer to head of the blocking notifier chain
56 * @val: Value passed unmodified to notifier function
57 * @v: Pointer passed unmodified to notifier function
58 * @nr_to_call: Number of notifier functions to be called. Don't care
59 * value of this parameter is -1.
60 * @nr_calls: Records the number of notifications sent. Don't care
61 * value of this field is NULL.
62 * Return: notifier_call_chain returns the value returned by the
63 * last notifier function called.
64 */
65static int notifier_call_chain(struct notifier_block **nl,
66 unsigned long val, void *v,
67 int nr_to_call, int *nr_calls)
68{
69 int ret = NOTIFY_DONE;
70 struct notifier_block *nb, *next_nb;
71
72 nb = rcu_dereference_raw(*nl);
73
74 while (nb && nr_to_call) {
75 next_nb = rcu_dereference_raw(nb->next);
76
77#ifdef CONFIG_DEBUG_NOTIFIERS
78 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
79 WARN(1, "Invalid notifier called!");
80 nb = next_nb;
81 continue;
82 }
83#endif
84 trace_notifier_run((void *)nb->notifier_call);
85 ret = nb->notifier_call(nb, val, v);
86
87 if (nr_calls)
88 (*nr_calls)++;
89
90 if (ret & NOTIFY_STOP_MASK)
91 break;
92 nb = next_nb;
93 nr_to_call--;
94 }
95 return ret;
96}
97NOKPROBE_SYMBOL(notifier_call_chain);
98
99/**
100 * notifier_call_chain_robust - Inform the registered notifiers about an event
101 * and rollback on error.
102 * @nl: Pointer to head of the blocking notifier chain
103 * @val_up: Value passed unmodified to the notifier function
104 * @val_down: Value passed unmodified to the notifier function when recovering
105 * from an error on @val_up
106 * @v: Pointer passed unmodified to the notifier function
107 *
108 * NOTE: It is important the @nl chain doesn't change between the two
109 * invocations of notifier_call_chain() such that we visit the
110 * exact same notifier callbacks; this rules out any RCU usage.
111 *
112 * Return: the return value of the @val_up call.
113 */
114static int notifier_call_chain_robust(struct notifier_block **nl,
115 unsigned long val_up, unsigned long val_down,
116 void *v)
117{
118 int ret, nr = 0;
119
120 ret = notifier_call_chain(nl, val_up, v, -1, &nr);
121 if (ret & NOTIFY_STOP_MASK)
122 notifier_call_chain(nl, val_down, v, nr-1, NULL);
123
124 return ret;
125}
126
127/*
128 * Atomic notifier chain routines. Registration and unregistration
129 * use a spinlock, and call_chain is synchronized by RCU (no locks).
130 */
131
132/**
133 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
134 * @nh: Pointer to head of the atomic notifier chain
135 * @n: New entry in notifier chain
136 *
137 * Adds a notifier to an atomic notifier chain.
138 *
139 * Returns 0 on success, %-EEXIST on error.
140 */
141int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
142 struct notifier_block *n)
143{
144 unsigned long flags;
145 int ret;
146
147 spin_lock_irqsave(&nh->lock, flags);
148 ret = notifier_chain_register(&nh->head, n, false);
149 spin_unlock_irqrestore(&nh->lock, flags);
150 return ret;
151}
152EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
153
154/**
155 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
156 * @nh: Pointer to head of the atomic notifier chain
157 * @n: New entry in notifier chain
158 *
159 * Adds a notifier to an atomic notifier chain if there is no other
160 * notifier registered using the same priority.
161 *
162 * Returns 0 on success, %-EEXIST or %-EBUSY on error.
163 */
164int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
165 struct notifier_block *n)
166{
167 unsigned long flags;
168 int ret;
169
170 spin_lock_irqsave(&nh->lock, flags);
171 ret = notifier_chain_register(&nh->head, n, true);
172 spin_unlock_irqrestore(&nh->lock, flags);
173 return ret;
174}
175EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
176
177/**
178 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
179 * @nh: Pointer to head of the atomic notifier chain
180 * @n: Entry to remove from notifier chain
181 *
182 * Removes a notifier from an atomic notifier chain.
183 *
184 * Returns zero on success or %-ENOENT on failure.
185 */
186int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
187 struct notifier_block *n)
188{
189 unsigned long flags;
190 int ret;
191
192 spin_lock_irqsave(&nh->lock, flags);
193 ret = notifier_chain_unregister(&nh->head, n);
194 spin_unlock_irqrestore(&nh->lock, flags);
195 synchronize_rcu();
196 return ret;
197}
198EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
199
200/**
201 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
202 * @nh: Pointer to head of the atomic notifier chain
203 * @val: Value passed unmodified to notifier function
204 * @v: Pointer passed unmodified to notifier function
205 *
206 * Calls each function in a notifier chain in turn. The functions
207 * run in an atomic context, so they must not block.
208 * This routine uses RCU to synchronize with changes to the chain.
209 *
210 * If the return value of the notifier can be and'ed
211 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
212 * will return immediately, with the return value of
213 * the notifier function which halted execution.
214 * Otherwise the return value is the return value
215 * of the last notifier function called.
216 */
217int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
218 unsigned long val, void *v)
219{
220 int ret;
221
222 rcu_read_lock();
223 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
224 rcu_read_unlock();
225
226 return ret;
227}
228EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
229NOKPROBE_SYMBOL(atomic_notifier_call_chain);
230
231/**
232 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
233 * @nh: Pointer to head of the atomic notifier chain
234 *
235 * Checks whether notifier chain is empty.
236 *
237 * Returns true is notifier chain is empty, false otherwise.
238 */
239bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
240{
241 return !rcu_access_pointer(nh->head);
242}
243
244/*
245 * Blocking notifier chain routines. All access to the chain is
246 * synchronized by an rwsem.
247 */
248
249static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
250 struct notifier_block *n,
251 bool unique_priority)
252{
253 int ret;
254
255 /*
256 * This code gets used during boot-up, when task switching is
257 * not yet working and interrupts must remain disabled. At
258 * such times we must not call down_write().
259 */
260 if (unlikely(system_state == SYSTEM_BOOTING))
261 return notifier_chain_register(&nh->head, n, unique_priority);
262
263 down_write(&nh->rwsem);
264 ret = notifier_chain_register(&nh->head, n, unique_priority);
265 up_write(&nh->rwsem);
266 return ret;
267}
268
269/**
270 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
271 * @nh: Pointer to head of the blocking notifier chain
272 * @n: New entry in notifier chain
273 *
274 * Adds a notifier to a blocking notifier chain.
275 * Must be called in process context.
276 *
277 * Returns 0 on success, %-EEXIST on error.
278 */
279int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
280 struct notifier_block *n)
281{
282 return __blocking_notifier_chain_register(nh, n, false);
283}
284EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
285
286/**
287 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
288 * @nh: Pointer to head of the blocking notifier chain
289 * @n: New entry in notifier chain
290 *
291 * Adds a notifier to an blocking notifier chain if there is no other
292 * notifier registered using the same priority.
293 *
294 * Returns 0 on success, %-EEXIST or %-EBUSY on error.
295 */
296int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
297 struct notifier_block *n)
298{
299 return __blocking_notifier_chain_register(nh, n, true);
300}
301EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
302
303/**
304 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
305 * @nh: Pointer to head of the blocking notifier chain
306 * @n: Entry to remove from notifier chain
307 *
308 * Removes a notifier from a blocking notifier chain.
309 * Must be called from process context.
310 *
311 * Returns zero on success or %-ENOENT on failure.
312 */
313int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
314 struct notifier_block *n)
315{
316 int ret;
317
318 /*
319 * This code gets used during boot-up, when task switching is
320 * not yet working and interrupts must remain disabled. At
321 * such times we must not call down_write().
322 */
323 if (unlikely(system_state == SYSTEM_BOOTING))
324 return notifier_chain_unregister(&nh->head, n);
325
326 down_write(&nh->rwsem);
327 ret = notifier_chain_unregister(&nh->head, n);
328 up_write(&nh->rwsem);
329 return ret;
330}
331EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
332
333int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
334 unsigned long val_up, unsigned long val_down, void *v)
335{
336 int ret = NOTIFY_DONE;
337
338 /*
339 * We check the head outside the lock, but if this access is
340 * racy then it does not matter what the result of the test
341 * is, we re-check the list after having taken the lock anyway:
342 */
343 if (rcu_access_pointer(nh->head)) {
344 down_read(&nh->rwsem);
345 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
346 up_read(&nh->rwsem);
347 }
348 return ret;
349}
350EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
351
352/**
353 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
354 * @nh: Pointer to head of the blocking notifier chain
355 * @val: Value passed unmodified to notifier function
356 * @v: Pointer passed unmodified to notifier function
357 *
358 * Calls each function in a notifier chain in turn. The functions
359 * run in a process context, so they are allowed to block.
360 *
361 * If the return value of the notifier can be and'ed
362 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
363 * will return immediately, with the return value of
364 * the notifier function which halted execution.
365 * Otherwise the return value is the return value
366 * of the last notifier function called.
367 */
368int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
369 unsigned long val, void *v)
370{
371 int ret = NOTIFY_DONE;
372
373 /*
374 * We check the head outside the lock, but if this access is
375 * racy then it does not matter what the result of the test
376 * is, we re-check the list after having taken the lock anyway:
377 */
378 if (rcu_access_pointer(nh->head)) {
379 down_read(&nh->rwsem);
380 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
381 up_read(&nh->rwsem);
382 }
383 return ret;
384}
385EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
386
387/*
388 * Raw notifier chain routines. There is no protection;
389 * the caller must provide it. Use at your own risk!
390 */
391
392/**
393 * raw_notifier_chain_register - Add notifier to a raw notifier chain
394 * @nh: Pointer to head of the raw notifier chain
395 * @n: New entry in notifier chain
396 *
397 * Adds a notifier to a raw notifier chain.
398 * All locking must be provided by the caller.
399 *
400 * Returns 0 on success, %-EEXIST on error.
401 */
402int raw_notifier_chain_register(struct raw_notifier_head *nh,
403 struct notifier_block *n)
404{
405 return notifier_chain_register(&nh->head, n, false);
406}
407EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
408
409/**
410 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
411 * @nh: Pointer to head of the raw notifier chain
412 * @n: Entry to remove from notifier chain
413 *
414 * Removes a notifier from a raw notifier chain.
415 * All locking must be provided by the caller.
416 *
417 * Returns zero on success or %-ENOENT on failure.
418 */
419int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
420 struct notifier_block *n)
421{
422 return notifier_chain_unregister(&nh->head, n);
423}
424EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
425
426int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
427 unsigned long val_up, unsigned long val_down, void *v)
428{
429 return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
430}
431EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
432
433/**
434 * raw_notifier_call_chain - Call functions in a raw notifier chain
435 * @nh: Pointer to head of the raw notifier chain
436 * @val: Value passed unmodified to notifier function
437 * @v: Pointer passed unmodified to notifier function
438 *
439 * Calls each function in a notifier chain in turn. The functions
440 * run in an undefined context.
441 * All locking must be provided by the caller.
442 *
443 * If the return value of the notifier can be and'ed
444 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
445 * will return immediately, with the return value of
446 * the notifier function which halted execution.
447 * Otherwise the return value is the return value
448 * of the last notifier function called.
449 */
450int raw_notifier_call_chain(struct raw_notifier_head *nh,
451 unsigned long val, void *v)
452{
453 return notifier_call_chain(&nh->head, val, v, -1, NULL);
454}
455EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
456
457/*
458 * SRCU notifier chain routines. Registration and unregistration
459 * use a mutex, and call_chain is synchronized by SRCU (no locks).
460 */
461
462/**
463 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
464 * @nh: Pointer to head of the SRCU notifier chain
465 * @n: New entry in notifier chain
466 *
467 * Adds a notifier to an SRCU notifier chain.
468 * Must be called in process context.
469 *
470 * Returns 0 on success, %-EEXIST on error.
471 */
472int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
473 struct notifier_block *n)
474{
475 int ret;
476
477 /*
478 * This code gets used during boot-up, when task switching is
479 * not yet working and interrupts must remain disabled. At
480 * such times we must not call mutex_lock().
481 */
482 if (unlikely(system_state == SYSTEM_BOOTING))
483 return notifier_chain_register(&nh->head, n, false);
484
485 mutex_lock(&nh->mutex);
486 ret = notifier_chain_register(&nh->head, n, false);
487 mutex_unlock(&nh->mutex);
488 return ret;
489}
490EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
491
492/**
493 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
494 * @nh: Pointer to head of the SRCU notifier chain
495 * @n: Entry to remove from notifier chain
496 *
497 * Removes a notifier from an SRCU notifier chain.
498 * Must be called from process context.
499 *
500 * Returns zero on success or %-ENOENT on failure.
501 */
502int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
503 struct notifier_block *n)
504{
505 int ret;
506
507 /*
508 * This code gets used during boot-up, when task switching is
509 * not yet working and interrupts must remain disabled. At
510 * such times we must not call mutex_lock().
511 */
512 if (unlikely(system_state == SYSTEM_BOOTING))
513 return notifier_chain_unregister(&nh->head, n);
514
515 mutex_lock(&nh->mutex);
516 ret = notifier_chain_unregister(&nh->head, n);
517 mutex_unlock(&nh->mutex);
518 synchronize_srcu(&nh->srcu);
519 return ret;
520}
521EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
522
523/**
524 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
525 * @nh: Pointer to head of the SRCU notifier chain
526 * @val: Value passed unmodified to notifier function
527 * @v: Pointer passed unmodified to notifier function
528 *
529 * Calls each function in a notifier chain in turn. The functions
530 * run in a process context, so they are allowed to block.
531 *
532 * If the return value of the notifier can be and'ed
533 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
534 * will return immediately, with the return value of
535 * the notifier function which halted execution.
536 * Otherwise the return value is the return value
537 * of the last notifier function called.
538 */
539int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
540 unsigned long val, void *v)
541{
542 int ret;
543 int idx;
544
545 idx = srcu_read_lock(&nh->srcu);
546 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
547 srcu_read_unlock(&nh->srcu, idx);
548 return ret;
549}
550EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
551
552/**
553 * srcu_init_notifier_head - Initialize an SRCU notifier head
554 * @nh: Pointer to head of the srcu notifier chain
555 *
556 * Unlike other sorts of notifier heads, SRCU notifier heads require
557 * dynamic initialization. Be sure to call this routine before
558 * calling any of the other SRCU notifier routines for this head.
559 *
560 * If an SRCU notifier head is deallocated, it must first be cleaned
561 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
562 * per-cpu data (used by the SRCU mechanism) will leak.
563 */
564void srcu_init_notifier_head(struct srcu_notifier_head *nh)
565{
566 mutex_init(&nh->mutex);
567 if (init_srcu_struct(&nh->srcu) < 0)
568 BUG();
569 nh->head = NULL;
570}
571EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
572
573static ATOMIC_NOTIFIER_HEAD(die_chain);
574
575int notrace notify_die(enum die_val val, const char *str,
576 struct pt_regs *regs, long err, int trap, int sig)
577{
578 struct die_args args = {
579 .regs = regs,
580 .str = str,
581 .err = err,
582 .trapnr = trap,
583 .signr = sig,
584
585 };
586 RCU_LOCKDEP_WARN(!rcu_is_watching(),
587 "notify_die called but RCU thinks we're quiescent");
588 return atomic_notifier_call_chain(&die_chain, val, &args);
589}
590NOKPROBE_SYMBOL(notify_die);
591
592int register_die_notifier(struct notifier_block *nb)
593{
594 return atomic_notifier_chain_register(&die_chain, nb);
595}
596EXPORT_SYMBOL_GPL(register_die_notifier);
597
598int unregister_die_notifier(struct notifier_block *nb)
599{
600 return atomic_notifier_chain_unregister(&die_chain, nb);
601}
602EXPORT_SYMBOL_GPL(unregister_die_notifier);