Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8#include <linux/reboot.h>
9
10#define CREATE_TRACE_POINTS
11#include <trace/events/notifier.h>
12
13/*
14 * Notifier list for kernel code which wants to be called
15 * at shutdown. This is used to stop any idling DMA operations
16 * and the like.
17 */
18BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
19
20/*
21 * Notifier chain core routines. The exported routines below
22 * are layered on top of these, with appropriate locking added.
23 */
24
25static int notifier_chain_register(struct notifier_block **nl,
26 struct notifier_block *n,
27 bool unique_priority)
28{
29 while ((*nl) != NULL) {
30 if (unlikely((*nl) == n)) {
31 WARN(1, "notifier callback %ps already registered",
32 n->notifier_call);
33 return -EEXIST;
34 }
35 if (n->priority > (*nl)->priority)
36 break;
37 if (n->priority == (*nl)->priority && unique_priority)
38 return -EBUSY;
39 nl = &((*nl)->next);
40 }
41 n->next = *nl;
42 rcu_assign_pointer(*nl, n);
43 trace_notifier_register((void *)n->notifier_call);
44 return 0;
45}
46
47static int notifier_chain_unregister(struct notifier_block **nl,
48 struct notifier_block *n)
49{
50 while ((*nl) != NULL) {
51 if ((*nl) == n) {
52 rcu_assign_pointer(*nl, n->next);
53 trace_notifier_unregister((void *)n->notifier_call);
54 return 0;
55 }
56 nl = &((*nl)->next);
57 }
58 return -ENOENT;
59}
60
61/**
62 * notifier_call_chain - Informs the registered notifiers about an event.
63 * @nl: Pointer to head of the blocking notifier chain
64 * @val: Value passed unmodified to notifier function
65 * @v: Pointer passed unmodified to notifier function
66 * @nr_to_call: Number of notifier functions to be called. Don't care
67 * value of this parameter is -1.
68 * @nr_calls: Records the number of notifications sent. Don't care
69 * value of this field is NULL.
70 * Return: notifier_call_chain returns the value returned by the
71 * last notifier function called.
72 */
73static int notifier_call_chain(struct notifier_block **nl,
74 unsigned long val, void *v,
75 int nr_to_call, int *nr_calls)
76{
77 int ret = NOTIFY_DONE;
78 struct notifier_block *nb, *next_nb;
79
80 nb = rcu_dereference_raw(*nl);
81
82 while (nb && nr_to_call) {
83 next_nb = rcu_dereference_raw(nb->next);
84
85#ifdef CONFIG_DEBUG_NOTIFIERS
86 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
87 WARN(1, "Invalid notifier called!");
88 nb = next_nb;
89 continue;
90 }
91#endif
92 trace_notifier_run((void *)nb->notifier_call);
93 ret = nb->notifier_call(nb, val, v);
94
95 if (nr_calls)
96 (*nr_calls)++;
97
98 if (ret & NOTIFY_STOP_MASK)
99 break;
100 nb = next_nb;
101 nr_to_call--;
102 }
103 return ret;
104}
105NOKPROBE_SYMBOL(notifier_call_chain);
106
107/**
108 * notifier_call_chain_robust - Inform the registered notifiers about an event
109 * and rollback on error.
110 * @nl: Pointer to head of the blocking notifier chain
111 * @val_up: Value passed unmodified to the notifier function
112 * @val_down: Value passed unmodified to the notifier function when recovering
113 * from an error on @val_up
114 * @v: Pointer passed unmodified to the notifier function
115 *
116 * NOTE: It is important the @nl chain doesn't change between the two
117 * invocations of notifier_call_chain() such that we visit the
118 * exact same notifier callbacks; this rules out any RCU usage.
119 *
120 * Return: the return value of the @val_up call.
121 */
122static int notifier_call_chain_robust(struct notifier_block **nl,
123 unsigned long val_up, unsigned long val_down,
124 void *v)
125{
126 int ret, nr = 0;
127
128 ret = notifier_call_chain(nl, val_up, v, -1, &nr);
129 if (ret & NOTIFY_STOP_MASK)
130 notifier_call_chain(nl, val_down, v, nr-1, NULL);
131
132 return ret;
133}
134
135/*
136 * Atomic notifier chain routines. Registration and unregistration
137 * use a spinlock, and call_chain is synchronized by RCU (no locks).
138 */
139
140/**
141 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
142 * @nh: Pointer to head of the atomic notifier chain
143 * @n: New entry in notifier chain
144 *
145 * Adds a notifier to an atomic notifier chain.
146 *
147 * Returns 0 on success, %-EEXIST on error.
148 */
149int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
150 struct notifier_block *n)
151{
152 unsigned long flags;
153 int ret;
154
155 spin_lock_irqsave(&nh->lock, flags);
156 ret = notifier_chain_register(&nh->head, n, false);
157 spin_unlock_irqrestore(&nh->lock, flags);
158 return ret;
159}
160EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
161
162/**
163 * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
164 * @nh: Pointer to head of the atomic notifier chain
165 * @n: New entry in notifier chain
166 *
167 * Adds a notifier to an atomic notifier chain if there is no other
168 * notifier registered using the same priority.
169 *
170 * Returns 0 on success, %-EEXIST or %-EBUSY on error.
171 */
172int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
173 struct notifier_block *n)
174{
175 unsigned long flags;
176 int ret;
177
178 spin_lock_irqsave(&nh->lock, flags);
179 ret = notifier_chain_register(&nh->head, n, true);
180 spin_unlock_irqrestore(&nh->lock, flags);
181 return ret;
182}
183EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
184
185/**
186 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
187 * @nh: Pointer to head of the atomic notifier chain
188 * @n: Entry to remove from notifier chain
189 *
190 * Removes a notifier from an atomic notifier chain.
191 *
192 * Returns zero on success or %-ENOENT on failure.
193 */
194int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
195 struct notifier_block *n)
196{
197 unsigned long flags;
198 int ret;
199
200 spin_lock_irqsave(&nh->lock, flags);
201 ret = notifier_chain_unregister(&nh->head, n);
202 spin_unlock_irqrestore(&nh->lock, flags);
203 synchronize_rcu();
204 return ret;
205}
206EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
207
208/**
209 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
210 * @nh: Pointer to head of the atomic notifier chain
211 * @val: Value passed unmodified to notifier function
212 * @v: Pointer passed unmodified to notifier function
213 *
214 * Calls each function in a notifier chain in turn. The functions
215 * run in an atomic context, so they must not block.
216 * This routine uses RCU to synchronize with changes to the chain.
217 *
218 * If the return value of the notifier can be and'ed
219 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
220 * will return immediately, with the return value of
221 * the notifier function which halted execution.
222 * Otherwise the return value is the return value
223 * of the last notifier function called.
224 */
225int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
226 unsigned long val, void *v)
227{
228 int ret;
229
230 rcu_read_lock();
231 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
232 rcu_read_unlock();
233
234 return ret;
235}
236EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
237NOKPROBE_SYMBOL(atomic_notifier_call_chain);
238
239/**
240 * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
241 * @nh: Pointer to head of the atomic notifier chain
242 *
243 * Checks whether notifier chain is empty.
244 *
245 * Returns true is notifier chain is empty, false otherwise.
246 */
247bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
248{
249 return !rcu_access_pointer(nh->head);
250}
251
252/*
253 * Blocking notifier chain routines. All access to the chain is
254 * synchronized by an rwsem.
255 */
256
257static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
258 struct notifier_block *n,
259 bool unique_priority)
260{
261 int ret;
262
263 /*
264 * This code gets used during boot-up, when task switching is
265 * not yet working and interrupts must remain disabled. At
266 * such times we must not call down_write().
267 */
268 if (unlikely(system_state == SYSTEM_BOOTING))
269 return notifier_chain_register(&nh->head, n, unique_priority);
270
271 down_write(&nh->rwsem);
272 ret = notifier_chain_register(&nh->head, n, unique_priority);
273 up_write(&nh->rwsem);
274 return ret;
275}
276
277/**
278 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
279 * @nh: Pointer to head of the blocking notifier chain
280 * @n: New entry in notifier chain
281 *
282 * Adds a notifier to a blocking notifier chain.
283 * Must be called in process context.
284 *
285 * Returns 0 on success, %-EEXIST on error.
286 */
287int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
288 struct notifier_block *n)
289{
290 return __blocking_notifier_chain_register(nh, n, false);
291}
292EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
293
294/**
295 * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
296 * @nh: Pointer to head of the blocking notifier chain
297 * @n: New entry in notifier chain
298 *
299 * Adds a notifier to an blocking notifier chain if there is no other
300 * notifier registered using the same priority.
301 *
302 * Returns 0 on success, %-EEXIST or %-EBUSY on error.
303 */
304int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
305 struct notifier_block *n)
306{
307 return __blocking_notifier_chain_register(nh, n, true);
308}
309EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
310
311/**
312 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
313 * @nh: Pointer to head of the blocking notifier chain
314 * @n: Entry to remove from notifier chain
315 *
316 * Removes a notifier from a blocking notifier chain.
317 * Must be called from process context.
318 *
319 * Returns zero on success or %-ENOENT on failure.
320 */
321int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
322 struct notifier_block *n)
323{
324 int ret;
325
326 /*
327 * This code gets used during boot-up, when task switching is
328 * not yet working and interrupts must remain disabled. At
329 * such times we must not call down_write().
330 */
331 if (unlikely(system_state == SYSTEM_BOOTING))
332 return notifier_chain_unregister(&nh->head, n);
333
334 down_write(&nh->rwsem);
335 ret = notifier_chain_unregister(&nh->head, n);
336 up_write(&nh->rwsem);
337 return ret;
338}
339EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
340
341int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
342 unsigned long val_up, unsigned long val_down, void *v)
343{
344 int ret = NOTIFY_DONE;
345
346 /*
347 * We check the head outside the lock, but if this access is
348 * racy then it does not matter what the result of the test
349 * is, we re-check the list after having taken the lock anyway:
350 */
351 if (rcu_access_pointer(nh->head)) {
352 down_read(&nh->rwsem);
353 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
354 up_read(&nh->rwsem);
355 }
356 return ret;
357}
358EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
359
360/**
361 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
362 * @nh: Pointer to head of the blocking notifier chain
363 * @val: Value passed unmodified to notifier function
364 * @v: Pointer passed unmodified to notifier function
365 *
366 * Calls each function in a notifier chain in turn. The functions
367 * run in a process context, so they are allowed to block.
368 *
369 * If the return value of the notifier can be and'ed
370 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
371 * will return immediately, with the return value of
372 * the notifier function which halted execution.
373 * Otherwise the return value is the return value
374 * of the last notifier function called.
375 */
376int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
377 unsigned long val, void *v)
378{
379 int ret = NOTIFY_DONE;
380
381 /*
382 * We check the head outside the lock, but if this access is
383 * racy then it does not matter what the result of the test
384 * is, we re-check the list after having taken the lock anyway:
385 */
386 if (rcu_access_pointer(nh->head)) {
387 down_read(&nh->rwsem);
388 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
389 up_read(&nh->rwsem);
390 }
391 return ret;
392}
393EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
394
395/*
396 * Raw notifier chain routines. There is no protection;
397 * the caller must provide it. Use at your own risk!
398 */
399
400/**
401 * raw_notifier_chain_register - Add notifier to a raw notifier chain
402 * @nh: Pointer to head of the raw notifier chain
403 * @n: New entry in notifier chain
404 *
405 * Adds a notifier to a raw notifier chain.
406 * All locking must be provided by the caller.
407 *
408 * Returns 0 on success, %-EEXIST on error.
409 */
410int raw_notifier_chain_register(struct raw_notifier_head *nh,
411 struct notifier_block *n)
412{
413 return notifier_chain_register(&nh->head, n, false);
414}
415EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
416
417/**
418 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
419 * @nh: Pointer to head of the raw notifier chain
420 * @n: Entry to remove from notifier chain
421 *
422 * Removes a notifier from a raw notifier chain.
423 * All locking must be provided by the caller.
424 *
425 * Returns zero on success or %-ENOENT on failure.
426 */
427int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
428 struct notifier_block *n)
429{
430 return notifier_chain_unregister(&nh->head, n);
431}
432EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
433
434int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
435 unsigned long val_up, unsigned long val_down, void *v)
436{
437 return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
438}
439EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
440
441/**
442 * raw_notifier_call_chain - Call functions in a raw notifier chain
443 * @nh: Pointer to head of the raw notifier chain
444 * @val: Value passed unmodified to notifier function
445 * @v: Pointer passed unmodified to notifier function
446 *
447 * Calls each function in a notifier chain in turn. The functions
448 * run in an undefined context.
449 * All locking must be provided by the caller.
450 *
451 * If the return value of the notifier can be and'ed
452 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
453 * will return immediately, with the return value of
454 * the notifier function which halted execution.
455 * Otherwise the return value is the return value
456 * of the last notifier function called.
457 */
458int raw_notifier_call_chain(struct raw_notifier_head *nh,
459 unsigned long val, void *v)
460{
461 return notifier_call_chain(&nh->head, val, v, -1, NULL);
462}
463EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
464
465/*
466 * SRCU notifier chain routines. Registration and unregistration
467 * use a mutex, and call_chain is synchronized by SRCU (no locks).
468 */
469
470/**
471 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
472 * @nh: Pointer to head of the SRCU notifier chain
473 * @n: New entry in notifier chain
474 *
475 * Adds a notifier to an SRCU notifier chain.
476 * Must be called in process context.
477 *
478 * Returns 0 on success, %-EEXIST on error.
479 */
480int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
481 struct notifier_block *n)
482{
483 int ret;
484
485 /*
486 * This code gets used during boot-up, when task switching is
487 * not yet working and interrupts must remain disabled. At
488 * such times we must not call mutex_lock().
489 */
490 if (unlikely(system_state == SYSTEM_BOOTING))
491 return notifier_chain_register(&nh->head, n, false);
492
493 mutex_lock(&nh->mutex);
494 ret = notifier_chain_register(&nh->head, n, false);
495 mutex_unlock(&nh->mutex);
496 return ret;
497}
498EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
499
500/**
501 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
502 * @nh: Pointer to head of the SRCU notifier chain
503 * @n: Entry to remove from notifier chain
504 *
505 * Removes a notifier from an SRCU notifier chain.
506 * Must be called from process context.
507 *
508 * Returns zero on success or %-ENOENT on failure.
509 */
510int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
511 struct notifier_block *n)
512{
513 int ret;
514
515 /*
516 * This code gets used during boot-up, when task switching is
517 * not yet working and interrupts must remain disabled. At
518 * such times we must not call mutex_lock().
519 */
520 if (unlikely(system_state == SYSTEM_BOOTING))
521 return notifier_chain_unregister(&nh->head, n);
522
523 mutex_lock(&nh->mutex);
524 ret = notifier_chain_unregister(&nh->head, n);
525 mutex_unlock(&nh->mutex);
526 synchronize_srcu(&nh->srcu);
527 return ret;
528}
529EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
530
531/**
532 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
533 * @nh: Pointer to head of the SRCU notifier chain
534 * @val: Value passed unmodified to notifier function
535 * @v: Pointer passed unmodified to notifier function
536 *
537 * Calls each function in a notifier chain in turn. The functions
538 * run in a process context, so they are allowed to block.
539 *
540 * If the return value of the notifier can be and'ed
541 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
542 * will return immediately, with the return value of
543 * the notifier function which halted execution.
544 * Otherwise the return value is the return value
545 * of the last notifier function called.
546 */
547int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
548 unsigned long val, void *v)
549{
550 int ret;
551 int idx;
552
553 idx = srcu_read_lock(&nh->srcu);
554 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
555 srcu_read_unlock(&nh->srcu, idx);
556 return ret;
557}
558EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
559
560/**
561 * srcu_init_notifier_head - Initialize an SRCU notifier head
562 * @nh: Pointer to head of the srcu notifier chain
563 *
564 * Unlike other sorts of notifier heads, SRCU notifier heads require
565 * dynamic initialization. Be sure to call this routine before
566 * calling any of the other SRCU notifier routines for this head.
567 *
568 * If an SRCU notifier head is deallocated, it must first be cleaned
569 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
570 * per-cpu data (used by the SRCU mechanism) will leak.
571 */
572void srcu_init_notifier_head(struct srcu_notifier_head *nh)
573{
574 mutex_init(&nh->mutex);
575 if (init_srcu_struct(&nh->srcu) < 0)
576 BUG();
577 nh->head = NULL;
578}
579EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
580
581static ATOMIC_NOTIFIER_HEAD(die_chain);
582
583int notrace notify_die(enum die_val val, const char *str,
584 struct pt_regs *regs, long err, int trap, int sig)
585{
586 struct die_args args = {
587 .regs = regs,
588 .str = str,
589 .err = err,
590 .trapnr = trap,
591 .signr = sig,
592
593 };
594 RCU_LOCKDEP_WARN(!rcu_is_watching(),
595 "notify_die called but RCU thinks we're quiescent");
596 return atomic_notifier_call_chain(&die_chain, val, &args);
597}
598NOKPROBE_SYMBOL(notify_die);
599
600int register_die_notifier(struct notifier_block *nb)
601{
602 return atomic_notifier_chain_register(&die_chain, nb);
603}
604EXPORT_SYMBOL_GPL(register_die_notifier);
605
606int unregister_die_notifier(struct notifier_block *nb)
607{
608 return atomic_notifier_chain_unregister(&die_chain, nb);
609}
610EXPORT_SYMBOL_GPL(unregister_die_notifier);
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kdebug.h>
3#include <linux/kprobes.h>
4#include <linux/export.h>
5#include <linux/notifier.h>
6#include <linux/rcupdate.h>
7#include <linux/vmalloc.h>
8#include <linux/reboot.h>
9
10/*
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
13 * and the like.
14 */
15BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16
17/*
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
20 */
21
22static int notifier_chain_register(struct notifier_block **nl,
23 struct notifier_block *n)
24{
25 while ((*nl) != NULL) {
26 if (unlikely((*nl) == n)) {
27 WARN(1, "double register detected");
28 return 0;
29 }
30 if (n->priority > (*nl)->priority)
31 break;
32 nl = &((*nl)->next);
33 }
34 n->next = *nl;
35 rcu_assign_pointer(*nl, n);
36 return 0;
37}
38
39static int notifier_chain_unregister(struct notifier_block **nl,
40 struct notifier_block *n)
41{
42 while ((*nl) != NULL) {
43 if ((*nl) == n) {
44 rcu_assign_pointer(*nl, n->next);
45 return 0;
46 }
47 nl = &((*nl)->next);
48 }
49 return -ENOENT;
50}
51
52/**
53 * notifier_call_chain - Informs the registered notifiers about an event.
54 * @nl: Pointer to head of the blocking notifier chain
55 * @val: Value passed unmodified to notifier function
56 * @v: Pointer passed unmodified to notifier function
57 * @nr_to_call: Number of notifier functions to be called. Don't care
58 * value of this parameter is -1.
59 * @nr_calls: Records the number of notifications sent. Don't care
60 * value of this field is NULL.
61 * @returns: notifier_call_chain returns the value returned by the
62 * last notifier function called.
63 */
64static int notifier_call_chain(struct notifier_block **nl,
65 unsigned long val, void *v,
66 int nr_to_call, int *nr_calls)
67{
68 int ret = NOTIFY_DONE;
69 struct notifier_block *nb, *next_nb;
70
71 nb = rcu_dereference_raw(*nl);
72
73 while (nb && nr_to_call) {
74 next_nb = rcu_dereference_raw(nb->next);
75
76#ifdef CONFIG_DEBUG_NOTIFIERS
77 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
78 WARN(1, "Invalid notifier called!");
79 nb = next_nb;
80 continue;
81 }
82#endif
83 ret = nb->notifier_call(nb, val, v);
84
85 if (nr_calls)
86 (*nr_calls)++;
87
88 if (ret & NOTIFY_STOP_MASK)
89 break;
90 nb = next_nb;
91 nr_to_call--;
92 }
93 return ret;
94}
95NOKPROBE_SYMBOL(notifier_call_chain);
96
97/*
98 * Atomic notifier chain routines. Registration and unregistration
99 * use a spinlock, and call_chain is synchronized by RCU (no locks).
100 */
101
102/**
103 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
104 * @nh: Pointer to head of the atomic notifier chain
105 * @n: New entry in notifier chain
106 *
107 * Adds a notifier to an atomic notifier chain.
108 *
109 * Currently always returns zero.
110 */
111int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
112 struct notifier_block *n)
113{
114 unsigned long flags;
115 int ret;
116
117 spin_lock_irqsave(&nh->lock, flags);
118 ret = notifier_chain_register(&nh->head, n);
119 spin_unlock_irqrestore(&nh->lock, flags);
120 return ret;
121}
122EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
123
124/**
125 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
126 * @nh: Pointer to head of the atomic notifier chain
127 * @n: Entry to remove from notifier chain
128 *
129 * Removes a notifier from an atomic notifier chain.
130 *
131 * Returns zero on success or %-ENOENT on failure.
132 */
133int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
134 struct notifier_block *n)
135{
136 unsigned long flags;
137 int ret;
138
139 spin_lock_irqsave(&nh->lock, flags);
140 ret = notifier_chain_unregister(&nh->head, n);
141 spin_unlock_irqrestore(&nh->lock, flags);
142 synchronize_rcu();
143 return ret;
144}
145EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
146
147/**
148 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
149 * @nh: Pointer to head of the atomic notifier chain
150 * @val: Value passed unmodified to notifier function
151 * @v: Pointer passed unmodified to notifier function
152 * @nr_to_call: See the comment for notifier_call_chain.
153 * @nr_calls: See the comment for notifier_call_chain.
154 *
155 * Calls each function in a notifier chain in turn. The functions
156 * run in an atomic context, so they must not block.
157 * This routine uses RCU to synchronize with changes to the chain.
158 *
159 * If the return value of the notifier can be and'ed
160 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
161 * will return immediately, with the return value of
162 * the notifier function which halted execution.
163 * Otherwise the return value is the return value
164 * of the last notifier function called.
165 */
166int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
167 unsigned long val, void *v,
168 int nr_to_call, int *nr_calls)
169{
170 int ret;
171
172 rcu_read_lock();
173 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
174 rcu_read_unlock();
175 return ret;
176}
177EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
178NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
179
180int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
181 unsigned long val, void *v)
182{
183 return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
184}
185EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
186NOKPROBE_SYMBOL(atomic_notifier_call_chain);
187
188/*
189 * Blocking notifier chain routines. All access to the chain is
190 * synchronized by an rwsem.
191 */
192
193/**
194 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
195 * @nh: Pointer to head of the blocking notifier chain
196 * @n: New entry in notifier chain
197 *
198 * Adds a notifier to a blocking notifier chain.
199 * Must be called in process context.
200 *
201 * Currently always returns zero.
202 */
203int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
204 struct notifier_block *n)
205{
206 int ret;
207
208 /*
209 * This code gets used during boot-up, when task switching is
210 * not yet working and interrupts must remain disabled. At
211 * such times we must not call down_write().
212 */
213 if (unlikely(system_state == SYSTEM_BOOTING))
214 return notifier_chain_register(&nh->head, n);
215
216 down_write(&nh->rwsem);
217 ret = notifier_chain_register(&nh->head, n);
218 up_write(&nh->rwsem);
219 return ret;
220}
221EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
222
223/**
224 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
225 * @nh: Pointer to head of the blocking notifier chain
226 * @n: Entry to remove from notifier chain
227 *
228 * Removes a notifier from a blocking notifier chain.
229 * Must be called from process context.
230 *
231 * Returns zero on success or %-ENOENT on failure.
232 */
233int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
234 struct notifier_block *n)
235{
236 int ret;
237
238 /*
239 * This code gets used during boot-up, when task switching is
240 * not yet working and interrupts must remain disabled. At
241 * such times we must not call down_write().
242 */
243 if (unlikely(system_state == SYSTEM_BOOTING))
244 return notifier_chain_unregister(&nh->head, n);
245
246 down_write(&nh->rwsem);
247 ret = notifier_chain_unregister(&nh->head, n);
248 up_write(&nh->rwsem);
249 return ret;
250}
251EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
252
253/**
254 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
255 * @nh: Pointer to head of the blocking notifier chain
256 * @val: Value passed unmodified to notifier function
257 * @v: Pointer passed unmodified to notifier function
258 * @nr_to_call: See comment for notifier_call_chain.
259 * @nr_calls: See comment for notifier_call_chain.
260 *
261 * Calls each function in a notifier chain in turn. The functions
262 * run in a process context, so they are allowed to block.
263 *
264 * If the return value of the notifier can be and'ed
265 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
266 * will return immediately, with the return value of
267 * the notifier function which halted execution.
268 * Otherwise the return value is the return value
269 * of the last notifier function called.
270 */
271int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
272 unsigned long val, void *v,
273 int nr_to_call, int *nr_calls)
274{
275 int ret = NOTIFY_DONE;
276
277 /*
278 * We check the head outside the lock, but if this access is
279 * racy then it does not matter what the result of the test
280 * is, we re-check the list after having taken the lock anyway:
281 */
282 if (rcu_access_pointer(nh->head)) {
283 down_read(&nh->rwsem);
284 ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
285 nr_calls);
286 up_read(&nh->rwsem);
287 }
288 return ret;
289}
290EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain);
291
292int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
293 unsigned long val, void *v)
294{
295 return __blocking_notifier_call_chain(nh, val, v, -1, NULL);
296}
297EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
298
299/*
300 * Raw notifier chain routines. There is no protection;
301 * the caller must provide it. Use at your own risk!
302 */
303
304/**
305 * raw_notifier_chain_register - Add notifier to a raw notifier chain
306 * @nh: Pointer to head of the raw notifier chain
307 * @n: New entry in notifier chain
308 *
309 * Adds a notifier to a raw notifier chain.
310 * All locking must be provided by the caller.
311 *
312 * Currently always returns zero.
313 */
314int raw_notifier_chain_register(struct raw_notifier_head *nh,
315 struct notifier_block *n)
316{
317 return notifier_chain_register(&nh->head, n);
318}
319EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
320
321/**
322 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
323 * @nh: Pointer to head of the raw notifier chain
324 * @n: Entry to remove from notifier chain
325 *
326 * Removes a notifier from a raw notifier chain.
327 * All locking must be provided by the caller.
328 *
329 * Returns zero on success or %-ENOENT on failure.
330 */
331int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
332 struct notifier_block *n)
333{
334 return notifier_chain_unregister(&nh->head, n);
335}
336EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
337
338/**
339 * __raw_notifier_call_chain - Call functions in a raw notifier chain
340 * @nh: Pointer to head of the raw notifier chain
341 * @val: Value passed unmodified to notifier function
342 * @v: Pointer passed unmodified to notifier function
343 * @nr_to_call: See comment for notifier_call_chain.
344 * @nr_calls: See comment for notifier_call_chain
345 *
346 * Calls each function in a notifier chain in turn. The functions
347 * run in an undefined context.
348 * All locking must be provided by the caller.
349 *
350 * If the return value of the notifier can be and'ed
351 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
352 * will return immediately, with the return value of
353 * the notifier function which halted execution.
354 * Otherwise the return value is the return value
355 * of the last notifier function called.
356 */
357int __raw_notifier_call_chain(struct raw_notifier_head *nh,
358 unsigned long val, void *v,
359 int nr_to_call, int *nr_calls)
360{
361 return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
362}
363EXPORT_SYMBOL_GPL(__raw_notifier_call_chain);
364
365int raw_notifier_call_chain(struct raw_notifier_head *nh,
366 unsigned long val, void *v)
367{
368 return __raw_notifier_call_chain(nh, val, v, -1, NULL);
369}
370EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
371
372#ifdef CONFIG_SRCU
373/*
374 * SRCU notifier chain routines. Registration and unregistration
375 * use a mutex, and call_chain is synchronized by SRCU (no locks).
376 */
377
378/**
379 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
380 * @nh: Pointer to head of the SRCU notifier chain
381 * @n: New entry in notifier chain
382 *
383 * Adds a notifier to an SRCU notifier chain.
384 * Must be called in process context.
385 *
386 * Currently always returns zero.
387 */
388int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
389 struct notifier_block *n)
390{
391 int ret;
392
393 /*
394 * This code gets used during boot-up, when task switching is
395 * not yet working and interrupts must remain disabled. At
396 * such times we must not call mutex_lock().
397 */
398 if (unlikely(system_state == SYSTEM_BOOTING))
399 return notifier_chain_register(&nh->head, n);
400
401 mutex_lock(&nh->mutex);
402 ret = notifier_chain_register(&nh->head, n);
403 mutex_unlock(&nh->mutex);
404 return ret;
405}
406EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
407
408/**
409 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
410 * @nh: Pointer to head of the SRCU notifier chain
411 * @n: Entry to remove from notifier chain
412 *
413 * Removes a notifier from an SRCU notifier chain.
414 * Must be called from process context.
415 *
416 * Returns zero on success or %-ENOENT on failure.
417 */
418int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
419 struct notifier_block *n)
420{
421 int ret;
422
423 /*
424 * This code gets used during boot-up, when task switching is
425 * not yet working and interrupts must remain disabled. At
426 * such times we must not call mutex_lock().
427 */
428 if (unlikely(system_state == SYSTEM_BOOTING))
429 return notifier_chain_unregister(&nh->head, n);
430
431 mutex_lock(&nh->mutex);
432 ret = notifier_chain_unregister(&nh->head, n);
433 mutex_unlock(&nh->mutex);
434 synchronize_srcu(&nh->srcu);
435 return ret;
436}
437EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
438
439/**
440 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
441 * @nh: Pointer to head of the SRCU notifier chain
442 * @val: Value passed unmodified to notifier function
443 * @v: Pointer passed unmodified to notifier function
444 * @nr_to_call: See comment for notifier_call_chain.
445 * @nr_calls: See comment for notifier_call_chain
446 *
447 * Calls each function in a notifier chain in turn. The functions
448 * run in a process context, so they are allowed to block.
449 *
450 * If the return value of the notifier can be and'ed
451 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
452 * will return immediately, with the return value of
453 * the notifier function which halted execution.
454 * Otherwise the return value is the return value
455 * of the last notifier function called.
456 */
457int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
458 unsigned long val, void *v,
459 int nr_to_call, int *nr_calls)
460{
461 int ret;
462 int idx;
463
464 idx = srcu_read_lock(&nh->srcu);
465 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
466 srcu_read_unlock(&nh->srcu, idx);
467 return ret;
468}
469EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain);
470
471int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
472 unsigned long val, void *v)
473{
474 return __srcu_notifier_call_chain(nh, val, v, -1, NULL);
475}
476EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
477
478/**
479 * srcu_init_notifier_head - Initialize an SRCU notifier head
480 * @nh: Pointer to head of the srcu notifier chain
481 *
482 * Unlike other sorts of notifier heads, SRCU notifier heads require
483 * dynamic initialization. Be sure to call this routine before
484 * calling any of the other SRCU notifier routines for this head.
485 *
486 * If an SRCU notifier head is deallocated, it must first be cleaned
487 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
488 * per-cpu data (used by the SRCU mechanism) will leak.
489 */
490void srcu_init_notifier_head(struct srcu_notifier_head *nh)
491{
492 mutex_init(&nh->mutex);
493 if (init_srcu_struct(&nh->srcu) < 0)
494 BUG();
495 nh->head = NULL;
496}
497EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
498
499#endif /* CONFIG_SRCU */
500
501static ATOMIC_NOTIFIER_HEAD(die_chain);
502
503int notrace notify_die(enum die_val val, const char *str,
504 struct pt_regs *regs, long err, int trap, int sig)
505{
506 struct die_args args = {
507 .regs = regs,
508 .str = str,
509 .err = err,
510 .trapnr = trap,
511 .signr = sig,
512
513 };
514 RCU_LOCKDEP_WARN(!rcu_is_watching(),
515 "notify_die called but RCU thinks we're quiescent");
516 return atomic_notifier_call_chain(&die_chain, val, &args);
517}
518NOKPROBE_SYMBOL(notify_die);
519
520int register_die_notifier(struct notifier_block *nb)
521{
522 return atomic_notifier_chain_register(&die_chain, nb);
523}
524EXPORT_SYMBOL_GPL(register_die_notifier);
525
526int unregister_die_notifier(struct notifier_block *nb)
527{
528 return atomic_notifier_chain_unregister(&die_chain, nb);
529}
530EXPORT_SYMBOL_GPL(unregister_die_notifier);