Loading...
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2017 Arm Ltd.
3#define pr_fmt(fmt) "sdei: " fmt
4
5#include <acpi/ghes.h>
6#include <linux/acpi.h>
7#include <linux/arm_sdei.h>
8#include <linux/arm-smccc.h>
9#include <linux/atomic.h>
10#include <linux/bitops.h>
11#include <linux/compiler.h>
12#include <linux/cpuhotplug.h>
13#include <linux/cpu.h>
14#include <linux/cpu_pm.h>
15#include <linux/errno.h>
16#include <linux/hardirq.h>
17#include <linux/kernel.h>
18#include <linux/kprobes.h>
19#include <linux/kvm_host.h>
20#include <linux/list.h>
21#include <linux/mutex.h>
22#include <linux/notifier.h>
23#include <linux/of.h>
24#include <linux/of_platform.h>
25#include <linux/percpu.h>
26#include <linux/platform_device.h>
27#include <linux/pm.h>
28#include <linux/ptrace.h>
29#include <linux/preempt.h>
30#include <linux/reboot.h>
31#include <linux/slab.h>
32#include <linux/smp.h>
33#include <linux/spinlock.h>
34#include <linux/uaccess.h>
35
36/*
37 * The call to use to reach the firmware.
38 */
39static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
40 unsigned long arg0, unsigned long arg1,
41 unsigned long arg2, unsigned long arg3,
42 unsigned long arg4, struct arm_smccc_res *res);
43
44/* entry point from firmware to arch asm code */
45static unsigned long sdei_entry_point;
46
47struct sdei_event {
48 /* These three are protected by the sdei_list_lock */
49 struct list_head list;
50 bool reregister;
51 bool reenable;
52
53 u32 event_num;
54 u8 type;
55 u8 priority;
56
57 /* This pointer is handed to firmware as the event argument. */
58 union {
59 /* Shared events */
60 struct sdei_registered_event *registered;
61
62 /* CPU private events */
63 struct sdei_registered_event __percpu *private_registered;
64 };
65};
66
67/* Take the mutex for any API call or modification. Take the mutex first. */
68static DEFINE_MUTEX(sdei_events_lock);
69
70/* and then hold this when modifying the list */
71static DEFINE_SPINLOCK(sdei_list_lock);
72static LIST_HEAD(sdei_list);
73
74/* Private events are registered/enabled via IPI passing one of these */
75struct sdei_crosscall_args {
76 struct sdei_event *event;
77 atomic_t errors;
78 int first_error;
79};
80
81#define CROSSCALL_INIT(arg, event) (arg.event = event, \
82 arg.first_error = 0, \
83 atomic_set(&arg.errors, 0))
84
85static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
86{
87 struct sdei_crosscall_args arg;
88
89 CROSSCALL_INIT(arg, event);
90 on_each_cpu(fn, &arg, true);
91
92 return arg.first_error;
93}
94
95static inline void
96sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
97{
98 if (err && (atomic_inc_return(&arg->errors) == 1))
99 arg->first_error = err;
100}
101
102static int sdei_to_linux_errno(unsigned long sdei_err)
103{
104 switch (sdei_err) {
105 case SDEI_NOT_SUPPORTED:
106 return -EOPNOTSUPP;
107 case SDEI_INVALID_PARAMETERS:
108 return -EINVAL;
109 case SDEI_DENIED:
110 return -EPERM;
111 case SDEI_PENDING:
112 return -EINPROGRESS;
113 case SDEI_OUT_OF_RESOURCE:
114 return -ENOMEM;
115 }
116
117 /* Not an error value ... */
118 return sdei_err;
119}
120
121/*
122 * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
123 * to translate.
124 */
125static int sdei_is_err(struct arm_smccc_res *res)
126{
127 switch (res->a0) {
128 case SDEI_NOT_SUPPORTED:
129 case SDEI_INVALID_PARAMETERS:
130 case SDEI_DENIED:
131 case SDEI_PENDING:
132 case SDEI_OUT_OF_RESOURCE:
133 return true;
134 }
135
136 return false;
137}
138
139static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
140 unsigned long arg1, unsigned long arg2,
141 unsigned long arg3, unsigned long arg4,
142 u64 *result)
143{
144 int err = 0;
145 struct arm_smccc_res res;
146
147 if (sdei_firmware_call) {
148 sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
149 &res);
150 if (sdei_is_err(&res))
151 err = sdei_to_linux_errno(res.a0);
152 } else {
153 /*
154 * !sdei_firmware_call means we failed to probe or called
155 * sdei_mark_interface_broken(). -EIO is not an error returned
156 * by sdei_to_linux_errno() and is used to suppress messages
157 * from this driver.
158 */
159 err = -EIO;
160 res.a0 = SDEI_NOT_SUPPORTED;
161 }
162
163 if (result)
164 *result = res.a0;
165
166 return err;
167}
168NOKPROBE_SYMBOL(invoke_sdei_fn);
169
170static struct sdei_event *sdei_event_find(u32 event_num)
171{
172 struct sdei_event *e, *found = NULL;
173
174 lockdep_assert_held(&sdei_events_lock);
175
176 spin_lock(&sdei_list_lock);
177 list_for_each_entry(e, &sdei_list, list) {
178 if (e->event_num == event_num) {
179 found = e;
180 break;
181 }
182 }
183 spin_unlock(&sdei_list_lock);
184
185 return found;
186}
187
188int sdei_api_event_context(u32 query, u64 *result)
189{
190 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
191 result);
192}
193NOKPROBE_SYMBOL(sdei_api_event_context);
194
195static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
196{
197 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
198 0, 0, result);
199}
200
201static struct sdei_event *sdei_event_create(u32 event_num,
202 sdei_event_callback *cb,
203 void *cb_arg)
204{
205 int err;
206 u64 result;
207 struct sdei_event *event;
208 struct sdei_registered_event *reg;
209
210 lockdep_assert_held(&sdei_events_lock);
211
212 event = kzalloc(sizeof(*event), GFP_KERNEL);
213 if (!event)
214 return ERR_PTR(-ENOMEM);
215
216 INIT_LIST_HEAD(&event->list);
217 event->event_num = event_num;
218
219 err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
220 &result);
221 if (err) {
222 kfree(event);
223 return ERR_PTR(err);
224 }
225 event->priority = result;
226
227 err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
228 &result);
229 if (err) {
230 kfree(event);
231 return ERR_PTR(err);
232 }
233 event->type = result;
234
235 if (event->type == SDEI_EVENT_TYPE_SHARED) {
236 reg = kzalloc(sizeof(*reg), GFP_KERNEL);
237 if (!reg) {
238 kfree(event);
239 return ERR_PTR(-ENOMEM);
240 }
241
242 reg->event_num = event_num;
243 reg->priority = event->priority;
244
245 reg->callback = cb;
246 reg->callback_arg = cb_arg;
247 event->registered = reg;
248 } else {
249 int cpu;
250 struct sdei_registered_event __percpu *regs;
251
252 regs = alloc_percpu(struct sdei_registered_event);
253 if (!regs) {
254 kfree(event);
255 return ERR_PTR(-ENOMEM);
256 }
257
258 for_each_possible_cpu(cpu) {
259 reg = per_cpu_ptr(regs, cpu);
260
261 reg->event_num = event->event_num;
262 reg->priority = event->priority;
263 reg->callback = cb;
264 reg->callback_arg = cb_arg;
265 }
266
267 event->private_registered = regs;
268 }
269
270 spin_lock(&sdei_list_lock);
271 list_add(&event->list, &sdei_list);
272 spin_unlock(&sdei_list_lock);
273
274 return event;
275}
276
277static void sdei_event_destroy_llocked(struct sdei_event *event)
278{
279 lockdep_assert_held(&sdei_events_lock);
280 lockdep_assert_held(&sdei_list_lock);
281
282 list_del(&event->list);
283
284 if (event->type == SDEI_EVENT_TYPE_SHARED)
285 kfree(event->registered);
286 else
287 free_percpu(event->private_registered);
288
289 kfree(event);
290}
291
292static void sdei_event_destroy(struct sdei_event *event)
293{
294 spin_lock(&sdei_list_lock);
295 sdei_event_destroy_llocked(event);
296 spin_unlock(&sdei_list_lock);
297}
298
299static int sdei_api_get_version(u64 *version)
300{
301 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
302}
303
304int sdei_mask_local_cpu(void)
305{
306 int err;
307
308 WARN_ON_ONCE(preemptible());
309
310 err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
311 if (err && err != -EIO) {
312 pr_warn_once("failed to mask CPU[%u]: %d\n",
313 smp_processor_id(), err);
314 return err;
315 }
316
317 return 0;
318}
319
320static void _ipi_mask_cpu(void *ignored)
321{
322 sdei_mask_local_cpu();
323}
324
325int sdei_unmask_local_cpu(void)
326{
327 int err;
328
329 WARN_ON_ONCE(preemptible());
330
331 err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
332 if (err && err != -EIO) {
333 pr_warn_once("failed to unmask CPU[%u]: %d\n",
334 smp_processor_id(), err);
335 return err;
336 }
337
338 return 0;
339}
340
341static void _ipi_unmask_cpu(void *ignored)
342{
343 sdei_unmask_local_cpu();
344}
345
346static void _ipi_private_reset(void *ignored)
347{
348 int err;
349
350 err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
351 NULL);
352 if (err && err != -EIO)
353 pr_warn_once("failed to reset CPU[%u]: %d\n",
354 smp_processor_id(), err);
355}
356
357static int sdei_api_shared_reset(void)
358{
359 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
360 NULL);
361}
362
363static void sdei_mark_interface_broken(void)
364{
365 pr_err("disabling SDEI firmware interface\n");
366 on_each_cpu(&_ipi_mask_cpu, NULL, true);
367 sdei_firmware_call = NULL;
368}
369
370static int sdei_platform_reset(void)
371{
372 int err;
373
374 on_each_cpu(&_ipi_private_reset, NULL, true);
375 err = sdei_api_shared_reset();
376 if (err) {
377 pr_err("Failed to reset platform: %d\n", err);
378 sdei_mark_interface_broken();
379 }
380
381 return err;
382}
383
384static int sdei_api_event_enable(u32 event_num)
385{
386 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
387 0, NULL);
388}
389
390/* Called directly by the hotplug callbacks */
391static void _local_event_enable(void *data)
392{
393 int err;
394 struct sdei_crosscall_args *arg = data;
395
396 WARN_ON_ONCE(preemptible());
397
398 err = sdei_api_event_enable(arg->event->event_num);
399
400 sdei_cross_call_return(arg, err);
401}
402
403int sdei_event_enable(u32 event_num)
404{
405 int err = -EINVAL;
406 struct sdei_event *event;
407
408 mutex_lock(&sdei_events_lock);
409 event = sdei_event_find(event_num);
410 if (!event) {
411 mutex_unlock(&sdei_events_lock);
412 return -ENOENT;
413 }
414
415
416 cpus_read_lock();
417 if (event->type == SDEI_EVENT_TYPE_SHARED)
418 err = sdei_api_event_enable(event->event_num);
419 else
420 err = sdei_do_cross_call(_local_event_enable, event);
421
422 if (!err) {
423 spin_lock(&sdei_list_lock);
424 event->reenable = true;
425 spin_unlock(&sdei_list_lock);
426 }
427 cpus_read_unlock();
428 mutex_unlock(&sdei_events_lock);
429
430 return err;
431}
432
433static int sdei_api_event_disable(u32 event_num)
434{
435 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
436 0, 0, NULL);
437}
438
439static void _ipi_event_disable(void *data)
440{
441 int err;
442 struct sdei_crosscall_args *arg = data;
443
444 err = sdei_api_event_disable(arg->event->event_num);
445
446 sdei_cross_call_return(arg, err);
447}
448
449int sdei_event_disable(u32 event_num)
450{
451 int err = -EINVAL;
452 struct sdei_event *event;
453
454 mutex_lock(&sdei_events_lock);
455 event = sdei_event_find(event_num);
456 if (!event) {
457 mutex_unlock(&sdei_events_lock);
458 return -ENOENT;
459 }
460
461 spin_lock(&sdei_list_lock);
462 event->reenable = false;
463 spin_unlock(&sdei_list_lock);
464
465 if (event->type == SDEI_EVENT_TYPE_SHARED)
466 err = sdei_api_event_disable(event->event_num);
467 else
468 err = sdei_do_cross_call(_ipi_event_disable, event);
469 mutex_unlock(&sdei_events_lock);
470
471 return err;
472}
473
474static int sdei_api_event_unregister(u32 event_num)
475{
476 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
477 0, 0, 0, NULL);
478}
479
480/* Called directly by the hotplug callbacks */
481static void _local_event_unregister(void *data)
482{
483 int err;
484 struct sdei_crosscall_args *arg = data;
485
486 WARN_ON_ONCE(preemptible());
487
488 err = sdei_api_event_unregister(arg->event->event_num);
489
490 sdei_cross_call_return(arg, err);
491}
492
493static int _sdei_event_unregister(struct sdei_event *event)
494{
495 lockdep_assert_held(&sdei_events_lock);
496
497 if (event->type == SDEI_EVENT_TYPE_SHARED)
498 return sdei_api_event_unregister(event->event_num);
499
500 return sdei_do_cross_call(_local_event_unregister, event);
501}
502
503int sdei_event_unregister(u32 event_num)
504{
505 int err;
506 struct sdei_event *event;
507
508 WARN_ON(in_nmi());
509
510 mutex_lock(&sdei_events_lock);
511 event = sdei_event_find(event_num);
512 do {
513 if (!event) {
514 pr_warn("Event %u not registered\n", event_num);
515 err = -ENOENT;
516 break;
517 }
518
519 spin_lock(&sdei_list_lock);
520 event->reregister = false;
521 event->reenable = false;
522 spin_unlock(&sdei_list_lock);
523
524 err = _sdei_event_unregister(event);
525 if (err)
526 break;
527
528 sdei_event_destroy(event);
529 } while (0);
530 mutex_unlock(&sdei_events_lock);
531
532 return err;
533}
534
535/*
536 * unregister events, but don't destroy them as they are re-registered by
537 * sdei_reregister_shared().
538 */
539static int sdei_unregister_shared(void)
540{
541 int err = 0;
542 struct sdei_event *event;
543
544 mutex_lock(&sdei_events_lock);
545 spin_lock(&sdei_list_lock);
546 list_for_each_entry(event, &sdei_list, list) {
547 if (event->type != SDEI_EVENT_TYPE_SHARED)
548 continue;
549
550 err = _sdei_event_unregister(event);
551 if (err)
552 break;
553 }
554 spin_unlock(&sdei_list_lock);
555 mutex_unlock(&sdei_events_lock);
556
557 return err;
558}
559
560static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
561 void *arg, u64 flags, u64 affinity)
562{
563 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
564 (unsigned long)entry_point, (unsigned long)arg,
565 flags, affinity, NULL);
566}
567
568/* Called directly by the hotplug callbacks */
569static void _local_event_register(void *data)
570{
571 int err;
572 struct sdei_registered_event *reg;
573 struct sdei_crosscall_args *arg = data;
574
575 WARN_ON(preemptible());
576
577 reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
578 err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
579 reg, 0, 0);
580
581 sdei_cross_call_return(arg, err);
582}
583
584static int _sdei_event_register(struct sdei_event *event)
585{
586 int err;
587
588 lockdep_assert_held(&sdei_events_lock);
589
590 if (event->type == SDEI_EVENT_TYPE_SHARED)
591 return sdei_api_event_register(event->event_num,
592 sdei_entry_point,
593 event->registered,
594 SDEI_EVENT_REGISTER_RM_ANY, 0);
595
596 err = sdei_do_cross_call(_local_event_register, event);
597 if (err)
598 sdei_do_cross_call(_local_event_unregister, event);
599
600 return err;
601}
602
603int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
604{
605 int err;
606 struct sdei_event *event;
607
608 WARN_ON(in_nmi());
609
610 mutex_lock(&sdei_events_lock);
611 do {
612 if (sdei_event_find(event_num)) {
613 pr_warn("Event %u already registered\n", event_num);
614 err = -EBUSY;
615 break;
616 }
617
618 event = sdei_event_create(event_num, cb, arg);
619 if (IS_ERR(event)) {
620 err = PTR_ERR(event);
621 pr_warn("Failed to create event %u: %d\n", event_num,
622 err);
623 break;
624 }
625
626 cpus_read_lock();
627 err = _sdei_event_register(event);
628 if (err) {
629 sdei_event_destroy(event);
630 pr_warn("Failed to register event %u: %d\n", event_num,
631 err);
632 } else {
633 spin_lock(&sdei_list_lock);
634 event->reregister = true;
635 spin_unlock(&sdei_list_lock);
636 }
637 cpus_read_unlock();
638 } while (0);
639 mutex_unlock(&sdei_events_lock);
640
641 return err;
642}
643
644static int sdei_reregister_event_llocked(struct sdei_event *event)
645{
646 int err;
647
648 lockdep_assert_held(&sdei_events_lock);
649 lockdep_assert_held(&sdei_list_lock);
650
651 err = _sdei_event_register(event);
652 if (err) {
653 pr_err("Failed to re-register event %u\n", event->event_num);
654 sdei_event_destroy_llocked(event);
655 return err;
656 }
657
658 if (event->reenable) {
659 if (event->type == SDEI_EVENT_TYPE_SHARED)
660 err = sdei_api_event_enable(event->event_num);
661 else
662 err = sdei_do_cross_call(_local_event_enable, event);
663 }
664
665 if (err)
666 pr_err("Failed to re-enable event %u\n", event->event_num);
667
668 return err;
669}
670
671static int sdei_reregister_shared(void)
672{
673 int err = 0;
674 struct sdei_event *event;
675
676 mutex_lock(&sdei_events_lock);
677 spin_lock(&sdei_list_lock);
678 list_for_each_entry(event, &sdei_list, list) {
679 if (event->type != SDEI_EVENT_TYPE_SHARED)
680 continue;
681
682 if (event->reregister) {
683 err = sdei_reregister_event_llocked(event);
684 if (err)
685 break;
686 }
687 }
688 spin_unlock(&sdei_list_lock);
689 mutex_unlock(&sdei_events_lock);
690
691 return err;
692}
693
694static int sdei_cpuhp_down(unsigned int cpu)
695{
696 struct sdei_event *event;
697 struct sdei_crosscall_args arg;
698
699 /* un-register private events */
700 spin_lock(&sdei_list_lock);
701 list_for_each_entry(event, &sdei_list, list) {
702 if (event->type == SDEI_EVENT_TYPE_SHARED)
703 continue;
704
705 CROSSCALL_INIT(arg, event);
706 /* call the cross-call function locally... */
707 _local_event_unregister(&arg);
708 if (arg.first_error)
709 pr_err("Failed to unregister event %u: %d\n",
710 event->event_num, arg.first_error);
711 }
712 spin_unlock(&sdei_list_lock);
713
714 return sdei_mask_local_cpu();
715}
716
717static int sdei_cpuhp_up(unsigned int cpu)
718{
719 struct sdei_event *event;
720 struct sdei_crosscall_args arg;
721
722 /* re-register/enable private events */
723 spin_lock(&sdei_list_lock);
724 list_for_each_entry(event, &sdei_list, list) {
725 if (event->type == SDEI_EVENT_TYPE_SHARED)
726 continue;
727
728 if (event->reregister) {
729 CROSSCALL_INIT(arg, event);
730 /* call the cross-call function locally... */
731 _local_event_register(&arg);
732 if (arg.first_error)
733 pr_err("Failed to re-register event %u: %d\n",
734 event->event_num, arg.first_error);
735 }
736
737 if (event->reenable) {
738 CROSSCALL_INIT(arg, event);
739 _local_event_enable(&arg);
740 if (arg.first_error)
741 pr_err("Failed to re-enable event %u: %d\n",
742 event->event_num, arg.first_error);
743 }
744 }
745 spin_unlock(&sdei_list_lock);
746
747 return sdei_unmask_local_cpu();
748}
749
750/* When entering idle, mask/unmask events for this cpu */
751static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
752 void *data)
753{
754 int rv;
755
756 switch (action) {
757 case CPU_PM_ENTER:
758 rv = sdei_mask_local_cpu();
759 break;
760 case CPU_PM_EXIT:
761 case CPU_PM_ENTER_FAILED:
762 rv = sdei_unmask_local_cpu();
763 break;
764 default:
765 return NOTIFY_DONE;
766 }
767
768 if (rv)
769 return notifier_from_errno(rv);
770
771 return NOTIFY_OK;
772}
773
774static struct notifier_block sdei_pm_nb = {
775 .notifier_call = sdei_pm_notifier,
776};
777
778static int sdei_device_suspend(struct device *dev)
779{
780 on_each_cpu(_ipi_mask_cpu, NULL, true);
781
782 return 0;
783}
784
785static int sdei_device_resume(struct device *dev)
786{
787 on_each_cpu(_ipi_unmask_cpu, NULL, true);
788
789 return 0;
790}
791
792/*
793 * We need all events to be reregistered when we resume from hibernate.
794 *
795 * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
796 * events during freeze, then re-register and re-enable them during thaw
797 * and restore.
798 */
799static int sdei_device_freeze(struct device *dev)
800{
801 int err;
802
803 /* unregister private events */
804 cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
805
806 err = sdei_unregister_shared();
807 if (err)
808 return err;
809
810 return 0;
811}
812
813static int sdei_device_thaw(struct device *dev)
814{
815 int err;
816
817 /* re-register shared events */
818 err = sdei_reregister_shared();
819 if (err) {
820 pr_warn("Failed to re-register shared events...\n");
821 sdei_mark_interface_broken();
822 return err;
823 }
824
825 err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
826 &sdei_cpuhp_up, &sdei_cpuhp_down);
827 if (err)
828 pr_warn("Failed to re-register CPU hotplug notifier...\n");
829
830 return err;
831}
832
833static int sdei_device_restore(struct device *dev)
834{
835 int err;
836
837 err = sdei_platform_reset();
838 if (err)
839 return err;
840
841 return sdei_device_thaw(dev);
842}
843
844static const struct dev_pm_ops sdei_pm_ops = {
845 .suspend = sdei_device_suspend,
846 .resume = sdei_device_resume,
847 .freeze = sdei_device_freeze,
848 .thaw = sdei_device_thaw,
849 .restore = sdei_device_restore,
850};
851
852/*
853 * Mask all CPUs and unregister all events on panic, reboot or kexec.
854 */
855static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
856 void *data)
857{
858 /*
859 * We are going to reset the interface, after this there is no point
860 * doing work when we take CPUs offline.
861 */
862 cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
863
864 sdei_platform_reset();
865
866 return NOTIFY_OK;
867}
868
869static struct notifier_block sdei_reboot_nb = {
870 .notifier_call = sdei_reboot_notifier,
871};
872
873static void sdei_smccc_smc(unsigned long function_id,
874 unsigned long arg0, unsigned long arg1,
875 unsigned long arg2, unsigned long arg3,
876 unsigned long arg4, struct arm_smccc_res *res)
877{
878 arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
879}
880NOKPROBE_SYMBOL(sdei_smccc_smc);
881
882static void sdei_smccc_hvc(unsigned long function_id,
883 unsigned long arg0, unsigned long arg1,
884 unsigned long arg2, unsigned long arg3,
885 unsigned long arg4, struct arm_smccc_res *res)
886{
887 arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
888}
889NOKPROBE_SYMBOL(sdei_smccc_hvc);
890
891int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb,
892 sdei_event_callback *critical_cb)
893{
894 int err;
895 u64 result;
896 u32 event_num;
897 sdei_event_callback *cb;
898
899 if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
900 return -EOPNOTSUPP;
901
902 event_num = ghes->generic->notify.vector;
903 if (event_num == 0) {
904 /*
905 * Event 0 is reserved by the specification for
906 * SDEI_EVENT_SIGNAL.
907 */
908 return -EINVAL;
909 }
910
911 err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
912 &result);
913 if (err)
914 return err;
915
916 if (result == SDEI_EVENT_PRIORITY_CRITICAL)
917 cb = critical_cb;
918 else
919 cb = normal_cb;
920
921 err = sdei_event_register(event_num, cb, ghes);
922 if (!err)
923 err = sdei_event_enable(event_num);
924
925 return err;
926}
927
928int sdei_unregister_ghes(struct ghes *ghes)
929{
930 int i;
931 int err;
932 u32 event_num = ghes->generic->notify.vector;
933
934 might_sleep();
935
936 if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
937 return -EOPNOTSUPP;
938
939 /*
940 * The event may be running on another CPU. Disable it
941 * to stop new events, then try to unregister a few times.
942 */
943 err = sdei_event_disable(event_num);
944 if (err)
945 return err;
946
947 for (i = 0; i < 3; i++) {
948 err = sdei_event_unregister(event_num);
949 if (err != -EINPROGRESS)
950 break;
951
952 schedule();
953 }
954
955 return err;
956}
957
958static int sdei_get_conduit(struct platform_device *pdev)
959{
960 const char *method;
961 struct device_node *np = pdev->dev.of_node;
962
963 sdei_firmware_call = NULL;
964 if (np) {
965 if (of_property_read_string(np, "method", &method)) {
966 pr_warn("missing \"method\" property\n");
967 return SMCCC_CONDUIT_NONE;
968 }
969
970 if (!strcmp("hvc", method)) {
971 sdei_firmware_call = &sdei_smccc_hvc;
972 return SMCCC_CONDUIT_HVC;
973 } else if (!strcmp("smc", method)) {
974 sdei_firmware_call = &sdei_smccc_smc;
975 return SMCCC_CONDUIT_SMC;
976 }
977
978 pr_warn("invalid \"method\" property: %s\n", method);
979 } else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
980 if (acpi_psci_use_hvc()) {
981 sdei_firmware_call = &sdei_smccc_hvc;
982 return SMCCC_CONDUIT_HVC;
983 } else {
984 sdei_firmware_call = &sdei_smccc_smc;
985 return SMCCC_CONDUIT_SMC;
986 }
987 }
988
989 return SMCCC_CONDUIT_NONE;
990}
991
992static int sdei_probe(struct platform_device *pdev)
993{
994 int err;
995 u64 ver = 0;
996 int conduit;
997
998 conduit = sdei_get_conduit(pdev);
999 if (!sdei_firmware_call)
1000 return 0;
1001
1002 err = sdei_api_get_version(&ver);
1003 if (err == -EOPNOTSUPP)
1004 pr_err("advertised but not implemented in platform firmware\n");
1005 if (err) {
1006 pr_err("Failed to get SDEI version: %d\n", err);
1007 sdei_mark_interface_broken();
1008 return err;
1009 }
1010
1011 pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
1012 (int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
1013 (int)SDEI_VERSION_VENDOR(ver));
1014
1015 if (SDEI_VERSION_MAJOR(ver) != 1) {
1016 pr_warn("Conflicting SDEI version detected.\n");
1017 sdei_mark_interface_broken();
1018 return -EINVAL;
1019 }
1020
1021 err = sdei_platform_reset();
1022 if (err)
1023 return err;
1024
1025 sdei_entry_point = sdei_arch_get_entry_point(conduit);
1026 if (!sdei_entry_point) {
1027 /* Not supported due to hardware or boot configuration */
1028 sdei_mark_interface_broken();
1029 return 0;
1030 }
1031
1032 err = cpu_pm_register_notifier(&sdei_pm_nb);
1033 if (err) {
1034 pr_warn("Failed to register CPU PM notifier...\n");
1035 goto error;
1036 }
1037
1038 err = register_reboot_notifier(&sdei_reboot_nb);
1039 if (err) {
1040 pr_warn("Failed to register reboot notifier...\n");
1041 goto remove_cpupm;
1042 }
1043
1044 err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
1045 &sdei_cpuhp_up, &sdei_cpuhp_down);
1046 if (err) {
1047 pr_warn("Failed to register CPU hotplug notifier...\n");
1048 goto remove_reboot;
1049 }
1050
1051 return 0;
1052
1053remove_reboot:
1054 unregister_reboot_notifier(&sdei_reboot_nb);
1055
1056remove_cpupm:
1057 cpu_pm_unregister_notifier(&sdei_pm_nb);
1058
1059error:
1060 sdei_mark_interface_broken();
1061 return err;
1062}
1063
1064static const struct of_device_id sdei_of_match[] = {
1065 { .compatible = "arm,sdei-1.0" },
1066 {}
1067};
1068
1069static struct platform_driver sdei_driver = {
1070 .driver = {
1071 .name = "sdei",
1072 .pm = &sdei_pm_ops,
1073 .of_match_table = sdei_of_match,
1074 },
1075 .probe = sdei_probe,
1076};
1077
1078static bool __init sdei_present_acpi(void)
1079{
1080 acpi_status status;
1081 struct acpi_table_header *sdei_table_header;
1082
1083 if (acpi_disabled)
1084 return false;
1085
1086 status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1087 if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1088 const char *msg = acpi_format_exception(status);
1089
1090 pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1091 }
1092 if (ACPI_FAILURE(status))
1093 return false;
1094
1095 acpi_put_table(sdei_table_header);
1096
1097 return true;
1098}
1099
1100static int __init sdei_init(void)
1101{
1102 int ret = platform_driver_register(&sdei_driver);
1103
1104 if (!ret && sdei_present_acpi()) {
1105 struct platform_device *pdev;
1106
1107 pdev = platform_device_register_simple(sdei_driver.driver.name,
1108 0, NULL, 0);
1109 if (IS_ERR(pdev))
1110 pr_info("Failed to register ACPI:SDEI platform device %ld\n",
1111 PTR_ERR(pdev));
1112 }
1113
1114 return ret;
1115}
1116
1117/*
1118 * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1119 * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1120 * by device_initcall(). We want to be called in the middle.
1121 */
1122subsys_initcall_sync(sdei_init);
1123
1124int sdei_event_handler(struct pt_regs *regs,
1125 struct sdei_registered_event *arg)
1126{
1127 int err;
1128 mm_segment_t orig_addr_limit;
1129 u32 event_num = arg->event_num;
1130
1131 /*
1132 * Save restore 'fs'.
1133 * The architecture's entry code save/restores 'fs' when taking an
1134 * exception from the kernel. This ensures addr_limit isn't inherited
1135 * if you interrupted something that allowed the uaccess routines to
1136 * access kernel memory.
1137 * Do the same here because this doesn't come via the same entry code.
1138 */
1139 orig_addr_limit = force_uaccess_begin();
1140
1141 err = arg->callback(event_num, regs, arg->callback_arg);
1142 if (err)
1143 pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1144 event_num, smp_processor_id(), err);
1145
1146 force_uaccess_end(orig_addr_limit);
1147
1148 return err;
1149}
1150NOKPROBE_SYMBOL(sdei_event_handler);
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2017 Arm Ltd.
3#define pr_fmt(fmt) "sdei: " fmt
4
5#include <linux/acpi.h>
6#include <linux/arm_sdei.h>
7#include <linux/arm-smccc.h>
8#include <linux/atomic.h>
9#include <linux/bitops.h>
10#include <linux/compiler.h>
11#include <linux/cpuhotplug.h>
12#include <linux/cpu.h>
13#include <linux/cpu_pm.h>
14#include <linux/errno.h>
15#include <linux/hardirq.h>
16#include <linux/kernel.h>
17#include <linux/kprobes.h>
18#include <linux/kvm_host.h>
19#include <linux/list.h>
20#include <linux/mutex.h>
21#include <linux/notifier.h>
22#include <linux/of.h>
23#include <linux/of_platform.h>
24#include <linux/percpu.h>
25#include <linux/platform_device.h>
26#include <linux/pm.h>
27#include <linux/ptrace.h>
28#include <linux/preempt.h>
29#include <linux/reboot.h>
30#include <linux/slab.h>
31#include <linux/smp.h>
32#include <linux/spinlock.h>
33#include <linux/uaccess.h>
34
35/*
36 * The call to use to reach the firmware.
37 */
38static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
39 unsigned long arg0, unsigned long arg1,
40 unsigned long arg2, unsigned long arg3,
41 unsigned long arg4, struct arm_smccc_res *res);
42
43/* entry point from firmware to arch asm code */
44static unsigned long sdei_entry_point;
45
46struct sdei_event {
47 /* These three are protected by the sdei_list_lock */
48 struct list_head list;
49 bool reregister;
50 bool reenable;
51
52 u32 event_num;
53 u8 type;
54 u8 priority;
55
56 /* This pointer is handed to firmware as the event argument. */
57 union {
58 /* Shared events */
59 struct sdei_registered_event *registered;
60
61 /* CPU private events */
62 struct sdei_registered_event __percpu *private_registered;
63 };
64};
65
66/* Take the mutex for any API call or modification. Take the mutex first. */
67static DEFINE_MUTEX(sdei_events_lock);
68
69/* and then hold this when modifying the list */
70static DEFINE_SPINLOCK(sdei_list_lock);
71static LIST_HEAD(sdei_list);
72
73/* Private events are registered/enabled via IPI passing one of these */
74struct sdei_crosscall_args {
75 struct sdei_event *event;
76 atomic_t errors;
77 int first_error;
78};
79
80#define CROSSCALL_INIT(arg, event) (arg.event = event, \
81 arg.first_error = 0, \
82 atomic_set(&arg.errors, 0))
83
84static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
85{
86 struct sdei_crosscall_args arg;
87
88 CROSSCALL_INIT(arg, event);
89 on_each_cpu(fn, &arg, true);
90
91 return arg.first_error;
92}
93
94static inline void
95sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
96{
97 if (err && (atomic_inc_return(&arg->errors) == 1))
98 arg->first_error = err;
99}
100
101static int sdei_to_linux_errno(unsigned long sdei_err)
102{
103 switch (sdei_err) {
104 case SDEI_NOT_SUPPORTED:
105 return -EOPNOTSUPP;
106 case SDEI_INVALID_PARAMETERS:
107 return -EINVAL;
108 case SDEI_DENIED:
109 return -EPERM;
110 case SDEI_PENDING:
111 return -EINPROGRESS;
112 case SDEI_OUT_OF_RESOURCE:
113 return -ENOMEM;
114 }
115
116 /* Not an error value ... */
117 return sdei_err;
118}
119
120/*
121 * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
122 * to translate.
123 */
124static int sdei_is_err(struct arm_smccc_res *res)
125{
126 switch (res->a0) {
127 case SDEI_NOT_SUPPORTED:
128 case SDEI_INVALID_PARAMETERS:
129 case SDEI_DENIED:
130 case SDEI_PENDING:
131 case SDEI_OUT_OF_RESOURCE:
132 return true;
133 }
134
135 return false;
136}
137
138static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
139 unsigned long arg1, unsigned long arg2,
140 unsigned long arg3, unsigned long arg4,
141 u64 *result)
142{
143 int err = 0;
144 struct arm_smccc_res res;
145
146 if (sdei_firmware_call) {
147 sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
148 &res);
149 if (sdei_is_err(&res))
150 err = sdei_to_linux_errno(res.a0);
151 } else {
152 /*
153 * !sdei_firmware_call means we failed to probe or called
154 * sdei_mark_interface_broken(). -EIO is not an error returned
155 * by sdei_to_linux_errno() and is used to suppress messages
156 * from this driver.
157 */
158 err = -EIO;
159 res.a0 = SDEI_NOT_SUPPORTED;
160 }
161
162 if (result)
163 *result = res.a0;
164
165 return err;
166}
167
168static struct sdei_event *sdei_event_find(u32 event_num)
169{
170 struct sdei_event *e, *found = NULL;
171
172 lockdep_assert_held(&sdei_events_lock);
173
174 spin_lock(&sdei_list_lock);
175 list_for_each_entry(e, &sdei_list, list) {
176 if (e->event_num == event_num) {
177 found = e;
178 break;
179 }
180 }
181 spin_unlock(&sdei_list_lock);
182
183 return found;
184}
185
186int sdei_api_event_context(u32 query, u64 *result)
187{
188 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
189 result);
190}
191NOKPROBE_SYMBOL(sdei_api_event_context);
192
193static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
194{
195 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
196 0, 0, result);
197}
198
199static struct sdei_event *sdei_event_create(u32 event_num,
200 sdei_event_callback *cb,
201 void *cb_arg)
202{
203 int err;
204 u64 result;
205 struct sdei_event *event;
206 struct sdei_registered_event *reg;
207
208 lockdep_assert_held(&sdei_events_lock);
209
210 event = kzalloc(sizeof(*event), GFP_KERNEL);
211 if (!event)
212 return ERR_PTR(-ENOMEM);
213
214 INIT_LIST_HEAD(&event->list);
215 event->event_num = event_num;
216
217 err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
218 &result);
219 if (err) {
220 kfree(event);
221 return ERR_PTR(err);
222 }
223 event->priority = result;
224
225 err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
226 &result);
227 if (err) {
228 kfree(event);
229 return ERR_PTR(err);
230 }
231 event->type = result;
232
233 if (event->type == SDEI_EVENT_TYPE_SHARED) {
234 reg = kzalloc(sizeof(*reg), GFP_KERNEL);
235 if (!reg) {
236 kfree(event);
237 return ERR_PTR(-ENOMEM);
238 }
239
240 reg->event_num = event_num;
241 reg->priority = event->priority;
242
243 reg->callback = cb;
244 reg->callback_arg = cb_arg;
245 event->registered = reg;
246 } else {
247 int cpu;
248 struct sdei_registered_event __percpu *regs;
249
250 regs = alloc_percpu(struct sdei_registered_event);
251 if (!regs) {
252 kfree(event);
253 return ERR_PTR(-ENOMEM);
254 }
255
256 for_each_possible_cpu(cpu) {
257 reg = per_cpu_ptr(regs, cpu);
258
259 reg->event_num = event->event_num;
260 reg->priority = event->priority;
261 reg->callback = cb;
262 reg->callback_arg = cb_arg;
263 }
264
265 event->private_registered = regs;
266 }
267
268 if (sdei_event_find(event_num)) {
269 kfree(event->registered);
270 kfree(event);
271 event = ERR_PTR(-EBUSY);
272 } else {
273 spin_lock(&sdei_list_lock);
274 list_add(&event->list, &sdei_list);
275 spin_unlock(&sdei_list_lock);
276 }
277
278 return event;
279}
280
281static void sdei_event_destroy(struct sdei_event *event)
282{
283 lockdep_assert_held(&sdei_events_lock);
284
285 spin_lock(&sdei_list_lock);
286 list_del(&event->list);
287 spin_unlock(&sdei_list_lock);
288
289 if (event->type == SDEI_EVENT_TYPE_SHARED)
290 kfree(event->registered);
291 else
292 free_percpu(event->private_registered);
293
294 kfree(event);
295}
296
297static int sdei_api_get_version(u64 *version)
298{
299 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
300}
301
302int sdei_mask_local_cpu(void)
303{
304 int err;
305
306 WARN_ON_ONCE(preemptible());
307
308 err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
309 if (err && err != -EIO) {
310 pr_warn_once("failed to mask CPU[%u]: %d\n",
311 smp_processor_id(), err);
312 return err;
313 }
314
315 return 0;
316}
317
318static void _ipi_mask_cpu(void *ignored)
319{
320 sdei_mask_local_cpu();
321}
322
323int sdei_unmask_local_cpu(void)
324{
325 int err;
326
327 WARN_ON_ONCE(preemptible());
328
329 err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
330 if (err && err != -EIO) {
331 pr_warn_once("failed to unmask CPU[%u]: %d\n",
332 smp_processor_id(), err);
333 return err;
334 }
335
336 return 0;
337}
338
339static void _ipi_unmask_cpu(void *ignored)
340{
341 sdei_unmask_local_cpu();
342}
343
344static void _ipi_private_reset(void *ignored)
345{
346 int err;
347
348 err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
349 NULL);
350 if (err && err != -EIO)
351 pr_warn_once("failed to reset CPU[%u]: %d\n",
352 smp_processor_id(), err);
353}
354
355static int sdei_api_shared_reset(void)
356{
357 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
358 NULL);
359}
360
361static void sdei_mark_interface_broken(void)
362{
363 pr_err("disabling SDEI firmware interface\n");
364 on_each_cpu(&_ipi_mask_cpu, NULL, true);
365 sdei_firmware_call = NULL;
366}
367
368static int sdei_platform_reset(void)
369{
370 int err;
371
372 on_each_cpu(&_ipi_private_reset, NULL, true);
373 err = sdei_api_shared_reset();
374 if (err) {
375 pr_err("Failed to reset platform: %d\n", err);
376 sdei_mark_interface_broken();
377 }
378
379 return err;
380}
381
382static int sdei_api_event_enable(u32 event_num)
383{
384 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
385 0, NULL);
386}
387
388/* Called directly by the hotplug callbacks */
389static void _local_event_enable(void *data)
390{
391 int err;
392 struct sdei_crosscall_args *arg = data;
393
394 WARN_ON_ONCE(preemptible());
395
396 err = sdei_api_event_enable(arg->event->event_num);
397
398 sdei_cross_call_return(arg, err);
399}
400
401int sdei_event_enable(u32 event_num)
402{
403 int err = -EINVAL;
404 struct sdei_event *event;
405
406 mutex_lock(&sdei_events_lock);
407 event = sdei_event_find(event_num);
408 if (!event) {
409 mutex_unlock(&sdei_events_lock);
410 return -ENOENT;
411 }
412
413 spin_lock(&sdei_list_lock);
414 event->reenable = true;
415 spin_unlock(&sdei_list_lock);
416
417 if (event->type == SDEI_EVENT_TYPE_SHARED)
418 err = sdei_api_event_enable(event->event_num);
419 else
420 err = sdei_do_cross_call(_local_event_enable, event);
421 mutex_unlock(&sdei_events_lock);
422
423 return err;
424}
425EXPORT_SYMBOL(sdei_event_enable);
426
427static int sdei_api_event_disable(u32 event_num)
428{
429 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
430 0, 0, NULL);
431}
432
433static void _ipi_event_disable(void *data)
434{
435 int err;
436 struct sdei_crosscall_args *arg = data;
437
438 err = sdei_api_event_disable(arg->event->event_num);
439
440 sdei_cross_call_return(arg, err);
441}
442
443int sdei_event_disable(u32 event_num)
444{
445 int err = -EINVAL;
446 struct sdei_event *event;
447
448 mutex_lock(&sdei_events_lock);
449 event = sdei_event_find(event_num);
450 if (!event) {
451 mutex_unlock(&sdei_events_lock);
452 return -ENOENT;
453 }
454
455 spin_lock(&sdei_list_lock);
456 event->reenable = false;
457 spin_unlock(&sdei_list_lock);
458
459 if (event->type == SDEI_EVENT_TYPE_SHARED)
460 err = sdei_api_event_disable(event->event_num);
461 else
462 err = sdei_do_cross_call(_ipi_event_disable, event);
463 mutex_unlock(&sdei_events_lock);
464
465 return err;
466}
467EXPORT_SYMBOL(sdei_event_disable);
468
469static int sdei_api_event_unregister(u32 event_num)
470{
471 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
472 0, 0, 0, NULL);
473}
474
475/* Called directly by the hotplug callbacks */
476static void _local_event_unregister(void *data)
477{
478 int err;
479 struct sdei_crosscall_args *arg = data;
480
481 WARN_ON_ONCE(preemptible());
482
483 err = sdei_api_event_unregister(arg->event->event_num);
484
485 sdei_cross_call_return(arg, err);
486}
487
488static int _sdei_event_unregister(struct sdei_event *event)
489{
490 lockdep_assert_held(&sdei_events_lock);
491
492 spin_lock(&sdei_list_lock);
493 event->reregister = false;
494 event->reenable = false;
495 spin_unlock(&sdei_list_lock);
496
497 if (event->type == SDEI_EVENT_TYPE_SHARED)
498 return sdei_api_event_unregister(event->event_num);
499
500 return sdei_do_cross_call(_local_event_unregister, event);
501}
502
503int sdei_event_unregister(u32 event_num)
504{
505 int err;
506 struct sdei_event *event;
507
508 WARN_ON(in_nmi());
509
510 mutex_lock(&sdei_events_lock);
511 event = sdei_event_find(event_num);
512 do {
513 if (!event) {
514 pr_warn("Event %u not registered\n", event_num);
515 err = -ENOENT;
516 break;
517 }
518
519 err = _sdei_event_unregister(event);
520 if (err)
521 break;
522
523 sdei_event_destroy(event);
524 } while (0);
525 mutex_unlock(&sdei_events_lock);
526
527 return err;
528}
529EXPORT_SYMBOL(sdei_event_unregister);
530
531/*
532 * unregister events, but don't destroy them as they are re-registered by
533 * sdei_reregister_shared().
534 */
535static int sdei_unregister_shared(void)
536{
537 int err = 0;
538 struct sdei_event *event;
539
540 mutex_lock(&sdei_events_lock);
541 spin_lock(&sdei_list_lock);
542 list_for_each_entry(event, &sdei_list, list) {
543 if (event->type != SDEI_EVENT_TYPE_SHARED)
544 continue;
545
546 err = _sdei_event_unregister(event);
547 if (err)
548 break;
549 }
550 spin_unlock(&sdei_list_lock);
551 mutex_unlock(&sdei_events_lock);
552
553 return err;
554}
555
556static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
557 void *arg, u64 flags, u64 affinity)
558{
559 return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
560 (unsigned long)entry_point, (unsigned long)arg,
561 flags, affinity, NULL);
562}
563
564/* Called directly by the hotplug callbacks */
565static void _local_event_register(void *data)
566{
567 int err;
568 struct sdei_registered_event *reg;
569 struct sdei_crosscall_args *arg = data;
570
571 WARN_ON(preemptible());
572
573 reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
574 err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
575 reg, 0, 0);
576
577 sdei_cross_call_return(arg, err);
578}
579
580static int _sdei_event_register(struct sdei_event *event)
581{
582 int err;
583
584 lockdep_assert_held(&sdei_events_lock);
585
586 spin_lock(&sdei_list_lock);
587 event->reregister = true;
588 spin_unlock(&sdei_list_lock);
589
590 if (event->type == SDEI_EVENT_TYPE_SHARED)
591 return sdei_api_event_register(event->event_num,
592 sdei_entry_point,
593 event->registered,
594 SDEI_EVENT_REGISTER_RM_ANY, 0);
595
596
597 err = sdei_do_cross_call(_local_event_register, event);
598 if (err) {
599 spin_lock(&sdei_list_lock);
600 event->reregister = false;
601 event->reenable = false;
602 spin_unlock(&sdei_list_lock);
603
604 sdei_do_cross_call(_local_event_unregister, event);
605 }
606
607 return err;
608}
609
610int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
611{
612 int err;
613 struct sdei_event *event;
614
615 WARN_ON(in_nmi());
616
617 mutex_lock(&sdei_events_lock);
618 do {
619 if (sdei_event_find(event_num)) {
620 pr_warn("Event %u already registered\n", event_num);
621 err = -EBUSY;
622 break;
623 }
624
625 event = sdei_event_create(event_num, cb, arg);
626 if (IS_ERR(event)) {
627 err = PTR_ERR(event);
628 pr_warn("Failed to create event %u: %d\n", event_num,
629 err);
630 break;
631 }
632
633 err = _sdei_event_register(event);
634 if (err) {
635 sdei_event_destroy(event);
636 pr_warn("Failed to register event %u: %d\n", event_num,
637 err);
638 }
639 } while (0);
640 mutex_unlock(&sdei_events_lock);
641
642 return err;
643}
644EXPORT_SYMBOL(sdei_event_register);
645
646static int sdei_reregister_event(struct sdei_event *event)
647{
648 int err;
649
650 lockdep_assert_held(&sdei_events_lock);
651
652 err = _sdei_event_register(event);
653 if (err) {
654 pr_err("Failed to re-register event %u\n", event->event_num);
655 sdei_event_destroy(event);
656 return err;
657 }
658
659 if (event->reenable) {
660 if (event->type == SDEI_EVENT_TYPE_SHARED)
661 err = sdei_api_event_enable(event->event_num);
662 else
663 err = sdei_do_cross_call(_local_event_enable, event);
664 }
665
666 if (err)
667 pr_err("Failed to re-enable event %u\n", event->event_num);
668
669 return err;
670}
671
672static int sdei_reregister_shared(void)
673{
674 int err = 0;
675 struct sdei_event *event;
676
677 mutex_lock(&sdei_events_lock);
678 spin_lock(&sdei_list_lock);
679 list_for_each_entry(event, &sdei_list, list) {
680 if (event->type != SDEI_EVENT_TYPE_SHARED)
681 continue;
682
683 if (event->reregister) {
684 err = sdei_reregister_event(event);
685 if (err)
686 break;
687 }
688 }
689 spin_unlock(&sdei_list_lock);
690 mutex_unlock(&sdei_events_lock);
691
692 return err;
693}
694
695static int sdei_cpuhp_down(unsigned int cpu)
696{
697 struct sdei_event *event;
698 struct sdei_crosscall_args arg;
699
700 /* un-register private events */
701 spin_lock(&sdei_list_lock);
702 list_for_each_entry(event, &sdei_list, list) {
703 if (event->type == SDEI_EVENT_TYPE_SHARED)
704 continue;
705
706 CROSSCALL_INIT(arg, event);
707 /* call the cross-call function locally... */
708 _local_event_unregister(&arg);
709 if (arg.first_error)
710 pr_err("Failed to unregister event %u: %d\n",
711 event->event_num, arg.first_error);
712 }
713 spin_unlock(&sdei_list_lock);
714
715 return sdei_mask_local_cpu();
716}
717
718static int sdei_cpuhp_up(unsigned int cpu)
719{
720 struct sdei_event *event;
721 struct sdei_crosscall_args arg;
722
723 /* re-register/enable private events */
724 spin_lock(&sdei_list_lock);
725 list_for_each_entry(event, &sdei_list, list) {
726 if (event->type == SDEI_EVENT_TYPE_SHARED)
727 continue;
728
729 if (event->reregister) {
730 CROSSCALL_INIT(arg, event);
731 /* call the cross-call function locally... */
732 _local_event_register(&arg);
733 if (arg.first_error)
734 pr_err("Failed to re-register event %u: %d\n",
735 event->event_num, arg.first_error);
736 }
737
738 if (event->reenable) {
739 CROSSCALL_INIT(arg, event);
740 _local_event_enable(&arg);
741 if (arg.first_error)
742 pr_err("Failed to re-enable event %u: %d\n",
743 event->event_num, arg.first_error);
744 }
745 }
746 spin_unlock(&sdei_list_lock);
747
748 return sdei_unmask_local_cpu();
749}
750
751/* When entering idle, mask/unmask events for this cpu */
752static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
753 void *data)
754{
755 int rv;
756
757 switch (action) {
758 case CPU_PM_ENTER:
759 rv = sdei_mask_local_cpu();
760 break;
761 case CPU_PM_EXIT:
762 case CPU_PM_ENTER_FAILED:
763 rv = sdei_unmask_local_cpu();
764 break;
765 default:
766 return NOTIFY_DONE;
767 }
768
769 if (rv)
770 return notifier_from_errno(rv);
771
772 return NOTIFY_OK;
773}
774
775static struct notifier_block sdei_pm_nb = {
776 .notifier_call = sdei_pm_notifier,
777};
778
779static int sdei_device_suspend(struct device *dev)
780{
781 on_each_cpu(_ipi_mask_cpu, NULL, true);
782
783 return 0;
784}
785
786static int sdei_device_resume(struct device *dev)
787{
788 on_each_cpu(_ipi_unmask_cpu, NULL, true);
789
790 return 0;
791}
792
793/*
794 * We need all events to be reregistered when we resume from hibernate.
795 *
796 * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
797 * events during freeze, then re-register and re-enable them during thaw
798 * and restore.
799 */
800static int sdei_device_freeze(struct device *dev)
801{
802 int err;
803
804 /* unregister private events */
805 cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
806
807 err = sdei_unregister_shared();
808 if (err)
809 return err;
810
811 return 0;
812}
813
814static int sdei_device_thaw(struct device *dev)
815{
816 int err;
817
818 /* re-register shared events */
819 err = sdei_reregister_shared();
820 if (err) {
821 pr_warn("Failed to re-register shared events...\n");
822 sdei_mark_interface_broken();
823 return err;
824 }
825
826 err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
827 &sdei_cpuhp_up, &sdei_cpuhp_down);
828 if (err)
829 pr_warn("Failed to re-register CPU hotplug notifier...\n");
830
831 return err;
832}
833
834static int sdei_device_restore(struct device *dev)
835{
836 int err;
837
838 err = sdei_platform_reset();
839 if (err)
840 return err;
841
842 return sdei_device_thaw(dev);
843}
844
845static const struct dev_pm_ops sdei_pm_ops = {
846 .suspend = sdei_device_suspend,
847 .resume = sdei_device_resume,
848 .freeze = sdei_device_freeze,
849 .thaw = sdei_device_thaw,
850 .restore = sdei_device_restore,
851};
852
853/*
854 * Mask all CPUs and unregister all events on panic, reboot or kexec.
855 */
856static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
857 void *data)
858{
859 /*
860 * We are going to reset the interface, after this there is no point
861 * doing work when we take CPUs offline.
862 */
863 cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
864
865 sdei_platform_reset();
866
867 return NOTIFY_OK;
868}
869
870static struct notifier_block sdei_reboot_nb = {
871 .notifier_call = sdei_reboot_notifier,
872};
873
874static void sdei_smccc_smc(unsigned long function_id,
875 unsigned long arg0, unsigned long arg1,
876 unsigned long arg2, unsigned long arg3,
877 unsigned long arg4, struct arm_smccc_res *res)
878{
879 arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
880}
881
882static void sdei_smccc_hvc(unsigned long function_id,
883 unsigned long arg0, unsigned long arg1,
884 unsigned long arg2, unsigned long arg3,
885 unsigned long arg4, struct arm_smccc_res *res)
886{
887 arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
888}
889
890static int sdei_get_conduit(struct platform_device *pdev)
891{
892 const char *method;
893 struct device_node *np = pdev->dev.of_node;
894
895 sdei_firmware_call = NULL;
896 if (np) {
897 if (of_property_read_string(np, "method", &method)) {
898 pr_warn("missing \"method\" property\n");
899 return CONDUIT_INVALID;
900 }
901
902 if (!strcmp("hvc", method)) {
903 sdei_firmware_call = &sdei_smccc_hvc;
904 return CONDUIT_HVC;
905 } else if (!strcmp("smc", method)) {
906 sdei_firmware_call = &sdei_smccc_smc;
907 return CONDUIT_SMC;
908 }
909
910 pr_warn("invalid \"method\" property: %s\n", method);
911 } else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
912 if (acpi_psci_use_hvc()) {
913 sdei_firmware_call = &sdei_smccc_hvc;
914 return CONDUIT_HVC;
915 } else {
916 sdei_firmware_call = &sdei_smccc_smc;
917 return CONDUIT_SMC;
918 }
919 }
920
921 return CONDUIT_INVALID;
922}
923
924static int sdei_probe(struct platform_device *pdev)
925{
926 int err;
927 u64 ver = 0;
928 int conduit;
929
930 conduit = sdei_get_conduit(pdev);
931 if (!sdei_firmware_call)
932 return 0;
933
934 err = sdei_api_get_version(&ver);
935 if (err == -EOPNOTSUPP)
936 pr_err("advertised but not implemented in platform firmware\n");
937 if (err) {
938 pr_err("Failed to get SDEI version: %d\n", err);
939 sdei_mark_interface_broken();
940 return err;
941 }
942
943 pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
944 (int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
945 (int)SDEI_VERSION_VENDOR(ver));
946
947 if (SDEI_VERSION_MAJOR(ver) != 1) {
948 pr_warn("Conflicting SDEI version detected.\n");
949 sdei_mark_interface_broken();
950 return -EINVAL;
951 }
952
953 err = sdei_platform_reset();
954 if (err)
955 return err;
956
957 sdei_entry_point = sdei_arch_get_entry_point(conduit);
958 if (!sdei_entry_point) {
959 /* Not supported due to hardware or boot configuration */
960 sdei_mark_interface_broken();
961 return 0;
962 }
963
964 err = cpu_pm_register_notifier(&sdei_pm_nb);
965 if (err) {
966 pr_warn("Failed to register CPU PM notifier...\n");
967 goto error;
968 }
969
970 err = register_reboot_notifier(&sdei_reboot_nb);
971 if (err) {
972 pr_warn("Failed to register reboot notifier...\n");
973 goto remove_cpupm;
974 }
975
976 err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
977 &sdei_cpuhp_up, &sdei_cpuhp_down);
978 if (err) {
979 pr_warn("Failed to register CPU hotplug notifier...\n");
980 goto remove_reboot;
981 }
982
983 return 0;
984
985remove_reboot:
986 unregister_reboot_notifier(&sdei_reboot_nb);
987
988remove_cpupm:
989 cpu_pm_unregister_notifier(&sdei_pm_nb);
990
991error:
992 sdei_mark_interface_broken();
993 return err;
994}
995
996static const struct of_device_id sdei_of_match[] = {
997 { .compatible = "arm,sdei-1.0" },
998 {}
999};
1000
1001static struct platform_driver sdei_driver = {
1002 .driver = {
1003 .name = "sdei",
1004 .pm = &sdei_pm_ops,
1005 .of_match_table = sdei_of_match,
1006 },
1007 .probe = sdei_probe,
1008};
1009
1010static bool __init sdei_present_dt(void)
1011{
1012 struct platform_device *pdev;
1013 struct device_node *np, *fw_np;
1014
1015 fw_np = of_find_node_by_name(NULL, "firmware");
1016 if (!fw_np)
1017 return false;
1018
1019 np = of_find_matching_node(fw_np, sdei_of_match);
1020 of_node_put(fw_np);
1021 if (!np)
1022 return false;
1023
1024 pdev = of_platform_device_create(np, sdei_driver.driver.name, NULL);
1025 of_node_put(np);
1026 if (!pdev)
1027 return false;
1028
1029 return true;
1030}
1031
1032static bool __init sdei_present_acpi(void)
1033{
1034 acpi_status status;
1035 struct platform_device *pdev;
1036 struct acpi_table_header *sdei_table_header;
1037
1038 if (acpi_disabled)
1039 return false;
1040
1041 status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1042 if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1043 const char *msg = acpi_format_exception(status);
1044
1045 pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1046 }
1047 if (ACPI_FAILURE(status))
1048 return false;
1049
1050 pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
1051 0);
1052 if (IS_ERR(pdev))
1053 return false;
1054
1055 return true;
1056}
1057
1058static int __init sdei_init(void)
1059{
1060 if (sdei_present_dt() || sdei_present_acpi())
1061 platform_driver_register(&sdei_driver);
1062
1063 return 0;
1064}
1065
1066/*
1067 * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1068 * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1069 * by device_initcall(). We want to be called in the middle.
1070 */
1071subsys_initcall_sync(sdei_init);
1072
1073int sdei_event_handler(struct pt_regs *regs,
1074 struct sdei_registered_event *arg)
1075{
1076 int err;
1077 mm_segment_t orig_addr_limit;
1078 u32 event_num = arg->event_num;
1079
1080 orig_addr_limit = get_fs();
1081 set_fs(USER_DS);
1082
1083 err = arg->callback(event_num, regs, arg->callback_arg);
1084 if (err)
1085 pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1086 event_num, smp_processor_id(), err);
1087
1088 set_fs(orig_addr_limit);
1089
1090 return err;
1091}
1092NOKPROBE_SYMBOL(sdei_event_handler);