Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Machine check exception handling.
4 *
5 * Copyright 2013 IBM Corporation
6 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
7 */
8
9#undef DEBUG
10#define pr_fmt(fmt) "mce: " fmt
11
12#include <linux/hardirq.h>
13#include <linux/types.h>
14#include <linux/ptrace.h>
15#include <linux/percpu.h>
16#include <linux/export.h>
17#include <linux/irq_work.h>
18#include <linux/extable.h>
19#include <linux/ftrace.h>
20#include <linux/memblock.h>
21#include <linux/of.h>
22
23#include <asm/interrupt.h>
24#include <asm/machdep.h>
25#include <asm/mce.h>
26#include <asm/nmi.h>
27
28#include "setup.h"
29
30static void machine_check_ue_event(struct machine_check_event *evt);
31static void machine_process_ue_event(struct work_struct *work);
32
33static DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
34
35static BLOCKING_NOTIFIER_HEAD(mce_notifier_list);
36
37int mce_register_notifier(struct notifier_block *nb)
38{
39 return blocking_notifier_chain_register(&mce_notifier_list, nb);
40}
41EXPORT_SYMBOL_GPL(mce_register_notifier);
42
43int mce_unregister_notifier(struct notifier_block *nb)
44{
45 return blocking_notifier_chain_unregister(&mce_notifier_list, nb);
46}
47EXPORT_SYMBOL_GPL(mce_unregister_notifier);
48
49static void mce_set_error_info(struct machine_check_event *mce,
50 struct mce_error_info *mce_err)
51{
52 mce->error_type = mce_err->error_type;
53 switch (mce_err->error_type) {
54 case MCE_ERROR_TYPE_UE:
55 mce->u.ue_error.ue_error_type = mce_err->u.ue_error_type;
56 break;
57 case MCE_ERROR_TYPE_SLB:
58 mce->u.slb_error.slb_error_type = mce_err->u.slb_error_type;
59 break;
60 case MCE_ERROR_TYPE_ERAT:
61 mce->u.erat_error.erat_error_type = mce_err->u.erat_error_type;
62 break;
63 case MCE_ERROR_TYPE_TLB:
64 mce->u.tlb_error.tlb_error_type = mce_err->u.tlb_error_type;
65 break;
66 case MCE_ERROR_TYPE_USER:
67 mce->u.user_error.user_error_type = mce_err->u.user_error_type;
68 break;
69 case MCE_ERROR_TYPE_RA:
70 mce->u.ra_error.ra_error_type = mce_err->u.ra_error_type;
71 break;
72 case MCE_ERROR_TYPE_LINK:
73 mce->u.link_error.link_error_type = mce_err->u.link_error_type;
74 break;
75 case MCE_ERROR_TYPE_UNKNOWN:
76 default:
77 break;
78 }
79}
80
81void mce_irq_work_queue(void)
82{
83 /* Raise decrementer interrupt */
84 arch_irq_work_raise();
85 set_mce_pending_irq_work();
86}
87
88/*
89 * Decode and save high level MCE information into per cpu buffer which
90 * is an array of machine_check_event structure.
91 */
92void save_mce_event(struct pt_regs *regs, long handled,
93 struct mce_error_info *mce_err,
94 uint64_t nip, uint64_t addr, uint64_t phys_addr)
95{
96 int index = local_paca->mce_info->mce_nest_count++;
97 struct machine_check_event *mce;
98
99 mce = &local_paca->mce_info->mce_event[index];
100 /*
101 * Return if we don't have enough space to log mce event.
102 * mce_nest_count may go beyond MAX_MC_EVT but that's ok,
103 * the check below will stop buffer overrun.
104 */
105 if (index >= MAX_MC_EVT)
106 return;
107
108 /* Populate generic machine check info */
109 mce->version = MCE_V1;
110 mce->srr0 = nip;
111 mce->srr1 = regs->msr;
112 mce->gpr3 = regs->gpr[3];
113 mce->in_use = 1;
114 mce->cpu = get_paca()->paca_index;
115
116 /* Mark it recovered if we have handled it and MSR(RI=1). */
117 if (handled && (regs->msr & MSR_RI))
118 mce->disposition = MCE_DISPOSITION_RECOVERED;
119 else
120 mce->disposition = MCE_DISPOSITION_NOT_RECOVERED;
121
122 mce->initiator = mce_err->initiator;
123 mce->severity = mce_err->severity;
124 mce->sync_error = mce_err->sync_error;
125 mce->error_class = mce_err->error_class;
126
127 /*
128 * Populate the mce error_type and type-specific error_type.
129 */
130 mce_set_error_info(mce, mce_err);
131 if (mce->error_type == MCE_ERROR_TYPE_UE)
132 mce->u.ue_error.ignore_event = mce_err->ignore_event;
133
134 if (!addr)
135 return;
136
137 if (mce->error_type == MCE_ERROR_TYPE_TLB) {
138 mce->u.tlb_error.effective_address_provided = true;
139 mce->u.tlb_error.effective_address = addr;
140 } else if (mce->error_type == MCE_ERROR_TYPE_SLB) {
141 mce->u.slb_error.effective_address_provided = true;
142 mce->u.slb_error.effective_address = addr;
143 } else if (mce->error_type == MCE_ERROR_TYPE_ERAT) {
144 mce->u.erat_error.effective_address_provided = true;
145 mce->u.erat_error.effective_address = addr;
146 } else if (mce->error_type == MCE_ERROR_TYPE_USER) {
147 mce->u.user_error.effective_address_provided = true;
148 mce->u.user_error.effective_address = addr;
149 } else if (mce->error_type == MCE_ERROR_TYPE_RA) {
150 mce->u.ra_error.effective_address_provided = true;
151 mce->u.ra_error.effective_address = addr;
152 } else if (mce->error_type == MCE_ERROR_TYPE_LINK) {
153 mce->u.link_error.effective_address_provided = true;
154 mce->u.link_error.effective_address = addr;
155 } else if (mce->error_type == MCE_ERROR_TYPE_UE) {
156 mce->u.ue_error.effective_address_provided = true;
157 mce->u.ue_error.effective_address = addr;
158 if (phys_addr != ULONG_MAX) {
159 mce->u.ue_error.physical_address_provided = true;
160 mce->u.ue_error.physical_address = phys_addr;
161 machine_check_ue_event(mce);
162 }
163 }
164 return;
165}
166
167/*
168 * get_mce_event:
169 * mce Pointer to machine_check_event structure to be filled.
170 * release Flag to indicate whether to free the event slot or not.
171 * 0 <= do not release the mce event. Caller will invoke
172 * release_mce_event() once event has been consumed.
173 * 1 <= release the slot.
174 *
175 * return 1 = success
176 * 0 = failure
177 *
178 * get_mce_event() will be called by platform specific machine check
179 * handle routine and in KVM.
180 * When we call get_mce_event(), we are still in interrupt context and
181 * preemption will not be scheduled until ret_from_expect() routine
182 * is called.
183 */
184int get_mce_event(struct machine_check_event *mce, bool release)
185{
186 int index = local_paca->mce_info->mce_nest_count - 1;
187 struct machine_check_event *mc_evt;
188 int ret = 0;
189
190 /* Sanity check */
191 if (index < 0)
192 return ret;
193
194 /* Check if we have MCE info to process. */
195 if (index < MAX_MC_EVT) {
196 mc_evt = &local_paca->mce_info->mce_event[index];
197 /* Copy the event structure and release the original */
198 if (mce)
199 *mce = *mc_evt;
200 if (release)
201 mc_evt->in_use = 0;
202 ret = 1;
203 }
204 /* Decrement the count to free the slot. */
205 if (release)
206 local_paca->mce_info->mce_nest_count--;
207
208 return ret;
209}
210
211void release_mce_event(void)
212{
213 get_mce_event(NULL, true);
214}
215
216static void machine_check_ue_work(void)
217{
218 schedule_work(&mce_ue_event_work);
219}
220
221/*
222 * Queue up the MCE event which then can be handled later.
223 */
224static void machine_check_ue_event(struct machine_check_event *evt)
225{
226 int index;
227
228 index = local_paca->mce_info->mce_ue_count++;
229 /* If queue is full, just return for now. */
230 if (index >= MAX_MC_EVT) {
231 local_paca->mce_info->mce_ue_count--;
232 return;
233 }
234 memcpy(&local_paca->mce_info->mce_ue_event_queue[index],
235 evt, sizeof(*evt));
236
237 /* Queue work to process this event later. */
238 mce_irq_work_queue();
239}
240
241/*
242 * Queue up the MCE event which then can be handled later.
243 */
244void machine_check_queue_event(void)
245{
246 int index;
247 struct machine_check_event evt;
248
249 if (!get_mce_event(&evt, MCE_EVENT_RELEASE))
250 return;
251
252 index = local_paca->mce_info->mce_queue_count++;
253 /* If queue is full, just return for now. */
254 if (index >= MAX_MC_EVT) {
255 local_paca->mce_info->mce_queue_count--;
256 return;
257 }
258 memcpy(&local_paca->mce_info->mce_event_queue[index],
259 &evt, sizeof(evt));
260
261 mce_irq_work_queue();
262}
263
264void mce_common_process_ue(struct pt_regs *regs,
265 struct mce_error_info *mce_err)
266{
267 const struct exception_table_entry *entry;
268
269 entry = search_kernel_exception_table(regs->nip);
270 if (entry) {
271 mce_err->ignore_event = true;
272 regs_set_return_ip(regs, extable_fixup(entry));
273 }
274}
275
276/*
277 * process pending MCE event from the mce event queue. This function will be
278 * called during syscall exit.
279 */
280static void machine_process_ue_event(struct work_struct *work)
281{
282 int index;
283 struct machine_check_event *evt;
284
285 while (local_paca->mce_info->mce_ue_count > 0) {
286 index = local_paca->mce_info->mce_ue_count - 1;
287 evt = &local_paca->mce_info->mce_ue_event_queue[index];
288 blocking_notifier_call_chain(&mce_notifier_list, 0, evt);
289#ifdef CONFIG_MEMORY_FAILURE
290 /*
291 * This should probably queued elsewhere, but
292 * oh! well
293 *
294 * Don't report this machine check because the caller has a
295 * asked us to ignore the event, it has a fixup handler which
296 * will do the appropriate error handling and reporting.
297 */
298 if (evt->error_type == MCE_ERROR_TYPE_UE) {
299 if (evt->u.ue_error.ignore_event) {
300 local_paca->mce_info->mce_ue_count--;
301 continue;
302 }
303
304 if (evt->u.ue_error.physical_address_provided) {
305 unsigned long pfn;
306
307 pfn = evt->u.ue_error.physical_address >>
308 PAGE_SHIFT;
309 memory_failure(pfn, 0);
310 } else
311 pr_warn("Failed to identify bad address from "
312 "where the uncorrectable error (UE) "
313 "was generated\n");
314 }
315#endif
316 local_paca->mce_info->mce_ue_count--;
317 }
318}
319/*
320 * process pending MCE event from the mce event queue. This function will be
321 * called during syscall exit.
322 */
323static void machine_check_process_queued_event(void)
324{
325 int index;
326 struct machine_check_event *evt;
327
328 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
329
330 /*
331 * For now just print it to console.
332 * TODO: log this error event to FSP or nvram.
333 */
334 while (local_paca->mce_info->mce_queue_count > 0) {
335 index = local_paca->mce_info->mce_queue_count - 1;
336 evt = &local_paca->mce_info->mce_event_queue[index];
337
338 if (evt->error_type == MCE_ERROR_TYPE_UE &&
339 evt->u.ue_error.ignore_event) {
340 local_paca->mce_info->mce_queue_count--;
341 continue;
342 }
343 machine_check_print_event_info(evt, false, false);
344 local_paca->mce_info->mce_queue_count--;
345 }
346}
347
348void set_mce_pending_irq_work(void)
349{
350 local_paca->mce_pending_irq_work = 1;
351}
352
353void clear_mce_pending_irq_work(void)
354{
355 local_paca->mce_pending_irq_work = 0;
356}
357
358void mce_run_irq_context_handlers(void)
359{
360 if (unlikely(local_paca->mce_pending_irq_work)) {
361 if (ppc_md.machine_check_log_err)
362 ppc_md.machine_check_log_err();
363 machine_check_process_queued_event();
364 machine_check_ue_work();
365 clear_mce_pending_irq_work();
366 }
367}
368
369void machine_check_print_event_info(struct machine_check_event *evt,
370 bool user_mode, bool in_guest)
371{
372 const char *level, *sevstr, *subtype, *err_type, *initiator;
373 uint64_t ea = 0, pa = 0;
374 int n = 0;
375 char dar_str[50];
376 char pa_str[50];
377 static const char *mc_ue_types[] = {
378 "Indeterminate",
379 "Instruction fetch",
380 "Page table walk ifetch",
381 "Load/Store",
382 "Page table walk Load/Store",
383 };
384 static const char *mc_slb_types[] = {
385 "Indeterminate",
386 "Parity",
387 "Multihit",
388 };
389 static const char *mc_erat_types[] = {
390 "Indeterminate",
391 "Parity",
392 "Multihit",
393 };
394 static const char *mc_tlb_types[] = {
395 "Indeterminate",
396 "Parity",
397 "Multihit",
398 };
399 static const char *mc_user_types[] = {
400 "Indeterminate",
401 "tlbie(l) invalid",
402 "scv invalid",
403 };
404 static const char *mc_ra_types[] = {
405 "Indeterminate",
406 "Instruction fetch (bad)",
407 "Instruction fetch (foreign/control memory)",
408 "Page table walk ifetch (bad)",
409 "Page table walk ifetch (foreign/control memory)",
410 "Load (bad)",
411 "Store (bad)",
412 "Page table walk Load/Store (bad)",
413 "Page table walk Load/Store (foreign/control memory)",
414 "Load/Store (foreign/control memory)",
415 };
416 static const char *mc_link_types[] = {
417 "Indeterminate",
418 "Instruction fetch (timeout)",
419 "Page table walk ifetch (timeout)",
420 "Load (timeout)",
421 "Store (timeout)",
422 "Page table walk Load/Store (timeout)",
423 };
424 static const char *mc_error_class[] = {
425 "Unknown",
426 "Hardware error",
427 "Probable Hardware error (some chance of software cause)",
428 "Software error",
429 "Probable Software error (some chance of hardware cause)",
430 };
431
432 /* Print things out */
433 if (evt->version != MCE_V1) {
434 pr_err("Machine Check Exception, Unknown event version %d !\n",
435 evt->version);
436 return;
437 }
438 switch (evt->severity) {
439 case MCE_SEV_NO_ERROR:
440 level = KERN_INFO;
441 sevstr = "Harmless";
442 break;
443 case MCE_SEV_WARNING:
444 level = KERN_WARNING;
445 sevstr = "Warning";
446 break;
447 case MCE_SEV_SEVERE:
448 level = KERN_ERR;
449 sevstr = "Severe";
450 break;
451 case MCE_SEV_FATAL:
452 default:
453 level = KERN_ERR;
454 sevstr = "Fatal";
455 break;
456 }
457
458 switch(evt->initiator) {
459 case MCE_INITIATOR_CPU:
460 initiator = "CPU";
461 break;
462 case MCE_INITIATOR_PCI:
463 initiator = "PCI";
464 break;
465 case MCE_INITIATOR_ISA:
466 initiator = "ISA";
467 break;
468 case MCE_INITIATOR_MEMORY:
469 initiator = "Memory";
470 break;
471 case MCE_INITIATOR_POWERMGM:
472 initiator = "Power Management";
473 break;
474 case MCE_INITIATOR_UNKNOWN:
475 default:
476 initiator = "Unknown";
477 break;
478 }
479
480 switch (evt->error_type) {
481 case MCE_ERROR_TYPE_UE:
482 err_type = "UE";
483 subtype = evt->u.ue_error.ue_error_type <
484 ARRAY_SIZE(mc_ue_types) ?
485 mc_ue_types[evt->u.ue_error.ue_error_type]
486 : "Unknown";
487 if (evt->u.ue_error.effective_address_provided)
488 ea = evt->u.ue_error.effective_address;
489 if (evt->u.ue_error.physical_address_provided)
490 pa = evt->u.ue_error.physical_address;
491 break;
492 case MCE_ERROR_TYPE_SLB:
493 err_type = "SLB";
494 subtype = evt->u.slb_error.slb_error_type <
495 ARRAY_SIZE(mc_slb_types) ?
496 mc_slb_types[evt->u.slb_error.slb_error_type]
497 : "Unknown";
498 if (evt->u.slb_error.effective_address_provided)
499 ea = evt->u.slb_error.effective_address;
500 break;
501 case MCE_ERROR_TYPE_ERAT:
502 err_type = "ERAT";
503 subtype = evt->u.erat_error.erat_error_type <
504 ARRAY_SIZE(mc_erat_types) ?
505 mc_erat_types[evt->u.erat_error.erat_error_type]
506 : "Unknown";
507 if (evt->u.erat_error.effective_address_provided)
508 ea = evt->u.erat_error.effective_address;
509 break;
510 case MCE_ERROR_TYPE_TLB:
511 err_type = "TLB";
512 subtype = evt->u.tlb_error.tlb_error_type <
513 ARRAY_SIZE(mc_tlb_types) ?
514 mc_tlb_types[evt->u.tlb_error.tlb_error_type]
515 : "Unknown";
516 if (evt->u.tlb_error.effective_address_provided)
517 ea = evt->u.tlb_error.effective_address;
518 break;
519 case MCE_ERROR_TYPE_USER:
520 err_type = "User";
521 subtype = evt->u.user_error.user_error_type <
522 ARRAY_SIZE(mc_user_types) ?
523 mc_user_types[evt->u.user_error.user_error_type]
524 : "Unknown";
525 if (evt->u.user_error.effective_address_provided)
526 ea = evt->u.user_error.effective_address;
527 break;
528 case MCE_ERROR_TYPE_RA:
529 err_type = "Real address";
530 subtype = evt->u.ra_error.ra_error_type <
531 ARRAY_SIZE(mc_ra_types) ?
532 mc_ra_types[evt->u.ra_error.ra_error_type]
533 : "Unknown";
534 if (evt->u.ra_error.effective_address_provided)
535 ea = evt->u.ra_error.effective_address;
536 break;
537 case MCE_ERROR_TYPE_LINK:
538 err_type = "Link";
539 subtype = evt->u.link_error.link_error_type <
540 ARRAY_SIZE(mc_link_types) ?
541 mc_link_types[evt->u.link_error.link_error_type]
542 : "Unknown";
543 if (evt->u.link_error.effective_address_provided)
544 ea = evt->u.link_error.effective_address;
545 break;
546 case MCE_ERROR_TYPE_DCACHE:
547 err_type = "D-Cache";
548 subtype = "Unknown";
549 break;
550 case MCE_ERROR_TYPE_ICACHE:
551 err_type = "I-Cache";
552 subtype = "Unknown";
553 break;
554 default:
555 case MCE_ERROR_TYPE_UNKNOWN:
556 err_type = "Unknown";
557 subtype = "";
558 break;
559 }
560
561 dar_str[0] = pa_str[0] = '\0';
562 if (ea && evt->srr0 != ea) {
563 /* Load/Store address */
564 n = sprintf(dar_str, "DAR: %016llx ", ea);
565 if (pa)
566 sprintf(dar_str + n, "paddr: %016llx ", pa);
567 } else if (pa) {
568 sprintf(pa_str, " paddr: %016llx", pa);
569 }
570
571 printk("%sMCE: CPU%d: machine check (%s) %s %s %s %s[%s]\n",
572 level, evt->cpu, sevstr, in_guest ? "Guest" : "",
573 err_type, subtype, dar_str,
574 evt->disposition == MCE_DISPOSITION_RECOVERED ?
575 "Recovered" : "Not recovered");
576
577 if (in_guest || user_mode) {
578 printk("%sMCE: CPU%d: PID: %d Comm: %s %sNIP: [%016llx]%s\n",
579 level, evt->cpu, current->pid, current->comm,
580 in_guest ? "Guest " : "", evt->srr0, pa_str);
581 } else {
582 printk("%sMCE: CPU%d: NIP: [%016llx] %pS%s\n",
583 level, evt->cpu, evt->srr0, (void *)evt->srr0, pa_str);
584 }
585
586 printk("%sMCE: CPU%d: Initiator %s\n", level, evt->cpu, initiator);
587
588 subtype = evt->error_class < ARRAY_SIZE(mc_error_class) ?
589 mc_error_class[evt->error_class] : "Unknown";
590 printk("%sMCE: CPU%d: %s\n", level, evt->cpu, subtype);
591
592#ifdef CONFIG_PPC_64S_HASH_MMU
593 /* Display faulty slb contents for SLB errors. */
594 if (evt->error_type == MCE_ERROR_TYPE_SLB && !in_guest)
595 slb_dump_contents(local_paca->mce_faulty_slbs);
596#endif
597}
598EXPORT_SYMBOL_GPL(machine_check_print_event_info);
599
600/*
601 * This function is called in real mode. Strictly no printk's please.
602 *
603 * regs->nip and regs->msr contains srr0 and ssr1.
604 */
605DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early)
606{
607 long handled = 0;
608
609 hv_nmi_check_nonrecoverable(regs);
610
611 /*
612 * See if platform is capable of handling machine check.
613 */
614 if (ppc_md.machine_check_early)
615 handled = ppc_md.machine_check_early(regs);
616
617 return handled;
618}
619
620/* Possible meanings for HMER_DEBUG_TRIG bit being set on POWER9 */
621static enum {
622 DTRIG_UNKNOWN,
623 DTRIG_VECTOR_CI, /* need to emulate vector CI load instr */
624 DTRIG_SUSPEND_ESCAPE, /* need to escape from TM suspend mode */
625} hmer_debug_trig_function;
626
627static int init_debug_trig_function(void)
628{
629 int pvr;
630 struct device_node *cpun;
631 struct property *prop = NULL;
632 const char *str;
633
634 /* First look in the device tree */
635 preempt_disable();
636 cpun = of_get_cpu_node(smp_processor_id(), NULL);
637 if (cpun) {
638 of_property_for_each_string(cpun, "ibm,hmi-special-triggers",
639 prop, str) {
640 if (strcmp(str, "bit17-vector-ci-load") == 0)
641 hmer_debug_trig_function = DTRIG_VECTOR_CI;
642 else if (strcmp(str, "bit17-tm-suspend-escape") == 0)
643 hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE;
644 }
645 of_node_put(cpun);
646 }
647 preempt_enable();
648
649 /* If we found the property, don't look at PVR */
650 if (prop)
651 goto out;
652
653 pvr = mfspr(SPRN_PVR);
654 /* Check for POWER9 Nimbus (scale-out) */
655 if ((PVR_VER(pvr) == PVR_POWER9) && (pvr & 0xe000) == 0) {
656 /* DD2.2 and later */
657 if ((pvr & 0xfff) >= 0x202)
658 hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE;
659 /* DD2.0 and DD2.1 - used for vector CI load emulation */
660 else if ((pvr & 0xfff) >= 0x200)
661 hmer_debug_trig_function = DTRIG_VECTOR_CI;
662 }
663
664 out:
665 switch (hmer_debug_trig_function) {
666 case DTRIG_VECTOR_CI:
667 pr_debug("HMI debug trigger used for vector CI load\n");
668 break;
669 case DTRIG_SUSPEND_ESCAPE:
670 pr_debug("HMI debug trigger used for TM suspend escape\n");
671 break;
672 default:
673 break;
674 }
675 return 0;
676}
677__initcall(init_debug_trig_function);
678
679/*
680 * Handle HMIs that occur as a result of a debug trigger.
681 * Return values:
682 * -1 means this is not a HMI cause that we know about
683 * 0 means no further handling is required
684 * 1 means further handling is required
685 */
686long hmi_handle_debugtrig(struct pt_regs *regs)
687{
688 unsigned long hmer = mfspr(SPRN_HMER);
689 long ret = 0;
690
691 /* HMER_DEBUG_TRIG bit is used for various workarounds on P9 */
692 if (!((hmer & HMER_DEBUG_TRIG)
693 && hmer_debug_trig_function != DTRIG_UNKNOWN))
694 return -1;
695
696 hmer &= ~HMER_DEBUG_TRIG;
697 /* HMER is a write-AND register */
698 mtspr(SPRN_HMER, ~HMER_DEBUG_TRIG);
699
700 switch (hmer_debug_trig_function) {
701 case DTRIG_VECTOR_CI:
702 /*
703 * Now to avoid problems with soft-disable we
704 * only do the emulation if we are coming from
705 * host user space
706 */
707 if (regs && user_mode(regs))
708 ret = local_paca->hmi_p9_special_emu = 1;
709
710 break;
711
712 default:
713 break;
714 }
715
716 /*
717 * See if any other HMI causes remain to be handled
718 */
719 if (hmer & mfspr(SPRN_HMEER))
720 return -1;
721
722 return ret;
723}
724
725/*
726 * Return values:
727 */
728DEFINE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode)
729{
730 int ret;
731
732 local_paca->hmi_irqs++;
733
734 ret = hmi_handle_debugtrig(regs);
735 if (ret >= 0)
736 return ret;
737
738 wait_for_subcore_guest_exit();
739
740 if (ppc_md.hmi_exception_early)
741 ppc_md.hmi_exception_early(regs);
742
743 wait_for_tb_resync();
744
745 return 1;
746}
747
748void __init mce_init(void)
749{
750 struct mce_info *mce_info;
751 u64 limit;
752 int i;
753
754 limit = min(ppc64_bolted_size(), ppc64_rma_size);
755 for_each_possible_cpu(i) {
756 mce_info = memblock_alloc_try_nid(sizeof(*mce_info),
757 __alignof__(*mce_info),
758 MEMBLOCK_LOW_LIMIT,
759 limit, early_cpu_to_node(i));
760 if (!mce_info)
761 goto err;
762 paca_ptrs[i]->mce_info = mce_info;
763 }
764 return;
765err:
766 panic("Failed to allocate memory for MCE event data\n");
767}
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Machine check exception handling.
4 *
5 * Copyright 2013 IBM Corporation
6 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
7 */
8
9#undef DEBUG
10#define pr_fmt(fmt) "mce: " fmt
11
12#include <linux/hardirq.h>
13#include <linux/types.h>
14#include <linux/ptrace.h>
15#include <linux/percpu.h>
16#include <linux/export.h>
17#include <linux/irq_work.h>
18
19#include <asm/machdep.h>
20#include <asm/mce.h>
21#include <asm/nmi.h>
22
23static DEFINE_PER_CPU(int, mce_nest_count);
24static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event);
25
26/* Queue for delayed MCE events. */
27static DEFINE_PER_CPU(int, mce_queue_count);
28static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event_queue);
29
30/* Queue for delayed MCE UE events. */
31static DEFINE_PER_CPU(int, mce_ue_count);
32static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT],
33 mce_ue_event_queue);
34
35static void machine_check_process_queued_event(struct irq_work *work);
36static void machine_check_ue_irq_work(struct irq_work *work);
37static void machine_check_ue_event(struct machine_check_event *evt);
38static void machine_process_ue_event(struct work_struct *work);
39
40static struct irq_work mce_event_process_work = {
41 .func = machine_check_process_queued_event,
42};
43
44static struct irq_work mce_ue_event_irq_work = {
45 .func = machine_check_ue_irq_work,
46};
47
48DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
49
50static void mce_set_error_info(struct machine_check_event *mce,
51 struct mce_error_info *mce_err)
52{
53 mce->error_type = mce_err->error_type;
54 switch (mce_err->error_type) {
55 case MCE_ERROR_TYPE_UE:
56 mce->u.ue_error.ue_error_type = mce_err->u.ue_error_type;
57 break;
58 case MCE_ERROR_TYPE_SLB:
59 mce->u.slb_error.slb_error_type = mce_err->u.slb_error_type;
60 break;
61 case MCE_ERROR_TYPE_ERAT:
62 mce->u.erat_error.erat_error_type = mce_err->u.erat_error_type;
63 break;
64 case MCE_ERROR_TYPE_TLB:
65 mce->u.tlb_error.tlb_error_type = mce_err->u.tlb_error_type;
66 break;
67 case MCE_ERROR_TYPE_USER:
68 mce->u.user_error.user_error_type = mce_err->u.user_error_type;
69 break;
70 case MCE_ERROR_TYPE_RA:
71 mce->u.ra_error.ra_error_type = mce_err->u.ra_error_type;
72 break;
73 case MCE_ERROR_TYPE_LINK:
74 mce->u.link_error.link_error_type = mce_err->u.link_error_type;
75 break;
76 case MCE_ERROR_TYPE_UNKNOWN:
77 default:
78 break;
79 }
80}
81
82/*
83 * Decode and save high level MCE information into per cpu buffer which
84 * is an array of machine_check_event structure.
85 */
86void save_mce_event(struct pt_regs *regs, long handled,
87 struct mce_error_info *mce_err,
88 uint64_t nip, uint64_t addr, uint64_t phys_addr)
89{
90 int index = __this_cpu_inc_return(mce_nest_count) - 1;
91 struct machine_check_event *mce = this_cpu_ptr(&mce_event[index]);
92
93 /*
94 * Return if we don't have enough space to log mce event.
95 * mce_nest_count may go beyond MAX_MC_EVT but that's ok,
96 * the check below will stop buffer overrun.
97 */
98 if (index >= MAX_MC_EVT)
99 return;
100
101 /* Populate generic machine check info */
102 mce->version = MCE_V1;
103 mce->srr0 = nip;
104 mce->srr1 = regs->msr;
105 mce->gpr3 = regs->gpr[3];
106 mce->in_use = 1;
107 mce->cpu = get_paca()->paca_index;
108
109 /* Mark it recovered if we have handled it and MSR(RI=1). */
110 if (handled && (regs->msr & MSR_RI))
111 mce->disposition = MCE_DISPOSITION_RECOVERED;
112 else
113 mce->disposition = MCE_DISPOSITION_NOT_RECOVERED;
114
115 mce->initiator = mce_err->initiator;
116 mce->severity = mce_err->severity;
117 mce->sync_error = mce_err->sync_error;
118 mce->error_class = mce_err->error_class;
119
120 /*
121 * Populate the mce error_type and type-specific error_type.
122 */
123 mce_set_error_info(mce, mce_err);
124
125 if (!addr)
126 return;
127
128 if (mce->error_type == MCE_ERROR_TYPE_TLB) {
129 mce->u.tlb_error.effective_address_provided = true;
130 mce->u.tlb_error.effective_address = addr;
131 } else if (mce->error_type == MCE_ERROR_TYPE_SLB) {
132 mce->u.slb_error.effective_address_provided = true;
133 mce->u.slb_error.effective_address = addr;
134 } else if (mce->error_type == MCE_ERROR_TYPE_ERAT) {
135 mce->u.erat_error.effective_address_provided = true;
136 mce->u.erat_error.effective_address = addr;
137 } else if (mce->error_type == MCE_ERROR_TYPE_USER) {
138 mce->u.user_error.effective_address_provided = true;
139 mce->u.user_error.effective_address = addr;
140 } else if (mce->error_type == MCE_ERROR_TYPE_RA) {
141 mce->u.ra_error.effective_address_provided = true;
142 mce->u.ra_error.effective_address = addr;
143 } else if (mce->error_type == MCE_ERROR_TYPE_LINK) {
144 mce->u.link_error.effective_address_provided = true;
145 mce->u.link_error.effective_address = addr;
146 } else if (mce->error_type == MCE_ERROR_TYPE_UE) {
147 mce->u.ue_error.effective_address_provided = true;
148 mce->u.ue_error.effective_address = addr;
149 if (phys_addr != ULONG_MAX) {
150 mce->u.ue_error.physical_address_provided = true;
151 mce->u.ue_error.physical_address = phys_addr;
152 mce->u.ue_error.ignore_event = mce_err->ignore_event;
153 machine_check_ue_event(mce);
154 }
155 }
156 return;
157}
158
159/*
160 * get_mce_event:
161 * mce Pointer to machine_check_event structure to be filled.
162 * release Flag to indicate whether to free the event slot or not.
163 * 0 <= do not release the mce event. Caller will invoke
164 * release_mce_event() once event has been consumed.
165 * 1 <= release the slot.
166 *
167 * return 1 = success
168 * 0 = failure
169 *
170 * get_mce_event() will be called by platform specific machine check
171 * handle routine and in KVM.
172 * When we call get_mce_event(), we are still in interrupt context and
173 * preemption will not be scheduled until ret_from_expect() routine
174 * is called.
175 */
176int get_mce_event(struct machine_check_event *mce, bool release)
177{
178 int index = __this_cpu_read(mce_nest_count) - 1;
179 struct machine_check_event *mc_evt;
180 int ret = 0;
181
182 /* Sanity check */
183 if (index < 0)
184 return ret;
185
186 /* Check if we have MCE info to process. */
187 if (index < MAX_MC_EVT) {
188 mc_evt = this_cpu_ptr(&mce_event[index]);
189 /* Copy the event structure and release the original */
190 if (mce)
191 *mce = *mc_evt;
192 if (release)
193 mc_evt->in_use = 0;
194 ret = 1;
195 }
196 /* Decrement the count to free the slot. */
197 if (release)
198 __this_cpu_dec(mce_nest_count);
199
200 return ret;
201}
202
203void release_mce_event(void)
204{
205 get_mce_event(NULL, true);
206}
207
208static void machine_check_ue_irq_work(struct irq_work *work)
209{
210 schedule_work(&mce_ue_event_work);
211}
212
213/*
214 * Queue up the MCE event which then can be handled later.
215 */
216static void machine_check_ue_event(struct machine_check_event *evt)
217{
218 int index;
219
220 index = __this_cpu_inc_return(mce_ue_count) - 1;
221 /* If queue is full, just return for now. */
222 if (index >= MAX_MC_EVT) {
223 __this_cpu_dec(mce_ue_count);
224 return;
225 }
226 memcpy(this_cpu_ptr(&mce_ue_event_queue[index]), evt, sizeof(*evt));
227
228 /* Queue work to process this event later. */
229 irq_work_queue(&mce_ue_event_irq_work);
230}
231
232/*
233 * Queue up the MCE event which then can be handled later.
234 */
235void machine_check_queue_event(void)
236{
237 int index;
238 struct machine_check_event evt;
239
240 if (!get_mce_event(&evt, MCE_EVENT_RELEASE))
241 return;
242
243 index = __this_cpu_inc_return(mce_queue_count) - 1;
244 /* If queue is full, just return for now. */
245 if (index >= MAX_MC_EVT) {
246 __this_cpu_dec(mce_queue_count);
247 return;
248 }
249 memcpy(this_cpu_ptr(&mce_event_queue[index]), &evt, sizeof(evt));
250
251 /* Queue irq work to process this event later. */
252 irq_work_queue(&mce_event_process_work);
253}
254/*
255 * process pending MCE event from the mce event queue. This function will be
256 * called during syscall exit.
257 */
258static void machine_process_ue_event(struct work_struct *work)
259{
260 int index;
261 struct machine_check_event *evt;
262
263 while (__this_cpu_read(mce_ue_count) > 0) {
264 index = __this_cpu_read(mce_ue_count) - 1;
265 evt = this_cpu_ptr(&mce_ue_event_queue[index]);
266#ifdef CONFIG_MEMORY_FAILURE
267 /*
268 * This should probably queued elsewhere, but
269 * oh! well
270 *
271 * Don't report this machine check because the caller has a
272 * asked us to ignore the event, it has a fixup handler which
273 * will do the appropriate error handling and reporting.
274 */
275 if (evt->error_type == MCE_ERROR_TYPE_UE) {
276 if (evt->u.ue_error.ignore_event) {
277 __this_cpu_dec(mce_ue_count);
278 continue;
279 }
280
281 if (evt->u.ue_error.physical_address_provided) {
282 unsigned long pfn;
283
284 pfn = evt->u.ue_error.physical_address >>
285 PAGE_SHIFT;
286 memory_failure(pfn, 0);
287 } else
288 pr_warn("Failed to identify bad address from "
289 "where the uncorrectable error (UE) "
290 "was generated\n");
291 }
292#endif
293 __this_cpu_dec(mce_ue_count);
294 }
295}
296/*
297 * process pending MCE event from the mce event queue. This function will be
298 * called during syscall exit.
299 */
300static void machine_check_process_queued_event(struct irq_work *work)
301{
302 int index;
303 struct machine_check_event *evt;
304
305 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
306
307 /*
308 * For now just print it to console.
309 * TODO: log this error event to FSP or nvram.
310 */
311 while (__this_cpu_read(mce_queue_count) > 0) {
312 index = __this_cpu_read(mce_queue_count) - 1;
313 evt = this_cpu_ptr(&mce_event_queue[index]);
314
315 if (evt->error_type == MCE_ERROR_TYPE_UE &&
316 evt->u.ue_error.ignore_event) {
317 __this_cpu_dec(mce_queue_count);
318 continue;
319 }
320 machine_check_print_event_info(evt, false, false);
321 __this_cpu_dec(mce_queue_count);
322 }
323}
324
325void machine_check_print_event_info(struct machine_check_event *evt,
326 bool user_mode, bool in_guest)
327{
328 const char *level, *sevstr, *subtype, *err_type, *initiator;
329 uint64_t ea = 0, pa = 0;
330 int n = 0;
331 char dar_str[50];
332 char pa_str[50];
333 static const char *mc_ue_types[] = {
334 "Indeterminate",
335 "Instruction fetch",
336 "Page table walk ifetch",
337 "Load/Store",
338 "Page table walk Load/Store",
339 };
340 static const char *mc_slb_types[] = {
341 "Indeterminate",
342 "Parity",
343 "Multihit",
344 };
345 static const char *mc_erat_types[] = {
346 "Indeterminate",
347 "Parity",
348 "Multihit",
349 };
350 static const char *mc_tlb_types[] = {
351 "Indeterminate",
352 "Parity",
353 "Multihit",
354 };
355 static const char *mc_user_types[] = {
356 "Indeterminate",
357 "tlbie(l) invalid",
358 };
359 static const char *mc_ra_types[] = {
360 "Indeterminate",
361 "Instruction fetch (bad)",
362 "Instruction fetch (foreign)",
363 "Page table walk ifetch (bad)",
364 "Page table walk ifetch (foreign)",
365 "Load (bad)",
366 "Store (bad)",
367 "Page table walk Load/Store (bad)",
368 "Page table walk Load/Store (foreign)",
369 "Load/Store (foreign)",
370 };
371 static const char *mc_link_types[] = {
372 "Indeterminate",
373 "Instruction fetch (timeout)",
374 "Page table walk ifetch (timeout)",
375 "Load (timeout)",
376 "Store (timeout)",
377 "Page table walk Load/Store (timeout)",
378 };
379 static const char *mc_error_class[] = {
380 "Unknown",
381 "Hardware error",
382 "Probable Hardware error (some chance of software cause)",
383 "Software error",
384 "Probable Software error (some chance of hardware cause)",
385 };
386
387 /* Print things out */
388 if (evt->version != MCE_V1) {
389 pr_err("Machine Check Exception, Unknown event version %d !\n",
390 evt->version);
391 return;
392 }
393 switch (evt->severity) {
394 case MCE_SEV_NO_ERROR:
395 level = KERN_INFO;
396 sevstr = "Harmless";
397 break;
398 case MCE_SEV_WARNING:
399 level = KERN_WARNING;
400 sevstr = "Warning";
401 break;
402 case MCE_SEV_SEVERE:
403 level = KERN_ERR;
404 sevstr = "Severe";
405 break;
406 case MCE_SEV_FATAL:
407 default:
408 level = KERN_ERR;
409 sevstr = "Fatal";
410 break;
411 }
412
413 switch(evt->initiator) {
414 case MCE_INITIATOR_CPU:
415 initiator = "CPU";
416 break;
417 case MCE_INITIATOR_PCI:
418 initiator = "PCI";
419 break;
420 case MCE_INITIATOR_ISA:
421 initiator = "ISA";
422 break;
423 case MCE_INITIATOR_MEMORY:
424 initiator = "Memory";
425 break;
426 case MCE_INITIATOR_POWERMGM:
427 initiator = "Power Management";
428 break;
429 case MCE_INITIATOR_UNKNOWN:
430 default:
431 initiator = "Unknown";
432 break;
433 }
434
435 switch (evt->error_type) {
436 case MCE_ERROR_TYPE_UE:
437 err_type = "UE";
438 subtype = evt->u.ue_error.ue_error_type <
439 ARRAY_SIZE(mc_ue_types) ?
440 mc_ue_types[evt->u.ue_error.ue_error_type]
441 : "Unknown";
442 if (evt->u.ue_error.effective_address_provided)
443 ea = evt->u.ue_error.effective_address;
444 if (evt->u.ue_error.physical_address_provided)
445 pa = evt->u.ue_error.physical_address;
446 break;
447 case MCE_ERROR_TYPE_SLB:
448 err_type = "SLB";
449 subtype = evt->u.slb_error.slb_error_type <
450 ARRAY_SIZE(mc_slb_types) ?
451 mc_slb_types[evt->u.slb_error.slb_error_type]
452 : "Unknown";
453 if (evt->u.slb_error.effective_address_provided)
454 ea = evt->u.slb_error.effective_address;
455 break;
456 case MCE_ERROR_TYPE_ERAT:
457 err_type = "ERAT";
458 subtype = evt->u.erat_error.erat_error_type <
459 ARRAY_SIZE(mc_erat_types) ?
460 mc_erat_types[evt->u.erat_error.erat_error_type]
461 : "Unknown";
462 if (evt->u.erat_error.effective_address_provided)
463 ea = evt->u.erat_error.effective_address;
464 break;
465 case MCE_ERROR_TYPE_TLB:
466 err_type = "TLB";
467 subtype = evt->u.tlb_error.tlb_error_type <
468 ARRAY_SIZE(mc_tlb_types) ?
469 mc_tlb_types[evt->u.tlb_error.tlb_error_type]
470 : "Unknown";
471 if (evt->u.tlb_error.effective_address_provided)
472 ea = evt->u.tlb_error.effective_address;
473 break;
474 case MCE_ERROR_TYPE_USER:
475 err_type = "User";
476 subtype = evt->u.user_error.user_error_type <
477 ARRAY_SIZE(mc_user_types) ?
478 mc_user_types[evt->u.user_error.user_error_type]
479 : "Unknown";
480 if (evt->u.user_error.effective_address_provided)
481 ea = evt->u.user_error.effective_address;
482 break;
483 case MCE_ERROR_TYPE_RA:
484 err_type = "Real address";
485 subtype = evt->u.ra_error.ra_error_type <
486 ARRAY_SIZE(mc_ra_types) ?
487 mc_ra_types[evt->u.ra_error.ra_error_type]
488 : "Unknown";
489 if (evt->u.ra_error.effective_address_provided)
490 ea = evt->u.ra_error.effective_address;
491 break;
492 case MCE_ERROR_TYPE_LINK:
493 err_type = "Link";
494 subtype = evt->u.link_error.link_error_type <
495 ARRAY_SIZE(mc_link_types) ?
496 mc_link_types[evt->u.link_error.link_error_type]
497 : "Unknown";
498 if (evt->u.link_error.effective_address_provided)
499 ea = evt->u.link_error.effective_address;
500 break;
501 case MCE_ERROR_TYPE_DCACHE:
502 err_type = "D-Cache";
503 subtype = "Unknown";
504 break;
505 case MCE_ERROR_TYPE_ICACHE:
506 err_type = "I-Cache";
507 subtype = "Unknown";
508 break;
509 default:
510 case MCE_ERROR_TYPE_UNKNOWN:
511 err_type = "Unknown";
512 subtype = "";
513 break;
514 }
515
516 dar_str[0] = pa_str[0] = '\0';
517 if (ea && evt->srr0 != ea) {
518 /* Load/Store address */
519 n = sprintf(dar_str, "DAR: %016llx ", ea);
520 if (pa)
521 sprintf(dar_str + n, "paddr: %016llx ", pa);
522 } else if (pa) {
523 sprintf(pa_str, " paddr: %016llx", pa);
524 }
525
526 printk("%sMCE: CPU%d: machine check (%s) %s %s %s %s[%s]\n",
527 level, evt->cpu, sevstr, in_guest ? "Guest" : "Host",
528 err_type, subtype, dar_str,
529 evt->disposition == MCE_DISPOSITION_RECOVERED ?
530 "Recovered" : "Not recovered");
531
532 if (in_guest || user_mode) {
533 printk("%sMCE: CPU%d: PID: %d Comm: %s %sNIP: [%016llx]%s\n",
534 level, evt->cpu, current->pid, current->comm,
535 in_guest ? "Guest " : "", evt->srr0, pa_str);
536 } else {
537 printk("%sMCE: CPU%d: NIP: [%016llx] %pS%s\n",
538 level, evt->cpu, evt->srr0, (void *)evt->srr0, pa_str);
539 }
540
541 printk("%sMCE: CPU%d: Initiator %s\n", level, evt->cpu, initiator);
542
543 subtype = evt->error_class < ARRAY_SIZE(mc_error_class) ?
544 mc_error_class[evt->error_class] : "Unknown";
545 printk("%sMCE: CPU%d: %s\n", level, evt->cpu, subtype);
546
547#ifdef CONFIG_PPC_BOOK3S_64
548 /* Display faulty slb contents for SLB errors. */
549 if (evt->error_type == MCE_ERROR_TYPE_SLB)
550 slb_dump_contents(local_paca->mce_faulty_slbs);
551#endif
552}
553EXPORT_SYMBOL_GPL(machine_check_print_event_info);
554
555/*
556 * This function is called in real mode. Strictly no printk's please.
557 *
558 * regs->nip and regs->msr contains srr0 and ssr1.
559 */
560long machine_check_early(struct pt_regs *regs)
561{
562 long handled = 0;
563
564 hv_nmi_check_nonrecoverable(regs);
565
566 /*
567 * See if platform is capable of handling machine check.
568 */
569 if (ppc_md.machine_check_early)
570 handled = ppc_md.machine_check_early(regs);
571 return handled;
572}
573
574/* Possible meanings for HMER_DEBUG_TRIG bit being set on POWER9 */
575static enum {
576 DTRIG_UNKNOWN,
577 DTRIG_VECTOR_CI, /* need to emulate vector CI load instr */
578 DTRIG_SUSPEND_ESCAPE, /* need to escape from TM suspend mode */
579} hmer_debug_trig_function;
580
581static int init_debug_trig_function(void)
582{
583 int pvr;
584 struct device_node *cpun;
585 struct property *prop = NULL;
586 const char *str;
587
588 /* First look in the device tree */
589 preempt_disable();
590 cpun = of_get_cpu_node(smp_processor_id(), NULL);
591 if (cpun) {
592 of_property_for_each_string(cpun, "ibm,hmi-special-triggers",
593 prop, str) {
594 if (strcmp(str, "bit17-vector-ci-load") == 0)
595 hmer_debug_trig_function = DTRIG_VECTOR_CI;
596 else if (strcmp(str, "bit17-tm-suspend-escape") == 0)
597 hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE;
598 }
599 of_node_put(cpun);
600 }
601 preempt_enable();
602
603 /* If we found the property, don't look at PVR */
604 if (prop)
605 goto out;
606
607 pvr = mfspr(SPRN_PVR);
608 /* Check for POWER9 Nimbus (scale-out) */
609 if ((PVR_VER(pvr) == PVR_POWER9) && (pvr & 0xe000) == 0) {
610 /* DD2.2 and later */
611 if ((pvr & 0xfff) >= 0x202)
612 hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE;
613 /* DD2.0 and DD2.1 - used for vector CI load emulation */
614 else if ((pvr & 0xfff) >= 0x200)
615 hmer_debug_trig_function = DTRIG_VECTOR_CI;
616 }
617
618 out:
619 switch (hmer_debug_trig_function) {
620 case DTRIG_VECTOR_CI:
621 pr_debug("HMI debug trigger used for vector CI load\n");
622 break;
623 case DTRIG_SUSPEND_ESCAPE:
624 pr_debug("HMI debug trigger used for TM suspend escape\n");
625 break;
626 default:
627 break;
628 }
629 return 0;
630}
631__initcall(init_debug_trig_function);
632
633/*
634 * Handle HMIs that occur as a result of a debug trigger.
635 * Return values:
636 * -1 means this is not a HMI cause that we know about
637 * 0 means no further handling is required
638 * 1 means further handling is required
639 */
640long hmi_handle_debugtrig(struct pt_regs *regs)
641{
642 unsigned long hmer = mfspr(SPRN_HMER);
643 long ret = 0;
644
645 /* HMER_DEBUG_TRIG bit is used for various workarounds on P9 */
646 if (!((hmer & HMER_DEBUG_TRIG)
647 && hmer_debug_trig_function != DTRIG_UNKNOWN))
648 return -1;
649
650 hmer &= ~HMER_DEBUG_TRIG;
651 /* HMER is a write-AND register */
652 mtspr(SPRN_HMER, ~HMER_DEBUG_TRIG);
653
654 switch (hmer_debug_trig_function) {
655 case DTRIG_VECTOR_CI:
656 /*
657 * Now to avoid problems with soft-disable we
658 * only do the emulation if we are coming from
659 * host user space
660 */
661 if (regs && user_mode(regs))
662 ret = local_paca->hmi_p9_special_emu = 1;
663
664 break;
665
666 default:
667 break;
668 }
669
670 /*
671 * See if any other HMI causes remain to be handled
672 */
673 if (hmer & mfspr(SPRN_HMEER))
674 return -1;
675
676 return ret;
677}
678
679/*
680 * Return values:
681 */
682long hmi_exception_realmode(struct pt_regs *regs)
683{
684 int ret;
685
686 __this_cpu_inc(irq_stat.hmi_exceptions);
687
688 ret = hmi_handle_debugtrig(regs);
689 if (ret >= 0)
690 return ret;
691
692 wait_for_subcore_guest_exit();
693
694 if (ppc_md.hmi_exception_early)
695 ppc_md.hmi_exception_early(regs);
696
697 wait_for_tb_resync();
698
699 return 1;
700}