1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *  Copyright (C) 1995-1996  Gary Thomas (gdt@linuxppc.org)
   4 *  Copyright 2007-2010 Freescale Semiconductor, Inc.
   5 *
   6 *  Modified by Cort Dougan (cort@cs.nmt.edu)
   7 *  and Paul Mackerras (paulus@samba.org)
   8 */
   9
  10/*
  11 * This file handles the architecture-dependent parts of hardware exceptions
  12 */
  13
  14#include <linux/errno.h>
  15#include <linux/sched.h>
  16#include <linux/sched/debug.h>
  17#include <linux/kernel.h>
  18#include <linux/mm.h>
  19#include <linux/pkeys.h>
  20#include <linux/stddef.h>
  21#include <linux/unistd.h>
  22#include <linux/ptrace.h>
  23#include <linux/user.h>
  24#include <linux/interrupt.h>
  25#include <linux/init.h>
  26#include <linux/extable.h>
  27#include <linux/module.h>	/* print_modules */
  28#include <linux/prctl.h>
  29#include <linux/delay.h>
  30#include <linux/kprobes.h>
  31#include <linux/kexec.h>
  32#include <linux/backlight.h>
  33#include <linux/bug.h>
  34#include <linux/kdebug.h>
  35#include <linux/ratelimit.h>
  36#include <linux/context_tracking.h>
  37#include <linux/smp.h>
  38#include <linux/console.h>
  39#include <linux/kmsg_dump.h>
  40#include <linux/debugfs.h>
  41
  42#include <asm/emulated_ops.h>
  43#include <linux/uaccess.h>
 
  44#include <asm/interrupt.h>
  45#include <asm/io.h>
  46#include <asm/machdep.h>
  47#include <asm/rtas.h>
  48#include <asm/pmc.h>
  49#include <asm/reg.h>
  50#ifdef CONFIG_PMAC_BACKLIGHT
  51#include <asm/backlight.h>
  52#endif
  53#ifdef CONFIG_PPC64
  54#include <asm/firmware.h>
  55#include <asm/processor.h>
  56#endif
  57#include <asm/kexec.h>
  58#include <asm/ppc-opcode.h>
  59#include <asm/rio.h>
  60#include <asm/fadump.h>
  61#include <asm/switch_to.h>
  62#include <asm/tm.h>
  63#include <asm/debug.h>
  64#include <asm/asm-prototypes.h>
  65#include <asm/hmi.h>
  66#include <sysdev/fsl_pci.h>
  67#include <asm/kprobes.h>
  68#include <asm/stacktrace.h>
  69#include <asm/nmi.h>
  70#include <asm/disassemble.h>
  71#include <asm/udbg.h>
  72
  73#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE)
  74int (*__debugger)(struct pt_regs *regs) __read_mostly;
  75int (*__debugger_ipi)(struct pt_regs *regs) __read_mostly;
  76int (*__debugger_bpt)(struct pt_regs *regs) __read_mostly;
  77int (*__debugger_sstep)(struct pt_regs *regs) __read_mostly;
  78int (*__debugger_iabr_match)(struct pt_regs *regs) __read_mostly;
  79int (*__debugger_break_match)(struct pt_regs *regs) __read_mostly;
  80int (*__debugger_fault_handler)(struct pt_regs *regs) __read_mostly;
  81
  82EXPORT_SYMBOL(__debugger);
  83EXPORT_SYMBOL(__debugger_ipi);
  84EXPORT_SYMBOL(__debugger_bpt);
  85EXPORT_SYMBOL(__debugger_sstep);
  86EXPORT_SYMBOL(__debugger_iabr_match);
  87EXPORT_SYMBOL(__debugger_break_match);
  88EXPORT_SYMBOL(__debugger_fault_handler);
  89#endif
  90
  91/* Transactional Memory trap debug */
  92#ifdef TM_DEBUG_SW
  93#define TM_DEBUG(x...) printk(KERN_INFO x)
  94#else
  95#define TM_DEBUG(x...) do { } while(0)
  96#endif
  97
  98static const char *signame(int signr)
  99{
 100	switch (signr) {
 101	case SIGBUS:	return "bus error";
 102	case SIGFPE:	return "floating point exception";
 103	case SIGILL:	return "illegal instruction";
 104	case SIGSEGV:	return "segfault";
 105	case SIGTRAP:	return "unhandled trap";
 106	}
 107
 108	return "unknown signal";
 109}
 110
 111/*
 112 * Trap & Exception support
 113 */
 114
 115#ifdef CONFIG_PMAC_BACKLIGHT
 116static void pmac_backlight_unblank(void)
 117{
 118	mutex_lock(&pmac_backlight_mutex);
 119	if (pmac_backlight) {
 120		struct backlight_properties *props;
 121
 122		props = &pmac_backlight->props;
 123		props->brightness = props->max_brightness;
 124		props->power = FB_BLANK_UNBLANK;
 125		backlight_update_status(pmac_backlight);
 126	}
 127	mutex_unlock(&pmac_backlight_mutex);
 128}
 129#else
 130static inline void pmac_backlight_unblank(void) { }
 131#endif
 132
 133/*
 134 * If oops/die is expected to crash the machine, return true here.
 135 *
 136 * This should not be expected to be 100% accurate, there may be
 137 * notifiers registered or other unexpected conditions that may bring
 138 * down the kernel. Or if the current process in the kernel is holding
 139 * locks or has other critical state, the kernel may become effectively
 140 * unusable anyway.
 141 */
 142bool die_will_crash(void)
 143{
 144	if (should_fadump_crash())
 145		return true;
 146	if (kexec_should_crash(current))
 147		return true;
 148	if (in_interrupt() || panic_on_oops ||
 149			!current->pid || is_global_init(current))
 150		return true;
 151
 152	return false;
 153}
 154
 155static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;
 156static int die_owner = -1;
 157static unsigned int die_nest_count;
 158static int die_counter;
 159
 160extern void panic_flush_kmsg_start(void)
 161{
 162	/*
 163	 * These are mostly taken from kernel/panic.c, but tries to do
 164	 * relatively minimal work. Don't use delay functions (TB may
 165	 * be broken), don't crash dump (need to set a firmware log),
 166	 * don't run notifiers. We do want to get some information to
 167	 * Linux console.
 168	 */
 169	console_verbose();
 170	bust_spinlocks(1);
 171}
 172
 173extern void panic_flush_kmsg_end(void)
 174{
 
 175	kmsg_dump(KMSG_DUMP_PANIC);
 176	bust_spinlocks(0);
 177	debug_locks_off();
 178	console_flush_on_panic(CONSOLE_FLUSH_PENDING);
 179}
 180
 181static unsigned long oops_begin(struct pt_regs *regs)
 182{
 183	int cpu;
 184	unsigned long flags;
 185
 186	oops_enter();
 187
 188	/* racy, but better than risking deadlock. */
 189	raw_local_irq_save(flags);
 190	cpu = smp_processor_id();
 191	if (!arch_spin_trylock(&die_lock)) {
 192		if (cpu == die_owner)
 193			/* nested oops. should stop eventually */;
 194		else
 195			arch_spin_lock(&die_lock);
 196	}
 197	die_nest_count++;
 198	die_owner = cpu;
 199	console_verbose();
 200	bust_spinlocks(1);
 201	if (machine_is(powermac))
 202		pmac_backlight_unblank();
 203	return flags;
 204}
 205NOKPROBE_SYMBOL(oops_begin);
 206
 207static void oops_end(unsigned long flags, struct pt_regs *regs,
 208			       int signr)
 209{
 210	bust_spinlocks(0);
 211	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 212	die_nest_count--;
 213	oops_exit();
 214	printk("\n");
 215	if (!die_nest_count) {
 216		/* Nest count reaches zero, release the lock. */
 217		die_owner = -1;
 218		arch_spin_unlock(&die_lock);
 219	}
 220	raw_local_irq_restore(flags);
 221
 222	/*
 223	 * system_reset_excption handles debugger, crash dump, panic, for 0x100
 224	 */
 225	if (TRAP(regs) == INTERRUPT_SYSTEM_RESET)
 226		return;
 227
 228	crash_fadump(regs, "die oops");
 229
 230	if (kexec_should_crash(current))
 231		crash_kexec(regs);
 232
 233	if (!signr)
 234		return;
 235
 236	/*
 237	 * While our oops output is serialised by a spinlock, output
 238	 * from panic() called below can race and corrupt it. If we
 239	 * know we are going to panic, delay for 1 second so we have a
 240	 * chance to get clean backtraces from all CPUs that are oopsing.
 241	 */
 242	if (in_interrupt() || panic_on_oops || !current->pid ||
 243	    is_global_init(current)) {
 244		mdelay(MSEC_PER_SEC);
 245	}
 246
 247	if (panic_on_oops)
 248		panic("Fatal exception");
 249	make_task_dead(signr);
 250}
 251NOKPROBE_SYMBOL(oops_end);
 252
 253static char *get_mmu_str(void)
 254{
 255	if (early_radix_enabled())
 256		return " MMU=Radix";
 257	if (early_mmu_has_feature(MMU_FTR_HPTE_TABLE))
 258		return " MMU=Hash";
 259	return "";
 260}
 261
 262static int __die(const char *str, struct pt_regs *regs, long err)
 263{
 264	printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter);
 265
 266	printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n",
 267	       IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE",
 268	       PAGE_SIZE / 1024, get_mmu_str(),
 269	       IS_ENABLED(CONFIG_PREEMPT) ? " PREEMPT" : "",
 270	       IS_ENABLED(CONFIG_SMP) ? " SMP" : "",
 271	       IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "",
 272	       debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "",
 273	       IS_ENABLED(CONFIG_NUMA) ? " NUMA" : "",
 274	       ppc_md.name ? ppc_md.name : "");
 275
 276	if (notify_die(DIE_OOPS, str, regs, err, 255, SIGSEGV) == NOTIFY_STOP)
 277		return 1;
 278
 279	print_modules();
 280	show_regs(regs);
 281
 282	return 0;
 283}
 284NOKPROBE_SYMBOL(__die);
 285
 286void die(const char *str, struct pt_regs *regs, long err)
 287{
 288	unsigned long flags;
 289
 290	/*
 291	 * system_reset_excption handles debugger, crash dump, panic, for 0x100
 292	 */
 293	if (TRAP(regs) != INTERRUPT_SYSTEM_RESET) {
 294		if (debugger(regs))
 295			return;
 296	}
 297
 298	flags = oops_begin(regs);
 299	if (__die(str, regs, err))
 300		err = 0;
 301	oops_end(flags, regs, err);
 302}
 303NOKPROBE_SYMBOL(die);
 304
 305void user_single_step_report(struct pt_regs *regs)
 306{
 307	force_sig_fault(SIGTRAP, TRAP_TRACE, (void __user *)regs->nip);
 308}
 309
 310static void show_signal_msg(int signr, struct pt_regs *regs, int code,
 311			    unsigned long addr)
 312{
 313	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 314				      DEFAULT_RATELIMIT_BURST);
 315
 316	if (!show_unhandled_signals)
 317		return;
 318
 319	if (!unhandled_signal(current, signr))
 320		return;
 321
 322	if (!__ratelimit(&rs))
 323		return;
 324
 325	pr_info("%s[%d]: %s (%d) at %lx nip %lx lr %lx code %x",
 326		current->comm, current->pid, signame(signr), signr,
 327		addr, regs->nip, regs->link, code);
 328
 329	print_vma_addr(KERN_CONT " in ", regs->nip);
 330
 331	pr_cont("\n");
 332
 333	show_user_instructions(regs);
 334}
 335
 336static bool exception_common(int signr, struct pt_regs *regs, int code,
 337			      unsigned long addr)
 338{
 339	if (!user_mode(regs)) {
 340		die("Exception in kernel mode", regs, signr);
 341		return false;
 342	}
 343
 344	/*
 345	 * Must not enable interrupts even for user-mode exception, because
 346	 * this can be called from machine check, which may be a NMI or IRQ
 347	 * which don't like interrupts being enabled. Could check for
 348	 * in_hardirq || in_nmi perhaps, but there doesn't seem to be a good
 349	 * reason why _exception() should enable irqs for an exception handler,
 350	 * the handlers themselves do that directly.
 351	 */
 352
 353	show_signal_msg(signr, regs, code, addr);
 354
 355	current->thread.trap_nr = code;
 356
 357	return true;
 358}
 359
 360void _exception_pkey(struct pt_regs *regs, unsigned long addr, int key)
 361{
 362	if (!exception_common(SIGSEGV, regs, SEGV_PKUERR, addr))
 363		return;
 364
 365	force_sig_pkuerr((void __user *) addr, key);
 366}
 367
 368void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr)
 369{
 370	if (!exception_common(signr, regs, code, addr))
 371		return;
 372
 373	force_sig_fault(signr, code, (void __user *)addr);
 374}
 375
 376/*
 377 * The interrupt architecture has a quirk in that the HV interrupts excluding
 378 * the NMIs (0x100 and 0x200) do not clear MSR[RI] at entry. The first thing
 379 * that an interrupt handler must do is save off a GPR into a scratch register,
 380 * and all interrupts on POWERNV (HV=1) use the HSPRG1 register as scratch.
 381 * Therefore an NMI can clobber an HV interrupt's live HSPRG1 without noticing
 382 * that it is non-reentrant, which leads to random data corruption.
 383 *
 384 * The solution is for NMI interrupts in HV mode to check if they originated
 385 * from these critical HV interrupt regions. If so, then mark them not
 386 * recoverable.
 387 *
 388 * An alternative would be for HV NMIs to use SPRG for scratch to avoid the
 389 * HSPRG1 clobber, however this would cause guest SPRG to be clobbered. Linux
 390 * guests should always have MSR[RI]=0 when its scratch SPRG is in use, so
 391 * that would work. However any other guest OS that may have the SPRG live
 392 * and MSR[RI]=1 could encounter silent corruption.
 393 *
 394 * Builds that do not support KVM could take this second option to increase
 395 * the recoverability of NMIs.
 396 */
 397noinstr void hv_nmi_check_nonrecoverable(struct pt_regs *regs)
 398{
 399#ifdef CONFIG_PPC_POWERNV
 400	unsigned long kbase = (unsigned long)_stext;
 401	unsigned long nip = regs->nip;
 402
 403	if (!(regs->msr & MSR_RI))
 404		return;
 405	if (!(regs->msr & MSR_HV))
 406		return;
 407	if (regs->msr & MSR_PR)
 408		return;
 409
 410	/*
 411	 * Now test if the interrupt has hit a range that may be using
 412	 * HSPRG1 without having RI=0 (i.e., an HSRR interrupt). The
 413	 * problem ranges all run un-relocated. Test real and virt modes
 414	 * at the same time by dropping the high bit of the nip (virt mode
 415	 * entry points still have the +0x4000 offset).
 416	 */
 417	nip &= ~0xc000000000000000ULL;
 418	if ((nip >= 0x500 && nip < 0x600) || (nip >= 0x4500 && nip < 0x4600))
 419		goto nonrecoverable;
 420	if ((nip >= 0x980 && nip < 0xa00) || (nip >= 0x4980 && nip < 0x4a00))
 421		goto nonrecoverable;
 422	if ((nip >= 0xe00 && nip < 0xec0) || (nip >= 0x4e00 && nip < 0x4ec0))
 423		goto nonrecoverable;
 424	if ((nip >= 0xf80 && nip < 0xfa0) || (nip >= 0x4f80 && nip < 0x4fa0))
 425		goto nonrecoverable;
 426
 427	/* Trampoline code runs un-relocated so subtract kbase. */
 428	if (nip >= (unsigned long)(start_real_trampolines - kbase) &&
 429			nip < (unsigned long)(end_real_trampolines - kbase))
 430		goto nonrecoverable;
 431	if (nip >= (unsigned long)(start_virt_trampolines - kbase) &&
 432			nip < (unsigned long)(end_virt_trampolines - kbase))
 433		goto nonrecoverable;
 434	return;
 435
 436nonrecoverable:
 437	regs->msr &= ~MSR_RI;
 438	local_paca->hsrr_valid = 0;
 439	local_paca->srr_valid = 0;
 440#endif
 441}
 442DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception)
 443{
 444	unsigned long hsrr0, hsrr1;
 445	bool saved_hsrrs = false;
 446
 447	/*
 448	 * System reset can interrupt code where HSRRs are live and MSR[RI]=1.
 449	 * The system reset interrupt itself may clobber HSRRs (e.g., to call
 450	 * OPAL), so save them here and restore them before returning.
 451	 *
 452	 * Machine checks don't need to save HSRRs, as the real mode handler
 453	 * is careful to avoid them, and the regular handler is not delivered
 454	 * as an NMI.
 455	 */
 456	if (cpu_has_feature(CPU_FTR_HVMODE)) {
 457		hsrr0 = mfspr(SPRN_HSRR0);
 458		hsrr1 = mfspr(SPRN_HSRR1);
 459		saved_hsrrs = true;
 460	}
 461
 462	hv_nmi_check_nonrecoverable(regs);
 463
 464	__this_cpu_inc(irq_stat.sreset_irqs);
 465
 466	/* See if any machine dependent calls */
 467	if (ppc_md.system_reset_exception) {
 468		if (ppc_md.system_reset_exception(regs))
 469			goto out;
 470	}
 471
 472	if (debugger(regs))
 473		goto out;
 474
 475	kmsg_dump(KMSG_DUMP_OOPS);
 476	/*
 477	 * A system reset is a request to dump, so we always send
 478	 * it through the crashdump code (if fadump or kdump are
 479	 * registered).
 480	 */
 481	crash_fadump(regs, "System Reset");
 482
 483	crash_kexec(regs);
 484
 485	/*
 486	 * We aren't the primary crash CPU. We need to send it
 487	 * to a holding pattern to avoid it ending up in the panic
 488	 * code.
 489	 */
 490	crash_kexec_secondary(regs);
 491
 492	/*
 493	 * No debugger or crash dump registered, print logs then
 494	 * panic.
 495	 */
 496	die("System Reset", regs, SIGABRT);
 497
 498	mdelay(2*MSEC_PER_SEC); /* Wait a little while for others to print */
 499	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 500	nmi_panic(regs, "System Reset");
 501
 502out:
 503#ifdef CONFIG_PPC_BOOK3S_64
 504	BUG_ON(get_paca()->in_nmi == 0);
 505	if (get_paca()->in_nmi > 1)
 506		die("Unrecoverable nested System Reset", regs, SIGABRT);
 507#endif
 508	/* Must die if the interrupt is not recoverable */
 509	if (regs_is_unrecoverable(regs)) {
 510		/* For the reason explained in die_mce, nmi_exit before die */
 511		nmi_exit();
 512		die("Unrecoverable System Reset", regs, SIGABRT);
 513	}
 514
 515	if (saved_hsrrs) {
 516		mtspr(SPRN_HSRR0, hsrr0);
 517		mtspr(SPRN_HSRR1, hsrr1);
 518	}
 519
 520	/* What should we do here? We could issue a shutdown or hard reset. */
 521
 522	return 0;
 523}
 524
 525/*
 526 * I/O accesses can cause machine checks on powermacs.
 527 * Check if the NIP corresponds to the address of a sync
 528 * instruction for which there is an entry in the exception
 529 * table.
 530 *  -- paulus.
 531 */
 532static inline int check_io_access(struct pt_regs *regs)
 533{
 534#ifdef CONFIG_PPC32
 535	unsigned long msr = regs->msr;
 536	const struct exception_table_entry *entry;
 537	unsigned int *nip = (unsigned int *)regs->nip;
 538
 539	if (((msr & 0xffff0000) == 0 || (msr & (0x80000 | 0x40000)))
 540	    && (entry = search_exception_tables(regs->nip)) != NULL) {
 541		/*
 542		 * Check that it's a sync instruction, or somewhere
 543		 * in the twi; isync; nop sequence that inb/inw/inl uses.
 544		 * As the address is in the exception table
 545		 * we should be able to read the instr there.
 546		 * For the debug message, we look at the preceding
 547		 * load or store.
 548		 */
 549		if (*nip == PPC_RAW_NOP())
 550			nip -= 2;
 551		else if (*nip == PPC_RAW_ISYNC())
 552			--nip;
 553		if (*nip == PPC_RAW_SYNC() || get_op(*nip) == OP_TRAP) {
 554			unsigned int rb;
 555
 556			--nip;
 557			rb = (*nip >> 11) & 0x1f;
 558			printk(KERN_DEBUG "%s bad port %lx at %p\n",
 559			       (*nip & 0x100)? "OUT to": "IN from",
 560			       regs->gpr[rb] - _IO_BASE, nip);
 561			regs_set_recoverable(regs);
 562			regs_set_return_ip(regs, extable_fixup(entry));
 563			return 1;
 564		}
 565	}
 566#endif /* CONFIG_PPC32 */
 567	return 0;
 568}
 569
 570#ifdef CONFIG_PPC_ADV_DEBUG_REGS
 571/* On 4xx, the reason for the machine check or program exception
 572   is in the ESR. */
 573#define get_reason(regs)	((regs)->esr)
 574#define REASON_FP		ESR_FP
 575#define REASON_ILLEGAL		(ESR_PIL | ESR_PUO)
 576#define REASON_PRIVILEGED	ESR_PPR
 577#define REASON_TRAP		ESR_PTR
 578#define REASON_PREFIXED		0
 579#define REASON_BOUNDARY		0
 580
 581/* single-step stuff */
 582#define single_stepping(regs)	(current->thread.debug.dbcr0 & DBCR0_IC)
 583#define clear_single_step(regs)	(current->thread.debug.dbcr0 &= ~DBCR0_IC)
 584#define clear_br_trace(regs)	do {} while(0)
 585#else
 586/* On non-4xx, the reason for the machine check or program
 587   exception is in the MSR. */
 588#define get_reason(regs)	((regs)->msr)
 589#define REASON_TM		SRR1_PROGTM
 590#define REASON_FP		SRR1_PROGFPE
 591#define REASON_ILLEGAL		SRR1_PROGILL
 592#define REASON_PRIVILEGED	SRR1_PROGPRIV
 593#define REASON_TRAP		SRR1_PROGTRAP
 594#define REASON_PREFIXED		SRR1_PREFIXED
 595#define REASON_BOUNDARY		SRR1_BOUNDARY
 596
 597#define single_stepping(regs)	((regs)->msr & MSR_SE)
 598#define clear_single_step(regs)	(regs_set_return_msr((regs), (regs)->msr & ~MSR_SE))
 599#define clear_br_trace(regs)	(regs_set_return_msr((regs), (regs)->msr & ~MSR_BE))
 600#endif
 601
 602#define inst_length(reason)	(((reason) & REASON_PREFIXED) ? 8 : 4)
 603
 604#if defined(CONFIG_PPC_E500)
 605int machine_check_e500mc(struct pt_regs *regs)
 606{
 607	unsigned long mcsr = mfspr(SPRN_MCSR);
 608	unsigned long pvr = mfspr(SPRN_PVR);
 609	unsigned long reason = mcsr;
 610	int recoverable = 1;
 611
 612	if (reason & MCSR_LD) {
 613		recoverable = fsl_rio_mcheck_exception(regs);
 614		if (recoverable == 1)
 615			goto silent_out;
 616	}
 617
 618	printk("Machine check in kernel mode.\n");
 619	printk("Caused by (from MCSR=%lx): ", reason);
 620
 621	if (reason & MCSR_MCP)
 622		pr_cont("Machine Check Signal\n");
 623
 624	if (reason & MCSR_ICPERR) {
 625		pr_cont("Instruction Cache Parity Error\n");
 626
 627		/*
 628		 * This is recoverable by invalidating the i-cache.
 629		 */
 630		mtspr(SPRN_L1CSR1, mfspr(SPRN_L1CSR1) | L1CSR1_ICFI);
 631		while (mfspr(SPRN_L1CSR1) & L1CSR1_ICFI)
 632			;
 633
 634		/*
 635		 * This will generally be accompanied by an instruction
 636		 * fetch error report -- only treat MCSR_IF as fatal
 637		 * if it wasn't due to an L1 parity error.
 638		 */
 639		reason &= ~MCSR_IF;
 640	}
 641
 642	if (reason & MCSR_DCPERR_MC) {
 643		pr_cont("Data Cache Parity Error\n");
 644
 645		/*
 646		 * In write shadow mode we auto-recover from the error, but it
 647		 * may still get logged and cause a machine check.  We should
 648		 * only treat the non-write shadow case as non-recoverable.
 649		 */
 650		/* On e6500 core, L1 DCWS (Data cache write shadow mode) bit
 651		 * is not implemented but L1 data cache always runs in write
 652		 * shadow mode. Hence on data cache parity errors HW will
 653		 * automatically invalidate the L1 Data Cache.
 654		 */
 655		if (PVR_VER(pvr) != PVR_VER_E6500) {
 656			if (!(mfspr(SPRN_L1CSR2) & L1CSR2_DCWS))
 657				recoverable = 0;
 658		}
 659	}
 660
 661	if (reason & MCSR_L2MMU_MHIT) {
 662		pr_cont("Hit on multiple TLB entries\n");
 663		recoverable = 0;
 664	}
 665
 666	if (reason & MCSR_NMI)
 667		pr_cont("Non-maskable interrupt\n");
 668
 669	if (reason & MCSR_IF) {
 670		pr_cont("Instruction Fetch Error Report\n");
 671		recoverable = 0;
 672	}
 673
 674	if (reason & MCSR_LD) {
 675		pr_cont("Load Error Report\n");
 676		recoverable = 0;
 677	}
 678
 679	if (reason & MCSR_ST) {
 680		pr_cont("Store Error Report\n");
 681		recoverable = 0;
 682	}
 683
 684	if (reason & MCSR_LDG) {
 685		pr_cont("Guarded Load Error Report\n");
 686		recoverable = 0;
 687	}
 688
 689	if (reason & MCSR_TLBSYNC)
 690		pr_cont("Simultaneous tlbsync operations\n");
 691
 692	if (reason & MCSR_BSL2_ERR) {
 693		pr_cont("Level 2 Cache Error\n");
 694		recoverable = 0;
 695	}
 696
 697	if (reason & MCSR_MAV) {
 698		u64 addr;
 699
 700		addr = mfspr(SPRN_MCAR);
 701		addr |= (u64)mfspr(SPRN_MCARU) << 32;
 702
 703		pr_cont("Machine Check %s Address: %#llx\n",
 704		       reason & MCSR_MEA ? "Effective" : "Physical", addr);
 705	}
 706
 707silent_out:
 708	mtspr(SPRN_MCSR, mcsr);
 709	return mfspr(SPRN_MCSR) == 0 && recoverable;
 710}
 711
 712int machine_check_e500(struct pt_regs *regs)
 713{
 714	unsigned long reason = mfspr(SPRN_MCSR);
 715
 716	if (reason & MCSR_BUS_RBERR) {
 717		if (fsl_rio_mcheck_exception(regs))
 718			return 1;
 719		if (fsl_pci_mcheck_exception(regs))
 720			return 1;
 721	}
 722
 723	printk("Machine check in kernel mode.\n");
 724	printk("Caused by (from MCSR=%lx): ", reason);
 725
 726	if (reason & MCSR_MCP)
 727		pr_cont("Machine Check Signal\n");
 728	if (reason & MCSR_ICPERR)
 729		pr_cont("Instruction Cache Parity Error\n");
 730	if (reason & MCSR_DCP_PERR)
 731		pr_cont("Data Cache Push Parity Error\n");
 732	if (reason & MCSR_DCPERR)
 733		pr_cont("Data Cache Parity Error\n");
 734	if (reason & MCSR_BUS_IAERR)
 735		pr_cont("Bus - Instruction Address Error\n");
 736	if (reason & MCSR_BUS_RAERR)
 737		pr_cont("Bus - Read Address Error\n");
 738	if (reason & MCSR_BUS_WAERR)
 739		pr_cont("Bus - Write Address Error\n");
 740	if (reason & MCSR_BUS_IBERR)
 741		pr_cont("Bus - Instruction Data Error\n");
 742	if (reason & MCSR_BUS_RBERR)
 743		pr_cont("Bus - Read Data Bus Error\n");
 744	if (reason & MCSR_BUS_WBERR)
 745		pr_cont("Bus - Write Data Bus Error\n");
 746	if (reason & MCSR_BUS_IPERR)
 747		pr_cont("Bus - Instruction Parity Error\n");
 748	if (reason & MCSR_BUS_RPERR)
 749		pr_cont("Bus - Read Parity Error\n");
 750
 751	return 0;
 752}
 753
 754int machine_check_generic(struct pt_regs *regs)
 755{
 756	return 0;
 757}
 758#elif defined(CONFIG_PPC32)
 759int machine_check_generic(struct pt_regs *regs)
 760{
 761	unsigned long reason = regs->msr;
 762
 763	printk("Machine check in kernel mode.\n");
 764	printk("Caused by (from SRR1=%lx): ", reason);
 765	switch (reason & 0x601F0000) {
 766	case 0x80000:
 767		pr_cont("Machine check signal\n");
 768		break;
 769	case 0x40000:
 770	case 0x140000:	/* 7450 MSS error and TEA */
 771		pr_cont("Transfer error ack signal\n");
 772		break;
 773	case 0x20000:
 774		pr_cont("Data parity error signal\n");
 775		break;
 776	case 0x10000:
 777		pr_cont("Address parity error signal\n");
 778		break;
 779	case 0x20000000:
 780		pr_cont("L1 Data Cache error\n");
 781		break;
 782	case 0x40000000:
 783		pr_cont("L1 Instruction Cache error\n");
 784		break;
 785	case 0x00100000:
 786		pr_cont("L2 data cache parity error\n");
 787		break;
 788	default:
 789		pr_cont("Unknown values in msr\n");
 790	}
 791	return 0;
 792}
 793#endif /* everything else */
 794
 795void die_mce(const char *str, struct pt_regs *regs, long err)
 796{
 797	/*
 798	 * The machine check wants to kill the interrupted context,
 799	 * but make_task_dead() checks for in_interrupt() and panics
 800	 * in that case, so exit the irq/nmi before calling die.
 801	 */
 802	if (in_nmi())
 803		nmi_exit();
 804	else
 805		irq_exit();
 806	die(str, regs, err);
 807}
 808
 809/*
 810 * BOOK3S_64 does not usually call this handler as a non-maskable interrupt
 811 * (it uses its own early real-mode handler to handle the MCE proper
 812 * and then raises irq_work to call this handler when interrupts are
 813 * enabled). The only time when this is not true is if the early handler
 814 * is unrecoverable, then it does call this directly to try to get a
 815 * message out.
 816 */
 817static void __machine_check_exception(struct pt_regs *regs)
 818{
 819	int recover = 0;
 820
 821	__this_cpu_inc(irq_stat.mce_exceptions);
 822
 823	add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
 824
 825	/* See if any machine dependent calls. In theory, we would want
 826	 * to call the CPU first, and call the ppc_md. one if the CPU
 827	 * one returns a positive number. However there is existing code
 828	 * that assumes the board gets a first chance, so let's keep it
 829	 * that way for now and fix things later. --BenH.
 830	 */
 831	if (ppc_md.machine_check_exception)
 832		recover = ppc_md.machine_check_exception(regs);
 833	else if (cur_cpu_spec->machine_check)
 834		recover = cur_cpu_spec->machine_check(regs);
 835
 836	if (recover > 0)
 837		goto bail;
 838
 839	if (debugger_fault_handler(regs))
 840		goto bail;
 841
 842	if (check_io_access(regs))
 843		goto bail;
 844
 845	die_mce("Machine check", regs, SIGBUS);
 846
 847bail:
 848	/* Must die if the interrupt is not recoverable */
 849	if (regs_is_unrecoverable(regs))
 850		die_mce("Unrecoverable Machine check", regs, SIGBUS);
 851}
 852
 853#ifdef CONFIG_PPC_BOOK3S_64
 854DEFINE_INTERRUPT_HANDLER_RAW(machine_check_early_boot)
 855{
 856	udbg_printf("Machine check (early boot)\n");
 857	udbg_printf("SRR0=0x%016lx   SRR1=0x%016lx\n", regs->nip, regs->msr);
 858	udbg_printf(" DAR=0x%016lx  DSISR=0x%08lx\n", regs->dar, regs->dsisr);
 859	udbg_printf("  LR=0x%016lx     R1=0x%08lx\n", regs->link, regs->gpr[1]);
 860	udbg_printf("------\n");
 861	die("Machine check (early boot)", regs, SIGBUS);
 862	for (;;)
 863		;
 864	return 0;
 865}
 866
 867DEFINE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async)
 868{
 869	__machine_check_exception(regs);
 870}
 871#endif
 872DEFINE_INTERRUPT_HANDLER_NMI(machine_check_exception)
 873{
 874	__machine_check_exception(regs);
 875
 876	return 0;
 877}
 878
 879DEFINE_INTERRUPT_HANDLER(SMIException) /* async? */
 880{
 881	die("System Management Interrupt", regs, SIGABRT);
 882}
 883
 884#ifdef CONFIG_VSX
 885static void p9_hmi_special_emu(struct pt_regs *regs)
 886{
 887	unsigned int ra, rb, t, i, sel, instr, rc;
 888	const void __user *addr;
 889	u8 vbuf[16] __aligned(16), *vdst;
 890	unsigned long ea, msr, msr_mask;
 891	bool swap;
 892
 893	if (__get_user(instr, (unsigned int __user *)regs->nip))
 894		return;
 895
 896	/*
 897	 * lxvb16x	opcode: 0x7c0006d8
 898	 * lxvd2x	opcode: 0x7c000698
 899	 * lxvh8x	opcode: 0x7c000658
 900	 * lxvw4x	opcode: 0x7c000618
 901	 */
 902	if ((instr & 0xfc00073e) != 0x7c000618) {
 903		pr_devel("HMI vec emu: not vector CI %i:%s[%d] nip=%016lx"
 904			 " instr=%08x\n",
 905			 smp_processor_id(), current->comm, current->pid,
 906			 regs->nip, instr);
 907		return;
 908	}
 909
 910	/* Grab vector registers into the task struct */
 911	msr = regs->msr; /* Grab msr before we flush the bits */
 912	flush_vsx_to_thread(current);
 913	enable_kernel_altivec();
 914
 915	/*
 916	 * Is userspace running with a different endian (this is rare but
 917	 * not impossible)
 918	 */
 919	swap = (msr & MSR_LE) != (MSR_KERNEL & MSR_LE);
 920
 921	/* Decode the instruction */
 922	ra = (instr >> 16) & 0x1f;
 923	rb = (instr >> 11) & 0x1f;
 924	t = (instr >> 21) & 0x1f;
 925	if (instr & 1)
 926		vdst = (u8 *)&current->thread.vr_state.vr[t];
 927	else
 928		vdst = (u8 *)&current->thread.fp_state.fpr[t][0];
 929
 930	/* Grab the vector address */
 931	ea = regs->gpr[rb] + (ra ? regs->gpr[ra] : 0);
 932	if (is_32bit_task())
 933		ea &= 0xfffffffful;
 934	addr = (__force const void __user *)ea;
 935
 936	/* Check it */
 937	if (!access_ok(addr, 16)) {
 938		pr_devel("HMI vec emu: bad access %i:%s[%d] nip=%016lx"
 939			 " instr=%08x addr=%016lx\n",
 940			 smp_processor_id(), current->comm, current->pid,
 941			 regs->nip, instr, (unsigned long)addr);
 942		return;
 943	}
 944
 945	/* Read the vector */
 946	rc = 0;
 947	if ((unsigned long)addr & 0xfUL)
 948		/* unaligned case */
 949		rc = __copy_from_user_inatomic(vbuf, addr, 16);
 950	else
 951		__get_user_atomic_128_aligned(vbuf, addr, rc);
 952	if (rc) {
 953		pr_devel("HMI vec emu: page fault %i:%s[%d] nip=%016lx"
 954			 " instr=%08x addr=%016lx\n",
 955			 smp_processor_id(), current->comm, current->pid,
 956			 regs->nip, instr, (unsigned long)addr);
 957		return;
 958	}
 959
 960	pr_devel("HMI vec emu: emulated vector CI %i:%s[%d] nip=%016lx"
 961		 " instr=%08x addr=%016lx\n",
 962		 smp_processor_id(), current->comm, current->pid, regs->nip,
 963		 instr, (unsigned long) addr);
 964
 965	/* Grab instruction "selector" */
 966	sel = (instr >> 6) & 3;
 967
 968	/*
 969	 * Check to make sure the facility is actually enabled. This
 970	 * could happen if we get a false positive hit.
 971	 *
 972	 * lxvd2x/lxvw4x always check MSR VSX sel = 0,2
 973	 * lxvh8x/lxvb16x check MSR VSX or VEC depending on VSR used sel = 1,3
 974	 */
 975	msr_mask = MSR_VSX;
 976	if ((sel & 1) && (instr & 1)) /* lxvh8x & lxvb16x + VSR >= 32 */
 977		msr_mask = MSR_VEC;
 978	if (!(msr & msr_mask)) {
 979		pr_devel("HMI vec emu: MSR fac clear %i:%s[%d] nip=%016lx"
 980			 " instr=%08x msr:%016lx\n",
 981			 smp_processor_id(), current->comm, current->pid,
 982			 regs->nip, instr, msr);
 983		return;
 984	}
 985
 986	/* Do logging here before we modify sel based on endian */
 987	switch (sel) {
 988	case 0:	/* lxvw4x */
 989		PPC_WARN_EMULATED(lxvw4x, regs);
 990		break;
 991	case 1: /* lxvh8x */
 992		PPC_WARN_EMULATED(lxvh8x, regs);
 993		break;
 994	case 2: /* lxvd2x */
 995		PPC_WARN_EMULATED(lxvd2x, regs);
 996		break;
 997	case 3: /* lxvb16x */
 998		PPC_WARN_EMULATED(lxvb16x, regs);
 999		break;
1000	}
1001
1002#ifdef __LITTLE_ENDIAN__
1003	/*
1004	 * An LE kernel stores the vector in the task struct as an LE
1005	 * byte array (effectively swapping both the components and
1006	 * the content of the components). Those instructions expect
1007	 * the components to remain in ascending address order, so we
1008	 * swap them back.
1009	 *
1010	 * If we are running a BE user space, the expectation is that
1011	 * of a simple memcpy, so forcing the emulation to look like
1012	 * a lxvb16x should do the trick.
1013	 */
1014	if (swap)
1015		sel = 3;
1016
1017	switch (sel) {
1018	case 0:	/* lxvw4x */
1019		for (i = 0; i < 4; i++)
1020			((u32 *)vdst)[i] = ((u32 *)vbuf)[3-i];
1021		break;
1022	case 1: /* lxvh8x */
1023		for (i = 0; i < 8; i++)
1024			((u16 *)vdst)[i] = ((u16 *)vbuf)[7-i];
1025		break;
1026	case 2: /* lxvd2x */
1027		for (i = 0; i < 2; i++)
1028			((u64 *)vdst)[i] = ((u64 *)vbuf)[1-i];
1029		break;
1030	case 3: /* lxvb16x */
1031		for (i = 0; i < 16; i++)
1032			vdst[i] = vbuf[15-i];
1033		break;
1034	}
1035#else /* __LITTLE_ENDIAN__ */
1036	/* On a big endian kernel, a BE userspace only needs a memcpy */
1037	if (!swap)
1038		sel = 3;
1039
1040	/* Otherwise, we need to swap the content of the components */
1041	switch (sel) {
1042	case 0:	/* lxvw4x */
1043		for (i = 0; i < 4; i++)
1044			((u32 *)vdst)[i] = cpu_to_le32(((u32 *)vbuf)[i]);
1045		break;
1046	case 1: /* lxvh8x */
1047		for (i = 0; i < 8; i++)
1048			((u16 *)vdst)[i] = cpu_to_le16(((u16 *)vbuf)[i]);
1049		break;
1050	case 2: /* lxvd2x */
1051		for (i = 0; i < 2; i++)
1052			((u64 *)vdst)[i] = cpu_to_le64(((u64 *)vbuf)[i]);
1053		break;
1054	case 3: /* lxvb16x */
1055		memcpy(vdst, vbuf, 16);
1056		break;
1057	}
1058#endif /* !__LITTLE_ENDIAN__ */
1059
1060	/* Go to next instruction */
1061	regs_add_return_ip(regs, 4);
1062}
1063#endif /* CONFIG_VSX */
1064
1065DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception)
1066{
1067	struct pt_regs *old_regs;
1068
1069	old_regs = set_irq_regs(regs);
1070
1071#ifdef CONFIG_VSX
1072	/* Real mode flagged P9 special emu is needed */
1073	if (local_paca->hmi_p9_special_emu) {
1074		local_paca->hmi_p9_special_emu = 0;
1075
1076		/*
1077		 * We don't want to take page faults while doing the
1078		 * emulation, we just replay the instruction if necessary.
1079		 */
1080		pagefault_disable();
1081		p9_hmi_special_emu(regs);
1082		pagefault_enable();
1083	}
1084#endif /* CONFIG_VSX */
1085
1086	if (ppc_md.handle_hmi_exception)
1087		ppc_md.handle_hmi_exception(regs);
1088
1089	set_irq_regs(old_regs);
1090}
1091
1092DEFINE_INTERRUPT_HANDLER(unknown_exception)
1093{
1094	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
1095	       regs->nip, regs->msr, regs->trap);
1096
1097	_exception(SIGTRAP, regs, TRAP_UNK, 0);
1098}
1099
1100DEFINE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception)
1101{
1102	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
1103	       regs->nip, regs->msr, regs->trap);
1104
1105	_exception(SIGTRAP, regs, TRAP_UNK, 0);
1106}
1107
1108DEFINE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception)
1109{
1110	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
1111	       regs->nip, regs->msr, regs->trap);
1112
1113	_exception(SIGTRAP, regs, TRAP_UNK, 0);
1114
1115	return 0;
1116}
1117
1118DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception)
1119{
1120	if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5,
1121					5, SIGTRAP) == NOTIFY_STOP)
1122		return;
1123	if (debugger_iabr_match(regs))
1124		return;
1125	_exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
1126}
1127
1128DEFINE_INTERRUPT_HANDLER(RunModeException)
1129{
1130	_exception(SIGTRAP, regs, TRAP_UNK, 0);
1131}
1132
1133static void __single_step_exception(struct pt_regs *regs)
1134{
1135	clear_single_step(regs);
1136	clear_br_trace(regs);
1137
1138	if (kprobe_post_handler(regs))
1139		return;
1140
1141	if (notify_die(DIE_SSTEP, "single_step", regs, 5,
1142					5, SIGTRAP) == NOTIFY_STOP)
1143		return;
1144	if (debugger_sstep(regs))
1145		return;
1146
1147	_exception(SIGTRAP, regs, TRAP_TRACE, regs->nip);
1148}
1149
1150DEFINE_INTERRUPT_HANDLER(single_step_exception)
1151{
1152	__single_step_exception(regs);
1153}
1154
1155/*
1156 * After we have successfully emulated an instruction, we have to
1157 * check if the instruction was being single-stepped, and if so,
1158 * pretend we got a single-step exception.  This was pointed out
1159 * by Kumar Gala.  -- paulus
1160 */
1161static void emulate_single_step(struct pt_regs *regs)
1162{
1163	if (single_stepping(regs))
1164		__single_step_exception(regs);
1165}
1166
1167static inline int __parse_fpscr(unsigned long fpscr)
1168{
1169	int ret = FPE_FLTUNK;
1170
1171	/* Invalid operation */
1172	if ((fpscr & FPSCR_VE) && (fpscr & FPSCR_VX))
1173		ret = FPE_FLTINV;
1174
1175	/* Overflow */
1176	else if ((fpscr & FPSCR_OE) && (fpscr & FPSCR_OX))
1177		ret = FPE_FLTOVF;
1178
1179	/* Underflow */
1180	else if ((fpscr & FPSCR_UE) && (fpscr & FPSCR_UX))
1181		ret = FPE_FLTUND;
1182
1183	/* Divide by zero */
1184	else if ((fpscr & FPSCR_ZE) && (fpscr & FPSCR_ZX))
1185		ret = FPE_FLTDIV;
1186
1187	/* Inexact result */
1188	else if ((fpscr & FPSCR_XE) && (fpscr & FPSCR_XX))
1189		ret = FPE_FLTRES;
1190
1191	return ret;
1192}
1193
1194static void parse_fpe(struct pt_regs *regs)
1195{
1196	int code = 0;
1197
1198	flush_fp_to_thread(current);
1199
1200#ifdef CONFIG_PPC_FPU_REGS
1201	code = __parse_fpscr(current->thread.fp_state.fpscr);
1202#endif
1203
1204	_exception(SIGFPE, regs, code, regs->nip);
1205}
1206
1207/*
1208 * Illegal instruction emulation support.  Originally written to
1209 * provide the PVR to user applications using the mfspr rd, PVR.
1210 * Return non-zero if we can't emulate, or -EFAULT if the associated
1211 * memory access caused an access fault.  Return zero on success.
1212 *
1213 * There are a couple of ways to do this, either "decode" the instruction
1214 * or directly match lots of bits.  In this case, matching lots of
1215 * bits is faster and easier.
1216 *
1217 */
1218static int emulate_string_inst(struct pt_regs *regs, u32 instword)
1219{
1220	u8 rT = (instword >> 21) & 0x1f;
1221	u8 rA = (instword >> 16) & 0x1f;
1222	u8 NB_RB = (instword >> 11) & 0x1f;
1223	u32 num_bytes;
1224	unsigned long EA;
1225	int pos = 0;
1226
1227	/* Early out if we are an invalid form of lswx */
1228	if ((instword & PPC_INST_STRING_MASK) == PPC_INST_LSWX)
1229		if ((rT == rA) || (rT == NB_RB))
1230			return -EINVAL;
1231
1232	EA = (rA == 0) ? 0 : regs->gpr[rA];
1233
1234	switch (instword & PPC_INST_STRING_MASK) {
1235		case PPC_INST_LSWX:
1236		case PPC_INST_STSWX:
1237			EA += NB_RB;
1238			num_bytes = regs->xer & 0x7f;
1239			break;
1240		case PPC_INST_LSWI:
1241		case PPC_INST_STSWI:
1242			num_bytes = (NB_RB == 0) ? 32 : NB_RB;
1243			break;
1244		default:
1245			return -EINVAL;
1246	}
1247
1248	while (num_bytes != 0)
1249	{
1250		u8 val;
1251		u32 shift = 8 * (3 - (pos & 0x3));
1252
1253		/* if process is 32-bit, clear upper 32 bits of EA */
1254		if ((regs->msr & MSR_64BIT) == 0)
1255			EA &= 0xFFFFFFFF;
1256
1257		switch ((instword & PPC_INST_STRING_MASK)) {
1258			case PPC_INST_LSWX:
1259			case PPC_INST_LSWI:
1260				if (get_user(val, (u8 __user *)EA))
1261					return -EFAULT;
1262				/* first time updating this reg,
1263				 * zero it out */
1264				if (pos == 0)
1265					regs->gpr[rT] = 0;
1266				regs->gpr[rT] |= val << shift;
1267				break;
1268			case PPC_INST_STSWI:
1269			case PPC_INST_STSWX:
1270				val = regs->gpr[rT] >> shift;
1271				if (put_user(val, (u8 __user *)EA))
1272					return -EFAULT;
1273				break;
1274		}
1275		/* move EA to next address */
1276		EA += 1;
1277		num_bytes--;
1278
1279		/* manage our position within the register */
1280		if (++pos == 4) {
1281			pos = 0;
1282			if (++rT == 32)
1283				rT = 0;
1284		}
1285	}
1286
1287	return 0;
1288}
1289
1290static int emulate_popcntb_inst(struct pt_regs *regs, u32 instword)
1291{
1292	u32 ra,rs;
1293	unsigned long tmp;
1294
1295	ra = (instword >> 16) & 0x1f;
1296	rs = (instword >> 21) & 0x1f;
1297
1298	tmp = regs->gpr[rs];
1299	tmp = tmp - ((tmp >> 1) & 0x5555555555555555ULL);
1300	tmp = (tmp & 0x3333333333333333ULL) + ((tmp >> 2) & 0x3333333333333333ULL);
1301	tmp = (tmp + (tmp >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
1302	regs->gpr[ra] = tmp;
1303
1304	return 0;
1305}
1306
1307static int emulate_isel(struct pt_regs *regs, u32 instword)
1308{
1309	u8 rT = (instword >> 21) & 0x1f;
1310	u8 rA = (instword >> 16) & 0x1f;
1311	u8 rB = (instword >> 11) & 0x1f;
1312	u8 BC = (instword >> 6) & 0x1f;
1313	u8 bit;
1314	unsigned long tmp;
1315
1316	tmp = (rA == 0) ? 0 : regs->gpr[rA];
1317	bit = (regs->ccr >> (31 - BC)) & 0x1;
1318
1319	regs->gpr[rT] = bit ? tmp : regs->gpr[rB];
1320
1321	return 0;
1322}
1323
1324#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1325static inline bool tm_abort_check(struct pt_regs *regs, int cause)
1326{
1327        /* If we're emulating a load/store in an active transaction, we cannot
1328         * emulate it as the kernel operates in transaction suspended context.
1329         * We need to abort the transaction.  This creates a persistent TM
1330         * abort so tell the user what caused it with a new code.
1331	 */
1332	if (MSR_TM_TRANSACTIONAL(regs->msr)) {
1333		tm_enable();
1334		tm_abort(cause);
1335		return true;
1336	}
1337	return false;
1338}
1339#else
1340static inline bool tm_abort_check(struct pt_regs *regs, int reason)
1341{
1342	return false;
1343}
1344#endif
1345
1346static int emulate_instruction(struct pt_regs *regs)
1347{
1348	u32 instword;
1349	u32 rd;
1350
1351	if (!user_mode(regs))
1352		return -EINVAL;
1353
1354	if (get_user(instword, (u32 __user *)(regs->nip)))
1355		return -EFAULT;
1356
1357	/* Emulate the mfspr rD, PVR. */
1358	if ((instword & PPC_INST_MFSPR_PVR_MASK) == PPC_INST_MFSPR_PVR) {
1359		PPC_WARN_EMULATED(mfpvr, regs);
1360		rd = (instword >> 21) & 0x1f;
1361		regs->gpr[rd] = mfspr(SPRN_PVR);
1362		return 0;
1363	}
1364
1365	/* Emulating the dcba insn is just a no-op.  */
1366	if ((instword & PPC_INST_DCBA_MASK) == PPC_INST_DCBA) {
1367		PPC_WARN_EMULATED(dcba, regs);
1368		return 0;
1369	}
1370
1371	/* Emulate the mcrxr insn.  */
1372	if ((instword & PPC_INST_MCRXR_MASK) == PPC_INST_MCRXR) {
1373		int shift = (instword >> 21) & 0x1c;
1374		unsigned long msk = 0xf0000000UL >> shift;
1375
1376		PPC_WARN_EMULATED(mcrxr, regs);
1377		regs->ccr = (regs->ccr & ~msk) | ((regs->xer >> shift) & msk);
1378		regs->xer &= ~0xf0000000UL;
1379		return 0;
1380	}
1381
1382	/* Emulate load/store string insn. */
1383	if ((instword & PPC_INST_STRING_GEN_MASK) == PPC_INST_STRING) {
1384		if (tm_abort_check(regs,
1385				   TM_CAUSE_EMULATE | TM_CAUSE_PERSISTENT))
1386			return -EINVAL;
1387		PPC_WARN_EMULATED(string, regs);
1388		return emulate_string_inst(regs, instword);
1389	}
1390
1391	/* Emulate the popcntb (Population Count Bytes) instruction. */
1392	if ((instword & PPC_INST_POPCNTB_MASK) == PPC_INST_POPCNTB) {
1393		PPC_WARN_EMULATED(popcntb, regs);
1394		return emulate_popcntb_inst(regs, instword);
1395	}
1396
1397	/* Emulate isel (Integer Select) instruction */
1398	if ((instword & PPC_INST_ISEL_MASK) == PPC_INST_ISEL) {
1399		PPC_WARN_EMULATED(isel, regs);
1400		return emulate_isel(regs, instword);
1401	}
1402
1403	/* Emulate sync instruction variants */
1404	if ((instword & PPC_INST_SYNC_MASK) == PPC_INST_SYNC) {
1405		PPC_WARN_EMULATED(sync, regs);
1406		asm volatile("sync");
1407		return 0;
1408	}
1409
1410#ifdef CONFIG_PPC64
1411	/* Emulate the mfspr rD, DSCR. */
1412	if ((((instword & PPC_INST_MFSPR_DSCR_USER_MASK) ==
1413		PPC_INST_MFSPR_DSCR_USER) ||
1414	     ((instword & PPC_INST_MFSPR_DSCR_MASK) ==
1415		PPC_INST_MFSPR_DSCR)) &&
1416			cpu_has_feature(CPU_FTR_DSCR)) {
1417		PPC_WARN_EMULATED(mfdscr, regs);
1418		rd = (instword >> 21) & 0x1f;
1419		regs->gpr[rd] = mfspr(SPRN_DSCR);
1420		return 0;
1421	}
1422	/* Emulate the mtspr DSCR, rD. */
1423	if ((((instword & PPC_INST_MTSPR_DSCR_USER_MASK) ==
1424		PPC_INST_MTSPR_DSCR_USER) ||
1425	     ((instword & PPC_INST_MTSPR_DSCR_MASK) ==
1426		PPC_INST_MTSPR_DSCR)) &&
1427			cpu_has_feature(CPU_FTR_DSCR)) {
1428		PPC_WARN_EMULATED(mtdscr, regs);
1429		rd = (instword >> 21) & 0x1f;
1430		current->thread.dscr = regs->gpr[rd];
1431		current->thread.dscr_inherit = 1;
1432		mtspr(SPRN_DSCR, current->thread.dscr);
1433		return 0;
1434	}
1435#endif
1436
1437	return -EINVAL;
1438}
1439
1440int is_valid_bugaddr(unsigned long addr)
1441{
1442	return is_kernel_addr(addr);
1443}
1444
1445#ifdef CONFIG_MATH_EMULATION
1446static int emulate_math(struct pt_regs *regs)
1447{
1448	int ret;
1449
1450	ret = do_mathemu(regs);
1451	if (ret >= 0)
1452		PPC_WARN_EMULATED(math, regs);
1453
1454	switch (ret) {
1455	case 0:
1456		emulate_single_step(regs);
1457		return 0;
1458	case 1: {
1459			int code = 0;
1460			code = __parse_fpscr(current->thread.fp_state.fpscr);
1461			_exception(SIGFPE, regs, code, regs->nip);
1462			return 0;
1463		}
1464	case -EFAULT:
1465		_exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip);
1466		return 0;
1467	}
1468
1469	return -1;
1470}
1471#else
1472static inline int emulate_math(struct pt_regs *regs) { return -1; }
1473#endif
1474
1475static void do_program_check(struct pt_regs *regs)
1476{
1477	unsigned int reason = get_reason(regs);
1478
1479	/* We can now get here via a FP Unavailable exception if the core
1480	 * has no FPU, in that case the reason flags will be 0 */
1481
1482	if (reason & REASON_FP) {
1483		/* IEEE FP exception */
1484		parse_fpe(regs);
1485		return;
1486	}
1487	if (reason & REASON_TRAP) {
1488		unsigned long bugaddr;
1489		/* Debugger is first in line to stop recursive faults in
1490		 * rcu_lock, notify_die, or atomic_notifier_call_chain */
1491		if (debugger_bpt(regs))
1492			return;
1493
1494		if (kprobe_handler(regs))
1495			return;
1496
1497		/* trap exception */
1498		if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP)
1499				== NOTIFY_STOP)
1500			return;
1501
1502		bugaddr = regs->nip;
1503		/*
1504		 * Fixup bugaddr for BUG_ON() in real mode
1505		 */
1506		if (!is_kernel_addr(bugaddr) && !(regs->msr & MSR_IR))
1507			bugaddr += PAGE_OFFSET;
1508
1509		if (!(regs->msr & MSR_PR) &&  /* not user-mode */
1510		    report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) {
1511			const struct exception_table_entry *entry;
1512
1513			entry = search_exception_tables(bugaddr);
1514			if (entry) {
1515				regs_set_return_ip(regs, extable_fixup(entry) + regs->nip - bugaddr);
1516				return;
1517			}
1518		}
1519		_exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
1520		return;
1521	}
1522#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1523	if (reason & REASON_TM) {
1524		/* This is a TM "Bad Thing Exception" program check.
1525		 * This occurs when:
1526		 * -  An rfid/hrfid/mtmsrd attempts to cause an illegal
1527		 *    transition in TM states.
1528		 * -  A trechkpt is attempted when transactional.
1529		 * -  A treclaim is attempted when non transactional.
1530		 * -  A tend is illegally attempted.
1531		 * -  writing a TM SPR when transactional.
1532		 *
1533		 * If usermode caused this, it's done something illegal and
1534		 * gets a SIGILL slap on the wrist.  We call it an illegal
1535		 * operand to distinguish from the instruction just being bad
1536		 * (e.g. executing a 'tend' on a CPU without TM!); it's an
1537		 * illegal /placement/ of a valid instruction.
1538		 */
1539		if (user_mode(regs)) {
1540			_exception(SIGILL, regs, ILL_ILLOPN, regs->nip);
1541			return;
1542		} else {
1543			printk(KERN_EMERG "Unexpected TM Bad Thing exception "
1544			       "at %lx (msr 0x%lx) tm_scratch=%llx\n",
1545			       regs->nip, regs->msr, get_paca()->tm_scratch);
1546			die("Unrecoverable exception", regs, SIGABRT);
1547		}
1548	}
1549#endif
1550
1551	/*
1552	 * If we took the program check in the kernel skip down to sending a
1553	 * SIGILL. The subsequent cases all relate to emulating instructions
1554	 * which we should only do for userspace. We also do not want to enable
1555	 * interrupts for kernel faults because that might lead to further
1556	 * faults, and loose the context of the original exception.
1557	 */
1558	if (!user_mode(regs))
1559		goto sigill;
1560
1561	interrupt_cond_local_irq_enable(regs);
1562
1563	/* (reason & REASON_ILLEGAL) would be the obvious thing here,
1564	 * but there seems to be a hardware bug on the 405GP (RevD)
1565	 * that means ESR is sometimes set incorrectly - either to
1566	 * ESR_DST (!?) or 0.  In the process of chasing this with the
1567	 * hardware people - not sure if it can happen on any illegal
1568	 * instruction or only on FP instructions, whether there is a
1569	 * pattern to occurrences etc. -dgibson 31/Mar/2003
1570	 */
1571	if (!emulate_math(regs))
1572		return;
1573
1574	/* Try to emulate it if we should. */
1575	if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) {
1576		switch (emulate_instruction(regs)) {
1577		case 0:
1578			regs_add_return_ip(regs, 4);
1579			emulate_single_step(regs);
1580			return;
1581		case -EFAULT:
1582			_exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip);
1583			return;
1584		}
1585	}
1586
1587sigill:
1588	if (reason & REASON_PRIVILEGED)
1589		_exception(SIGILL, regs, ILL_PRVOPC, regs->nip);
1590	else
1591		_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
1592
1593}
1594
1595DEFINE_INTERRUPT_HANDLER(program_check_exception)
1596{
1597	do_program_check(regs);
1598}
1599
1600/*
1601 * This occurs when running in hypervisor mode on POWER6 or later
1602 * and an illegal instruction is encountered.
1603 */
1604DEFINE_INTERRUPT_HANDLER(emulation_assist_interrupt)
1605{
1606	regs_set_return_msr(regs, regs->msr | REASON_ILLEGAL);
1607	do_program_check(regs);
1608}
1609
1610DEFINE_INTERRUPT_HANDLER(alignment_exception)
1611{
1612	int sig, code, fixed = 0;
1613	unsigned long  reason;
1614
1615	interrupt_cond_local_irq_enable(regs);
1616
1617	reason = get_reason(regs);
1618	if (reason & REASON_BOUNDARY) {
1619		sig = SIGBUS;
1620		code = BUS_ADRALN;
1621		goto bad;
1622	}
1623
1624	if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT))
1625		return;
1626
1627	/* we don't implement logging of alignment exceptions */
1628	if (!(current->thread.align_ctl & PR_UNALIGN_SIGBUS))
1629		fixed = fix_alignment(regs);
1630
1631	if (fixed == 1) {
1632		/* skip over emulated instruction */
1633		regs_add_return_ip(regs, inst_length(reason));
1634		emulate_single_step(regs);
1635		return;
1636	}
1637
1638	/* Operand address was bad */
1639	if (fixed == -EFAULT) {
1640		sig = SIGSEGV;
1641		code = SEGV_ACCERR;
1642	} else {
1643		sig = SIGBUS;
1644		code = BUS_ADRALN;
1645	}
1646bad:
1647	if (user_mode(regs))
1648		_exception(sig, regs, code, regs->dar);
1649	else
1650		bad_page_fault(regs, sig);
1651}
1652
1653DEFINE_INTERRUPT_HANDLER(stack_overflow_exception)
1654{
1655	die("Kernel stack overflow", regs, SIGSEGV);
1656}
1657
1658DEFINE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception)
1659{
1660	printk(KERN_EMERG "Unrecoverable FP Unavailable Exception "
1661			  "%lx at %lx\n", regs->trap, regs->nip);
1662	die("Unrecoverable FP Unavailable Exception", regs, SIGABRT);
1663}
1664
1665DEFINE_INTERRUPT_HANDLER(altivec_unavailable_exception)
1666{
1667	if (user_mode(regs)) {
1668		/* A user program has executed an altivec instruction,
1669		   but this kernel doesn't support altivec. */
1670		_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
1671		return;
1672	}
1673
1674	printk(KERN_EMERG "Unrecoverable VMX/Altivec Unavailable Exception "
1675			"%lx at %lx\n", regs->trap, regs->nip);
1676	die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT);
1677}
1678
1679DEFINE_INTERRUPT_HANDLER(vsx_unavailable_exception)
1680{
1681	if (user_mode(regs)) {
1682		/* A user program has executed an vsx instruction,
1683		   but this kernel doesn't support vsx. */
1684		_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
1685		return;
1686	}
1687
1688	printk(KERN_EMERG "Unrecoverable VSX Unavailable Exception "
1689			"%lx at %lx\n", regs->trap, regs->nip);
1690	die("Unrecoverable VSX Unavailable Exception", regs, SIGABRT);
1691}
1692
1693#ifdef CONFIG_PPC_BOOK3S_64
1694static void tm_unavailable(struct pt_regs *regs)
1695{
1696#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1697	if (user_mode(regs)) {
1698		current->thread.load_tm++;
1699		regs_set_return_msr(regs, regs->msr | MSR_TM);
1700		tm_enable();
1701		tm_restore_sprs(&current->thread);
1702		return;
1703	}
1704#endif
1705	pr_emerg("Unrecoverable TM Unavailable Exception "
1706			"%lx at %lx\n", regs->trap, regs->nip);
1707	die("Unrecoverable TM Unavailable Exception", regs, SIGABRT);
1708}
1709
1710DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception)
1711{
1712	static char *facility_strings[] = {
1713		[FSCR_FP_LG] = "FPU",
1714		[FSCR_VECVSX_LG] = "VMX/VSX",
1715		[FSCR_DSCR_LG] = "DSCR",
1716		[FSCR_PM_LG] = "PMU SPRs",
1717		[FSCR_BHRB_LG] = "BHRB",
1718		[FSCR_TM_LG] = "TM",
1719		[FSCR_EBB_LG] = "EBB",
1720		[FSCR_TAR_LG] = "TAR",
1721		[FSCR_MSGP_LG] = "MSGP",
1722		[FSCR_SCV_LG] = "SCV",
1723		[FSCR_PREFIX_LG] = "PREFIX",
1724	};
1725	char *facility = "unknown";
1726	u64 value;
1727	u32 instword, rd;
1728	u8 status;
1729	bool hv;
1730
1731	hv = (TRAP(regs) == INTERRUPT_H_FAC_UNAVAIL);
1732	if (hv)
1733		value = mfspr(SPRN_HFSCR);
1734	else
1735		value = mfspr(SPRN_FSCR);
1736
1737	status = value >> 56;
1738	if ((hv || status >= 2) &&
1739	    (status < ARRAY_SIZE(facility_strings)) &&
1740	    facility_strings[status])
1741		facility = facility_strings[status];
1742
1743	/* We should not have taken this interrupt in kernel */
1744	if (!user_mode(regs)) {
1745		pr_emerg("Facility '%s' unavailable (%d) exception in kernel mode at %lx\n",
1746			 facility, status, regs->nip);
1747		die("Unexpected facility unavailable exception", regs, SIGABRT);
1748	}
1749
1750	interrupt_cond_local_irq_enable(regs);
1751
1752	if (status == FSCR_DSCR_LG) {
1753		/*
1754		 * User is accessing the DSCR register using the problem
1755		 * state only SPR number (0x03) either through a mfspr or
1756		 * a mtspr instruction. If it is a write attempt through
1757		 * a mtspr, then we set the inherit bit. This also allows
1758		 * the user to write or read the register directly in the
1759		 * future by setting via the FSCR DSCR bit. But in case it
1760		 * is a read DSCR attempt through a mfspr instruction, we
1761		 * just emulate the instruction instead. This code path will
1762		 * always emulate all the mfspr instructions till the user
1763		 * has attempted at least one mtspr instruction. This way it
1764		 * preserves the same behaviour when the user is accessing
1765		 * the DSCR through privilege level only SPR number (0x11)
1766		 * which is emulated through illegal instruction exception.
1767		 * We always leave HFSCR DSCR set.
1768		 */
1769		if (get_user(instword, (u32 __user *)(regs->nip))) {
1770			pr_err("Failed to fetch the user instruction\n");
1771			return;
1772		}
1773
1774		/* Write into DSCR (mtspr 0x03, RS) */
1775		if ((instword & PPC_INST_MTSPR_DSCR_USER_MASK)
1776				== PPC_INST_MTSPR_DSCR_USER) {
1777			rd = (instword >> 21) & 0x1f;
1778			current->thread.dscr = regs->gpr[rd];
1779			current->thread.dscr_inherit = 1;
1780			current->thread.fscr |= FSCR_DSCR;
1781			mtspr(SPRN_FSCR, current->thread.fscr);
1782		}
1783
1784		/* Read from DSCR (mfspr RT, 0x03) */
1785		if ((instword & PPC_INST_MFSPR_DSCR_USER_MASK)
1786				== PPC_INST_MFSPR_DSCR_USER) {
1787			if (emulate_instruction(regs)) {
1788				pr_err("DSCR based mfspr emulation failed\n");
1789				return;
1790			}
1791			regs_add_return_ip(regs, 4);
1792			emulate_single_step(regs);
1793		}
1794		return;
1795	}
1796
1797	if (status == FSCR_TM_LG) {
1798		/*
1799		 * If we're here then the hardware is TM aware because it
1800		 * generated an exception with FSRM_TM set.
1801		 *
1802		 * If cpu_has_feature(CPU_FTR_TM) is false, then either firmware
1803		 * told us not to do TM, or the kernel is not built with TM
1804		 * support.
1805		 *
1806		 * If both of those things are true, then userspace can spam the
1807		 * console by triggering the printk() below just by continually
1808		 * doing tbegin (or any TM instruction). So in that case just
1809		 * send the process a SIGILL immediately.
1810		 */
1811		if (!cpu_has_feature(CPU_FTR_TM))
1812			goto out;
1813
1814		tm_unavailable(regs);
1815		return;
1816	}
1817
1818	pr_err_ratelimited("%sFacility '%s' unavailable (%d), exception at 0x%lx, MSR=%lx\n",
1819		hv ? "Hypervisor " : "", facility, status, regs->nip, regs->msr);
1820
1821out:
1822	_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
1823}
1824#endif
1825
1826#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1827
1828DEFINE_INTERRUPT_HANDLER(fp_unavailable_tm)
1829{
1830	/* Note:  This does not handle any kind of FP laziness. */
1831
1832	TM_DEBUG("FP Unavailable trap whilst transactional at 0x%lx, MSR=%lx\n",
1833		 regs->nip, regs->msr);
1834
1835        /* We can only have got here if the task started using FP after
1836         * beginning the transaction.  So, the transactional regs are just a
1837         * copy of the checkpointed ones.  But, we still need to recheckpoint
1838         * as we're enabling FP for the process; it will return, abort the
1839         * transaction, and probably retry but now with FP enabled.  So the
1840         * checkpointed FP registers need to be loaded.
1841	 */
1842	tm_reclaim_current(TM_CAUSE_FAC_UNAV);
1843
1844	/*
1845	 * Reclaim initially saved out bogus (lazy) FPRs to ckfp_state, and
1846	 * then it was overwrite by the thr->fp_state by tm_reclaim_thread().
1847	 *
1848	 * At this point, ck{fp,vr}_state contains the exact values we want to
1849	 * recheckpoint.
1850	 */
1851
1852	/* Enable FP for the task: */
1853	current->thread.load_fp = 1;
1854
1855	/*
1856	 * Recheckpoint all the checkpointed ckpt, ck{fp, vr}_state registers.
1857	 */
1858	tm_recheckpoint(&current->thread);
1859}
1860
1861DEFINE_INTERRUPT_HANDLER(altivec_unavailable_tm)
1862{
1863	/* See the comments in fp_unavailable_tm().  This function operates
1864	 * the same way.
1865	 */
1866
1867	TM_DEBUG("Vector Unavailable trap whilst transactional at 0x%lx,"
1868		 "MSR=%lx\n",
1869		 regs->nip, regs->msr);
1870	tm_reclaim_current(TM_CAUSE_FAC_UNAV);
1871	current->thread.load_vec = 1;
1872	tm_recheckpoint(&current->thread);
1873	current->thread.used_vr = 1;
1874}
1875
1876DEFINE_INTERRUPT_HANDLER(vsx_unavailable_tm)
1877{
1878	/* See the comments in fp_unavailable_tm().  This works similarly,
1879	 * though we're loading both FP and VEC registers in here.
1880	 *
1881	 * If FP isn't in use, load FP regs.  If VEC isn't in use, load VEC
1882	 * regs.  Either way, set MSR_VSX.
1883	 */
1884
1885	TM_DEBUG("VSX Unavailable trap whilst transactional at 0x%lx,"
1886		 "MSR=%lx\n",
1887		 regs->nip, regs->msr);
1888
1889	current->thread.used_vsr = 1;
1890
1891	/* This reclaims FP and/or VR regs if they're already enabled */
1892	tm_reclaim_current(TM_CAUSE_FAC_UNAV);
1893
1894	current->thread.load_vec = 1;
1895	current->thread.load_fp = 1;
1896
1897	tm_recheckpoint(&current->thread);
1898}
1899#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1900
1901#ifdef CONFIG_PPC64
1902DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi);
1903DEFINE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi)
1904{
1905	__this_cpu_inc(irq_stat.pmu_irqs);
1906
1907	perf_irq(regs);
1908
1909	return 0;
1910}
1911#endif
1912
1913DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async);
1914DEFINE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async)
1915{
1916	__this_cpu_inc(irq_stat.pmu_irqs);
1917
1918	perf_irq(regs);
1919}
1920
1921DEFINE_INTERRUPT_HANDLER_RAW(performance_monitor_exception)
1922{
1923	/*
1924	 * On 64-bit, if perf interrupts hit in a local_irq_disable
1925	 * (soft-masked) region, we consider them as NMIs. This is required to
1926	 * prevent hash faults on user addresses when reading callchains (and
1927	 * looks better from an irq tracing perspective).
1928	 */
1929	if (IS_ENABLED(CONFIG_PPC64) && unlikely(arch_irq_disabled_regs(regs)))
1930		performance_monitor_exception_nmi(regs);
1931	else
1932		performance_monitor_exception_async(regs);
1933
1934	return 0;
1935}
1936
1937#ifdef CONFIG_PPC_ADV_DEBUG_REGS
1938static void handle_debug(struct pt_regs *regs, unsigned long debug_status)
1939{
1940	int changed = 0;
1941	/*
1942	 * Determine the cause of the debug event, clear the
1943	 * event flags and send a trap to the handler. Torez
1944	 */
1945	if (debug_status & (DBSR_DAC1R | DBSR_DAC1W)) {
1946		dbcr_dac(current) &= ~(DBCR_DAC1R | DBCR_DAC1W);
1947#ifdef CONFIG_PPC_ADV_DEBUG_DAC_RANGE
1948		current->thread.debug.dbcr2 &= ~DBCR2_DAC12MODE;
1949#endif
1950		do_send_trap(regs, mfspr(SPRN_DAC1), debug_status,
1951			     5);
1952		changed |= 0x01;
1953	}  else if (debug_status & (DBSR_DAC2R | DBSR_DAC2W)) {
1954		dbcr_dac(current) &= ~(DBCR_DAC2R | DBCR_DAC2W);
1955		do_send_trap(regs, mfspr(SPRN_DAC2), debug_status,
1956			     6);
1957		changed |= 0x01;
1958	}  else if (debug_status & DBSR_IAC1) {
1959		current->thread.debug.dbcr0 &= ~DBCR0_IAC1;
1960		dbcr_iac_range(current) &= ~DBCR_IAC12MODE;
1961		do_send_trap(regs, mfspr(SPRN_IAC1), debug_status,
1962			     1);
1963		changed |= 0x01;
1964	}  else if (debug_status & DBSR_IAC2) {
1965		current->thread.debug.dbcr0 &= ~DBCR0_IAC2;
1966		do_send_trap(regs, mfspr(SPRN_IAC2), debug_status,
1967			     2);
1968		changed |= 0x01;
1969	}  else if (debug_status & DBSR_IAC3) {
1970		current->thread.debug.dbcr0 &= ~DBCR0_IAC3;
1971		dbcr_iac_range(current) &= ~DBCR_IAC34MODE;
1972		do_send_trap(regs, mfspr(SPRN_IAC3), debug_status,
1973			     3);
1974		changed |= 0x01;
1975	}  else if (debug_status & DBSR_IAC4) {
1976		current->thread.debug.dbcr0 &= ~DBCR0_IAC4;
1977		do_send_trap(regs, mfspr(SPRN_IAC4), debug_status,
1978			     4);
1979		changed |= 0x01;
1980	}
1981	/*
1982	 * At the point this routine was called, the MSR(DE) was turned off.
1983	 * Check all other debug flags and see if that bit needs to be turned
1984	 * back on or not.
1985	 */
1986	if (DBCR_ACTIVE_EVENTS(current->thread.debug.dbcr0,
1987			       current->thread.debug.dbcr1))
1988		regs_set_return_msr(regs, regs->msr | MSR_DE);
1989	else
1990		/* Make sure the IDM flag is off */
1991		current->thread.debug.dbcr0 &= ~DBCR0_IDM;
1992
1993	if (changed & 0x01)
1994		mtspr(SPRN_DBCR0, current->thread.debug.dbcr0);
1995}
1996
1997DEFINE_INTERRUPT_HANDLER(DebugException)
1998{
1999	unsigned long debug_status = regs->dsisr;
2000
2001	current->thread.debug.dbsr = debug_status;
2002
2003	/* Hack alert: On BookE, Branch Taken stops on the branch itself, while
2004	 * on server, it stops on the target of the branch. In order to simulate
2005	 * the server behaviour, we thus restart right away with a single step
2006	 * instead of stopping here when hitting a BT
2007	 */
2008	if (debug_status & DBSR_BT) {
2009		regs_set_return_msr(regs, regs->msr & ~MSR_DE);
2010
2011		/* Disable BT */
2012		mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~DBCR0_BT);
2013		/* Clear the BT event */
2014		mtspr(SPRN_DBSR, DBSR_BT);
2015
2016		/* Do the single step trick only when coming from userspace */
2017		if (user_mode(regs)) {
2018			current->thread.debug.dbcr0 &= ~DBCR0_BT;
2019			current->thread.debug.dbcr0 |= DBCR0_IDM | DBCR0_IC;
2020			regs_set_return_msr(regs, regs->msr | MSR_DE);
2021			return;
2022		}
2023
2024		if (kprobe_post_handler(regs))
2025			return;
2026
2027		if (notify_die(DIE_SSTEP, "block_step", regs, 5,
2028			       5, SIGTRAP) == NOTIFY_STOP) {
2029			return;
2030		}
2031		if (debugger_sstep(regs))
2032			return;
2033	} else if (debug_status & DBSR_IC) { 	/* Instruction complete */
2034		regs_set_return_msr(regs, regs->msr & ~MSR_DE);
2035
2036		/* Disable instruction completion */
2037		mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~DBCR0_IC);
2038		/* Clear the instruction completion event */
2039		mtspr(SPRN_DBSR, DBSR_IC);
2040
2041		if (kprobe_post_handler(regs))
2042			return;
2043
2044		if (notify_die(DIE_SSTEP, "single_step", regs, 5,
2045			       5, SIGTRAP) == NOTIFY_STOP) {
2046			return;
2047		}
2048
2049		if (debugger_sstep(regs))
2050			return;
2051
2052		if (user_mode(regs)) {
2053			current->thread.debug.dbcr0 &= ~DBCR0_IC;
2054			if (DBCR_ACTIVE_EVENTS(current->thread.debug.dbcr0,
2055					       current->thread.debug.dbcr1))
2056				regs_set_return_msr(regs, regs->msr | MSR_DE);
2057			else
2058				/* Make sure the IDM bit is off */
2059				current->thread.debug.dbcr0 &= ~DBCR0_IDM;
2060		}
2061
2062		_exception(SIGTRAP, regs, TRAP_TRACE, regs->nip);
2063	} else
2064		handle_debug(regs, debug_status);
2065}
2066#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
2067
2068#ifdef CONFIG_ALTIVEC
2069DEFINE_INTERRUPT_HANDLER(altivec_assist_exception)
2070{
2071	int err;
2072
2073	if (!user_mode(regs)) {
2074		printk(KERN_EMERG "VMX/Altivec assist exception in kernel mode"
2075		       " at %lx\n", regs->nip);
2076		die("Kernel VMX/Altivec assist exception", regs, SIGILL);
2077	}
2078
2079	flush_altivec_to_thread(current);
2080
2081	PPC_WARN_EMULATED(altivec, regs);
2082	err = emulate_altivec(regs);
2083	if (err == 0) {
2084		regs_add_return_ip(regs, 4); /* skip emulated instruction */
2085		emulate_single_step(regs);
2086		return;
2087	}
2088
2089	if (err == -EFAULT) {
2090		/* got an error reading the instruction */
2091		_exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip);
2092	} else {
2093		/* didn't recognize the instruction */
2094		/* XXX quick hack for now: set the non-Java bit in the VSCR */
2095		printk_ratelimited(KERN_ERR "Unrecognized altivec instruction "
2096				   "in %s at %lx\n", current->comm, regs->nip);
2097		current->thread.vr_state.vscr.u[3] |= 0x10000;
2098	}
2099}
2100#endif /* CONFIG_ALTIVEC */
2101
2102#ifdef CONFIG_PPC_85xx
2103DEFINE_INTERRUPT_HANDLER(CacheLockingException)
2104{
2105	unsigned long error_code = regs->dsisr;
2106
2107	/* We treat cache locking instructions from the user
2108	 * as priv ops, in the future we could try to do
2109	 * something smarter
2110	 */
2111	if (error_code & (ESR_DLK|ESR_ILK))
2112		_exception(SIGILL, regs, ILL_PRVOPC, regs->nip);
2113	return;
2114}
2115#endif /* CONFIG_PPC_85xx */
2116
2117#ifdef CONFIG_SPE
2118DEFINE_INTERRUPT_HANDLER(SPEFloatingPointException)
2119{
 
2120	unsigned long spefscr;
2121	int fpexc_mode;
2122	int code = FPE_FLTUNK;
2123	int err;
2124
2125	interrupt_cond_local_irq_enable(regs);
2126
2127	flush_spe_to_thread(current);
2128
2129	spefscr = current->thread.spefscr;
2130	fpexc_mode = current->thread.fpexc_mode;
2131
2132	if ((spefscr & SPEFSCR_FOVF) && (fpexc_mode & PR_FP_EXC_OVF)) {
2133		code = FPE_FLTOVF;
2134	}
2135	else if ((spefscr & SPEFSCR_FUNF) && (fpexc_mode & PR_FP_EXC_UND)) {
2136		code = FPE_FLTUND;
2137	}
2138	else if ((spefscr & SPEFSCR_FDBZ) && (fpexc_mode & PR_FP_EXC_DIV))
2139		code = FPE_FLTDIV;
2140	else if ((spefscr & SPEFSCR_FINV) && (fpexc_mode & PR_FP_EXC_INV)) {
2141		code = FPE_FLTINV;
2142	}
2143	else if ((spefscr & (SPEFSCR_FG | SPEFSCR_FX)) && (fpexc_mode & PR_FP_EXC_RES))
2144		code = FPE_FLTRES;
2145
2146	err = do_spe_mathemu(regs);
2147	if (err == 0) {
2148		regs_add_return_ip(regs, 4); /* skip emulated instruction */
2149		emulate_single_step(regs);
2150		return;
2151	}
2152
2153	if (err == -EFAULT) {
2154		/* got an error reading the instruction */
2155		_exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip);
2156	} else if (err == -EINVAL) {
2157		/* didn't recognize the instruction */
2158		printk(KERN_ERR "unrecognized spe instruction "
2159		       "in %s at %lx\n", current->comm, regs->nip);
2160	} else {
2161		_exception(SIGFPE, regs, code, regs->nip);
2162	}
2163
2164	return;
2165}
2166
2167DEFINE_INTERRUPT_HANDLER(SPEFloatingPointRoundException)
2168{
 
2169	int err;
2170
2171	interrupt_cond_local_irq_enable(regs);
2172
2173	preempt_disable();
2174	if (regs->msr & MSR_SPE)
2175		giveup_spe(current);
2176	preempt_enable();
2177
2178	regs_add_return_ip(regs, -4);
2179	err = speround_handler(regs);
2180	if (err == 0) {
2181		regs_add_return_ip(regs, 4); /* skip emulated instruction */
2182		emulate_single_step(regs);
2183		return;
2184	}
2185
2186	if (err == -EFAULT) {
2187		/* got an error reading the instruction */
2188		_exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip);
2189	} else if (err == -EINVAL) {
2190		/* didn't recognize the instruction */
2191		printk(KERN_ERR "unrecognized spe instruction "
2192		       "in %s at %lx\n", current->comm, regs->nip);
2193	} else {
2194		_exception(SIGFPE, regs, FPE_FLTUNK, regs->nip);
2195		return;
2196	}
2197}
2198#endif
2199
2200/*
2201 * We enter here if we get an unrecoverable exception, that is, one
2202 * that happened at a point where the RI (recoverable interrupt) bit
2203 * in the MSR is 0.  This indicates that SRR0/1 are live, and that
2204 * we therefore lost state by taking this exception.
2205 */
2206void __noreturn unrecoverable_exception(struct pt_regs *regs)
2207{
2208	pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n",
2209		 regs->trap, regs->nip, regs->msr);
2210	die("Unrecoverable exception", regs, SIGABRT);
2211	/* die() should not return */
2212	for (;;)
2213		;
2214}
2215
2216#if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x)
2217/*
2218 * Default handler for a Watchdog exception,
2219 * spins until a reboot occurs
2220 */
2221void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs)
2222{
2223	/* Generic WatchdogHandler, implement your own */
2224	mtspr(SPRN_TCR, mfspr(SPRN_TCR)&(~TCR_WIE));
2225	return;
2226}
2227
2228DEFINE_INTERRUPT_HANDLER_NMI(WatchdogException)
2229{
2230	printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n");
2231	WatchdogHandler(regs);
2232	return 0;
2233}
2234#endif
2235
2236/*
2237 * We enter here if we discover during exception entry that we are
2238 * running in supervisor mode with a userspace value in the stack pointer.
2239 */
2240DEFINE_INTERRUPT_HANDLER(kernel_bad_stack)
2241{
2242	printk(KERN_EMERG "Bad kernel stack pointer %lx at %lx\n",
2243	       regs->gpr[1], regs->nip);
2244	die("Bad kernel stack pointer", regs, SIGABRT);
2245}
2246
 
 
 
 
 
2247#ifdef CONFIG_PPC_EMULATED_STATS
2248
2249#define WARN_EMULATED_SETUP(type)	.type = { .name = #type }
2250
2251struct ppc_emulated ppc_emulated = {
2252#ifdef CONFIG_ALTIVEC
2253	WARN_EMULATED_SETUP(altivec),
2254#endif
2255	WARN_EMULATED_SETUP(dcba),
2256	WARN_EMULATED_SETUP(dcbz),
2257	WARN_EMULATED_SETUP(fp_pair),
2258	WARN_EMULATED_SETUP(isel),
2259	WARN_EMULATED_SETUP(mcrxr),
2260	WARN_EMULATED_SETUP(mfpvr),
2261	WARN_EMULATED_SETUP(multiple),
2262	WARN_EMULATED_SETUP(popcntb),
2263	WARN_EMULATED_SETUP(spe),
2264	WARN_EMULATED_SETUP(string),
2265	WARN_EMULATED_SETUP(sync),
2266	WARN_EMULATED_SETUP(unaligned),
2267#ifdef CONFIG_MATH_EMULATION
2268	WARN_EMULATED_SETUP(math),
2269#endif
2270#ifdef CONFIG_VSX
2271	WARN_EMULATED_SETUP(vsx),
2272#endif
2273#ifdef CONFIG_PPC64
2274	WARN_EMULATED_SETUP(mfdscr),
2275	WARN_EMULATED_SETUP(mtdscr),
2276	WARN_EMULATED_SETUP(lq_stq),
2277	WARN_EMULATED_SETUP(lxvw4x),
2278	WARN_EMULATED_SETUP(lxvh8x),
2279	WARN_EMULATED_SETUP(lxvd2x),
2280	WARN_EMULATED_SETUP(lxvb16x),
2281#endif
2282};
2283
2284u32 ppc_warn_emulated;
2285
2286void ppc_warn_emulated_print(const char *type)
2287{
2288	pr_warn_ratelimited("%s used emulated %s instruction\n", current->comm,
2289			    type);
2290}
2291
2292static int __init ppc_warn_emulated_init(void)
2293{
2294	struct dentry *dir;
2295	unsigned int i;
2296	struct ppc_emulated_entry *entries = (void *)&ppc_emulated;
2297
2298	dir = debugfs_create_dir("emulated_instructions",
2299				 arch_debugfs_dir);
2300
2301	debugfs_create_u32("do_warn", 0644, dir, &ppc_warn_emulated);
2302
2303	for (i = 0; i < sizeof(ppc_emulated)/sizeof(*entries); i++)
2304		debugfs_create_u32(entries[i].name, 0644, dir,
2305				   (u32 *)&entries[i].val.counter);
2306
2307	return 0;
2308}
2309
2310device_initcall(ppc_warn_emulated_init);
2311
2312#endif /* CONFIG_PPC_EMULATED_STATS */