1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * This file contains the 64-bit "server" PowerPC variant
   4 * of the low level exception handling including exception
   5 * vectors, exception return, part of the slb and stab
   6 * handling and other fixed offset specific things.
   7 *
   8 * This file is meant to be #included from head_64.S due to
   9 * position dependent assembly.
  10 *
  11 * Most of this originates from head_64.S and thus has the same
  12 * copyright history.
  13 *
  14 */
  15
  16#include <asm/hw_irq.h>
  17#include <asm/exception-64s.h>
  18#include <asm/ptrace.h>
  19#include <asm/cpuidle.h>
  20#include <asm/head-64.h>
  21#include <asm/feature-fixups.h>
  22#include <asm/kup.h>
  23
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  24/*
  25 * Following are fixed section helper macros.
  26 *
  27 * EXC_REAL_BEGIN/END  - real, unrelocated exception vectors
  28 * EXC_VIRT_BEGIN/END  - virt (AIL), unrelocated exception vectors
  29 * TRAMP_REAL_BEGIN    - real, unrelocated helpers (virt may call these)
  30 * TRAMP_VIRT_BEGIN    - virt, unreloc helpers (in practice, real can use)
 
  31 * EXC_COMMON          - After switching to virtual, relocated mode.
  32 */
  33
  34#define EXC_REAL_BEGIN(name, start, size)			\
  35	FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
  36
  37#define EXC_REAL_END(name, start, size)				\
  38	FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
  39
  40#define EXC_VIRT_BEGIN(name, start, size)			\
  41	FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
  42
  43#define EXC_VIRT_END(name, start, size)				\
  44	FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
  45
  46#define EXC_COMMON_BEGIN(name)					\
  47	USE_TEXT_SECTION();					\
  48	.balign IFETCH_ALIGN_BYTES;				\
  49	.global name;						\
  50	_ASM_NOKPROBE_SYMBOL(name);				\
  51	DEFINE_FIXED_SYMBOL(name);				\
  52name:
  53
  54#define TRAMP_REAL_BEGIN(name)					\
  55	FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name)
  56
  57#define TRAMP_VIRT_BEGIN(name)					\
  58	FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
  59
 
 
 
 
 
 
 
  60#define EXC_REAL_NONE(start, size)				\
  61	FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \
  62	FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size)
  63
  64#define EXC_VIRT_NONE(start, size)				\
  65	FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \
  66	FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size)
  67
  68/*
  69 * We're short on space and time in the exception prolog, so we can't
  70 * use the normal LOAD_REG_IMMEDIATE macro to load the address of label.
  71 * Instead we get the base of the kernel from paca->kernelbase and or in the low
  72 * part of label. This requires that the label be within 64KB of kernelbase, and
  73 * that kernelbase be 64K aligned.
  74 */
  75#define LOAD_HANDLER(reg, label)					\
  76	ld	reg,PACAKBASE(r13);	/* get high part of &label */	\
  77	ori	reg,reg,FIXED_SYMBOL_ABS_ADDR(label)
  78
  79#define __LOAD_HANDLER(reg, label)					\
  80	ld	reg,PACAKBASE(r13);					\
  81	ori	reg,reg,(ABS_ADDR(label))@l
  82
  83/*
  84 * Branches from unrelocated code (e.g., interrupts) to labels outside
  85 * head-y require >64K offsets.
  86 */
  87#define __LOAD_FAR_HANDLER(reg, label)					\
  88	ld	reg,PACAKBASE(r13);					\
  89	ori	reg,reg,(ABS_ADDR(label))@l;				\
  90	addis	reg,reg,(ABS_ADDR(label))@h
  91
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  92/*
  93 * Branch to label using its 0xC000 address. This results in instruction
  94 * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
  95 * on using mtmsr rather than rfid.
  96 *
  97 * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
  98 * load KBASE for a slight optimisation.
  99 */
 100#define BRANCH_TO_C000(reg, label)					\
 101	__LOAD_FAR_HANDLER(reg, label);					\
 102	mtctr	reg;							\
 103	bctr
 104
 
 
 
 
 
 
 
 105/*
 106 * Interrupt code generation macros
 
 
 107 */
 108#define IVEC		.L_IVEC_\name\()	/* Interrupt vector address */
 109#define IHSRR		.L_IHSRR_\name\()	/* Sets SRR or HSRR registers */
 110#define IHSRR_IF_HVMODE	.L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */
 111#define IAREA		.L_IAREA_\name\()	/* PACA save area */
 112#define IVIRT		.L_IVIRT_\name\()	/* Has virt mode entry point */
 113#define IISIDE		.L_IISIDE_\name\()	/* Uses SRR0/1 not DAR/DSISR */
 114#define IDAR		.L_IDAR_\name\()	/* Uses DAR (or SRR0) */
 115#define IDSISR		.L_IDSISR_\name\()	/* Uses DSISR (or SRR1) */
 116#define ISET_RI		.L_ISET_RI_\name\()	/* Run common code w/ MSR[RI]=1 */
 117#define IBRANCH_TO_COMMON	.L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */
 118#define IREALMODE_COMMON	.L_IREALMODE_COMMON_\name\() /* Common runs in realmode */
 119#define IMASK		.L_IMASK_\name\()	/* IRQ soft-mask bit */
 120#define IKVM_REAL	.L_IKVM_REAL_\name\()	/* Real entry tests KVM */
 121#define __IKVM_REAL(name)	.L_IKVM_REAL_ ## name
 122#define IKVM_VIRT	.L_IKVM_VIRT_\name\()	/* Virt entry tests KVM */
 123#define ISTACK		.L_ISTACK_\name\()	/* Set regular kernel stack */
 124#define __ISTACK(name)	.L_ISTACK_ ## name
 125#define IKUAP		.L_IKUAP_\name\()	/* Do KUAP lock */
 126
 127#define INT_DEFINE_BEGIN(n)						\
 128.macro int_define_ ## n name
 129
 130#define INT_DEFINE_END(n)						\
 131.endm ;									\
 132int_define_ ## n n ;							\
 133do_define_int n
 134
 135.macro do_define_int name
 136	.ifndef IVEC
 137		.error "IVEC not defined"
 138	.endif
 139	.ifndef IHSRR
 140		IHSRR=0
 141	.endif
 142	.ifndef IHSRR_IF_HVMODE
 143		IHSRR_IF_HVMODE=0
 144	.endif
 145	.ifndef IAREA
 146		IAREA=PACA_EXGEN
 147	.endif
 148	.ifndef IVIRT
 149		IVIRT=1
 150	.endif
 151	.ifndef IISIDE
 152		IISIDE=0
 153	.endif
 154	.ifndef IDAR
 155		IDAR=0
 156	.endif
 157	.ifndef IDSISR
 158		IDSISR=0
 159	.endif
 160	.ifndef ISET_RI
 161		ISET_RI=1
 162	.endif
 163	.ifndef IBRANCH_TO_COMMON
 164		IBRANCH_TO_COMMON=1
 165	.endif
 166	.ifndef IREALMODE_COMMON
 167		IREALMODE_COMMON=0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 168	.else
 169		.if ! IBRANCH_TO_COMMON
 170			.error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0"
 171		.endif
 172	.endif
 173	.ifndef IMASK
 174		IMASK=0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 175	.endif
 176	.ifndef IKVM_REAL
 177		IKVM_REAL=0
 178	.endif
 179	.ifndef IKVM_VIRT
 180		IKVM_VIRT=0
 
 
 
 
 
 
 
 
 
 
 
 181	.endif
 182	.ifndef ISTACK
 183		ISTACK=1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 184	.endif
 185	.ifndef IKUAP
 186		IKUAP=1
 
 
 
 
 
 
 
 
 
 
 
 
 
 187	.endif
 
 188.endm
 189
 190/*
 191 * All interrupts which set HSRR registers, as well as SRESET and MCE and
 192 * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken,
 193 * so they all generally need to test whether they were taken in guest context.
 194 *
 195 * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be
 196 * taken with MSR[HV]=0.
 197 *
 198 * Interrupts which set SRR registers (with the above exceptions) do not
 199 * elevate to MSR[HV]=1 mode, though most can be taken when running with
 200 * MSR[HV]=1  (e.g., bare metal kernel and userspace). So these interrupts do
 201 * not need to test whether a guest is running because they get delivered to
 202 * the guest directly, including nested HV KVM guests.
 203 *
 204 * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host
 205 * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the
 206 * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be
 207 * delivered to the real-mode entry point, therefore such interrupts only test
 208 * KVM in their real mode handlers, and only when PR KVM is possible.
 209 *
 210 * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always
 211 * delivered in real-mode when the MMU is in hash mode because the MMU
 212 * registers are not set appropriately to translate host addresses. In nested
 213 * radix mode these can be delivered in virt-mode as the host translations are
 214 * used implicitly (see: effective LPID, effective PID).
 215 */
 216
 217/*
 218 * If an interrupt is taken while a guest is running, it is immediately routed
 219 * to KVM to handle.
 220 */
 221
 222.macro KVMTEST name handler
 223#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 224	lbz	r10,HSTATE_IN_GUEST(r13)
 225	cmpwi	r10,0
 226	/* HSRR variants have the 0x2 bit added to their trap number */
 227	.if IHSRR_IF_HVMODE
 228	BEGIN_FTR_SECTION
 229	li	r10,(IVEC + 0x2)
 230	FTR_SECTION_ELSE
 231	li	r10,(IVEC)
 232	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 233	.elseif IHSRR
 234	li	r10,(IVEC + 0x2)
 235	.else
 236	li	r10,(IVEC)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 237	.endif
 238	bne	\handler
 
 
 239#endif
 240.endm
 241
 242/*
 243 * This is the BOOK3S interrupt entry code macro.
 244 *
 245 * This can result in one of several things happening:
 246 * - Branch to the _common handler, relocated, in virtual mode.
 247 *   These are normal interrupts (synchronous and asynchronous) handled by
 248 *   the kernel.
 249 * - Branch to KVM, relocated but real mode interrupts remain in real mode.
 250 *   These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by
 251 *   / intended for host or guest kernel, but KVM must always be involved
 252 *   because the machine state is set for guest execution.
 253 * - Branch to the masked handler, unrelocated.
 254 *   These occur when maskable asynchronous interrupts are taken with the
 255 *   irq_soft_mask set.
 256 * - Branch to an "early" handler in real mode but relocated.
 257 *   This is done if early=1. MCE and HMI use these to handle errors in real
 258 *   mode.
 259 * - Fall through and continue executing in real, unrelocated mode.
 260 *   This is done if early=2.
 261 */
 262
 263.macro GEN_BRANCH_TO_COMMON name, virt
 264	.if IREALMODE_COMMON
 265	LOAD_HANDLER(r10, \name\()_common)
 266	mtctr	r10
 267	bctr
 268	.else
 269	.if \virt
 270#ifndef CONFIG_RELOCATABLE
 271	b	\name\()_common_virt
 272#else
 273	LOAD_HANDLER(r10, \name\()_common_virt)
 274	mtctr	r10
 275	bctr
 276#endif
 277	.else
 278	LOAD_HANDLER(r10, \name\()_common_real)
 279	mtctr	r10
 280	bctr
 281	.endif
 282	.endif
 283.endm
 284
 285.macro GEN_INT_ENTRY name, virt, ool=0
 286	SET_SCRATCH0(r13)			/* save r13 */
 287	GET_PACA(r13)
 288	std	r9,IAREA+EX_R9(r13)		/* save r9 */
 289BEGIN_FTR_SECTION
 290	mfspr	r9,SPRN_PPR
 291END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 292	HMT_MEDIUM
 293	std	r10,IAREA+EX_R10(r13)		/* save r10 - r12 */
 294BEGIN_FTR_SECTION
 295	mfspr	r10,SPRN_CFAR
 296END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 297	.if \ool
 298	.if !\virt
 299	b	tramp_real_\name
 300	.pushsection .text
 301	TRAMP_REAL_BEGIN(tramp_real_\name)
 302	.else
 303	b	tramp_virt_\name
 304	.pushsection .text
 305	TRAMP_VIRT_BEGIN(tramp_virt_\name)
 306	.endif
 307	.endif
 308
 309BEGIN_FTR_SECTION
 310	std	r9,IAREA+EX_PPR(r13)
 311END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 312BEGIN_FTR_SECTION
 313	std	r10,IAREA+EX_CFAR(r13)
 314END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 315	INTERRUPT_TO_KERNEL
 316	mfctr	r10
 317	std	r10,IAREA+EX_CTR(r13)
 318	mfcr	r9
 319	std	r11,IAREA+EX_R11(r13)
 320	std	r12,IAREA+EX_R12(r13)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 321
 322	/*
 323	 * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],
 324	 * because a d-side MCE will clobber those registers so is
 325	 * not recoverable if they are live.
 326	 */
 327	GET_SCRATCH0(r10)
 328	std	r10,IAREA+EX_R13(r13)
 329	.if IDAR && !IISIDE
 330	.if IHSRR
 331	mfspr	r10,SPRN_HDAR
 332	.else
 333	mfspr	r10,SPRN_DAR
 334	.endif
 335	std	r10,IAREA+EX_DAR(r13)
 336	.endif
 337	.if IDSISR && !IISIDE
 338	.if IHSRR
 339	mfspr	r10,SPRN_HDSISR
 340	.else
 341	mfspr	r10,SPRN_DSISR
 342	.endif
 343	stw	r10,IAREA+EX_DSISR(r13)
 344	.endif
 345
 346	.if IHSRR_IF_HVMODE
 347	BEGIN_FTR_SECTION
 348	mfspr	r11,SPRN_HSRR0		/* save HSRR0 */
 349	mfspr	r12,SPRN_HSRR1		/* and HSRR1 */
 350	FTR_SECTION_ELSE
 351	mfspr	r11,SPRN_SRR0		/* save SRR0 */
 352	mfspr	r12,SPRN_SRR1		/* and SRR1 */
 353	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 354	.elseif IHSRR
 355	mfspr	r11,SPRN_HSRR0		/* save HSRR0 */
 356	mfspr	r12,SPRN_HSRR1		/* and HSRR1 */
 357	.else
 358	mfspr	r11,SPRN_SRR0		/* save SRR0 */
 359	mfspr	r12,SPRN_SRR1		/* and SRR1 */
 360	.endif
 361
 362	.if IBRANCH_TO_COMMON
 363	GEN_BRANCH_TO_COMMON \name \virt
 364	.endif
 365
 366	.if \ool
 367	.popsection
 368	.endif
 369.endm
 370
 371/*
 372 * __GEN_COMMON_ENTRY is required to receive the branch from interrupt
 373 * entry, except in the case of the real-mode handlers which require
 374 * __GEN_REALMODE_COMMON_ENTRY.
 375 *
 376 * This switches to virtual mode and sets MSR[RI].
 377 */
 378.macro __GEN_COMMON_ENTRY name
 379DEFINE_FIXED_SYMBOL(\name\()_common_real)
 380\name\()_common_real:
 381	.if IKVM_REAL
 382		KVMTEST \name kvm_interrupt
 383	.endif
 384
 385	ld	r10,PACAKMSR(r13)	/* get MSR value for kernel */
 386	/* MSR[RI] is clear iff using SRR regs */
 387	.if IHSRR_IF_HVMODE
 388	BEGIN_FTR_SECTION
 389	xori	r10,r10,MSR_RI
 390	END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
 391	.elseif ! IHSRR
 392	xori	r10,r10,MSR_RI
 393	.endif
 394	mtmsrd	r10
 395
 396	.if IVIRT
 397	.if IKVM_VIRT
 398	b	1f /* skip the virt test coming from real */
 399	.endif
 400
 401	.balign IFETCH_ALIGN_BYTES
 402DEFINE_FIXED_SYMBOL(\name\()_common_virt)
 403\name\()_common_virt:
 404	.if IKVM_VIRT
 405		KVMTEST \name kvm_interrupt
 4061:
 407	.endif
 408	.endif /* IVIRT */
 409.endm
 410
 411/*
 412 * Don't switch to virt mode. Used for early MCE and HMI handlers that
 413 * want to run in real mode.
 414 */
 415.macro __GEN_REALMODE_COMMON_ENTRY name
 416DEFINE_FIXED_SYMBOL(\name\()_common_real)
 417\name\()_common_real:
 418	.if IKVM_REAL
 419		KVMTEST \name kvm_interrupt
 420	.endif
 421.endm
 422
 423.macro __GEN_COMMON_BODY name
 424	.if IMASK
 425		.if ! ISTACK
 426		.error "No support for masked interrupt to use custom stack"
 427		.endif
 428
 429		/* If coming from user, skip soft-mask tests. */
 430		andi.	r10,r12,MSR_PR
 431		bne	3f
 432
 433		/*
 434		 * Kernel code running below __end_soft_masked may be
 435		 * implicitly soft-masked if it is within the regions
 436		 * in the soft mask table.
 437		 */
 438		LOAD_HANDLER(r10, __end_soft_masked)
 439		cmpld	r11,r10
 440		bge+	1f
 441
 442		/* SEARCH_SOFT_MASK_TABLE clobbers r9,r10,r12 */
 443		mtctr	r12
 444		stw	r9,PACA_EXGEN+EX_CCR(r13)
 445		SEARCH_SOFT_MASK_TABLE
 446		cmpdi	r12,0
 447		mfctr	r12		/* Restore r12 to SRR1 */
 448		lwz	r9,PACA_EXGEN+EX_CCR(r13)
 449		beq	1f		/* Not in soft-mask table */
 450		li	r10,IMASK
 451		b	2f		/* In soft-mask table, always mask */
 452
 453		/* Test the soft mask state against our interrupt's bit */
 4541:		lbz	r10,PACAIRQSOFTMASK(r13)
 4552:		andi.	r10,r10,IMASK
 456		/* Associate vector numbers with bits in paca->irq_happened */
 457		.if IVEC == 0x500 || IVEC == 0xea0
 458		li	r10,PACA_IRQ_EE
 459		.elseif IVEC == 0x900
 460		li	r10,PACA_IRQ_DEC
 461		.elseif IVEC == 0xa00 || IVEC == 0xe80
 462		li	r10,PACA_IRQ_DBELL
 463		.elseif IVEC == 0xe60
 464		li	r10,PACA_IRQ_HMI
 465		.elseif IVEC == 0xf00
 466		li	r10,PACA_IRQ_PMI
 467		.else
 468		.abort "Bad maskable vector"
 469		.endif
 470
 471		.if IHSRR_IF_HVMODE
 472		BEGIN_FTR_SECTION
 473		bne	masked_Hinterrupt
 474		FTR_SECTION_ELSE
 475		bne	masked_interrupt
 476		ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 477		.elseif IHSRR
 478		bne	masked_Hinterrupt
 479		.else
 480		bne	masked_interrupt
 481		.endif
 482	.endif
 483
 484	.if ISTACK
 485	andi.	r10,r12,MSR_PR		/* See if coming from user	*/
 4863:	mr	r10,r1			/* Save r1			*/
 487	subi	r1,r1,INT_FRAME_SIZE	/* alloc frame on kernel stack	*/
 488	beq-	100f
 489	ld	r1,PACAKSAVE(r13)	/* kernel stack to use		*/
 490100:	tdgei	r1,-INT_FRAME_SIZE	/* trap if r1 is in userspace	*/
 491	EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0
 492	.endif
 493
 494	std	r9,_CCR(r1)		/* save CR in stackframe	*/
 495	std	r11,_NIP(r1)		/* save SRR0 in stackframe	*/
 496	std	r12,_MSR(r1)		/* save SRR1 in stackframe	*/
 497	std	r10,0(r1)		/* make stack chain pointer	*/
 498	std	r0,GPR0(r1)		/* save r0 in stackframe	*/
 499	std	r10,GPR1(r1)		/* save r1 in stackframe	*/
 500
 501	/* Mark our [H]SRRs valid for return */
 502	li	r10,1
 503	.if IHSRR_IF_HVMODE
 504	BEGIN_FTR_SECTION
 505	stb	r10,PACAHSRR_VALID(r13)
 506	FTR_SECTION_ELSE
 507	stb	r10,PACASRR_VALID(r13)
 508	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 509	.elseif IHSRR
 510	stb	r10,PACAHSRR_VALID(r13)
 511	.else
 512	stb	r10,PACASRR_VALID(r13)
 513	.endif
 514
 515	.if ISET_RI
 516	li	r10,MSR_RI
 517	mtmsrd	r10,1			/* Set MSR_RI */
 518	.endif
 519
 520	.if ISTACK
 521	.if IKUAP
 522	kuap_save_amr_and_lock r9, r10, cr1, cr0
 523	.endif
 524	beq	101f			/* if from kernel mode		*/
 525BEGIN_FTR_SECTION
 526	ld	r9,IAREA+EX_PPR(r13)	/* Read PPR from paca		*/
 527	std	r9,_PPR(r1)
 528END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 529101:
 530	.else
 531	.if IKUAP
 532	kuap_save_amr_and_lock r9, r10, cr1
 533	.endif
 534	.endif
 535
 536	/* Save original regs values from save area to stack frame. */
 537	ld	r9,IAREA+EX_R9(r13)	/* move r9, r10 to stackframe	*/
 538	ld	r10,IAREA+EX_R10(r13)
 539	std	r9,GPR9(r1)
 540	std	r10,GPR10(r1)
 541	ld	r9,IAREA+EX_R11(r13)	/* move r11 - r13 to stackframe	*/
 542	ld	r10,IAREA+EX_R12(r13)
 543	ld	r11,IAREA+EX_R13(r13)
 544	std	r9,GPR11(r1)
 545	std	r10,GPR12(r1)
 546	std	r11,GPR13(r1)
 547
 548	SAVE_NVGPRS(r1)
 549
 550	.if IDAR
 551	.if IISIDE
 552	ld	r10,_NIP(r1)
 553	.else
 554	ld	r10,IAREA+EX_DAR(r13)
 555	.endif
 556	std	r10,_DAR(r1)
 557	.endif
 558
 559	.if IDSISR
 560	.if IISIDE
 561	ld	r10,_MSR(r1)
 562	lis	r11,DSISR_SRR1_MATCH_64S@h
 563	and	r10,r10,r11
 564	.else
 565	lwz	r10,IAREA+EX_DSISR(r13)
 566	.endif
 567	std	r10,_DSISR(r1)
 568	.endif
 569
 570BEGIN_FTR_SECTION
 571	ld	r10,IAREA+EX_CFAR(r13)
 572	std	r10,ORIG_GPR3(r1)
 573END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 574	ld	r10,IAREA+EX_CTR(r13)
 575	std	r10,_CTR(r1)
 576	std	r2,GPR2(r1)		/* save r2 in stackframe	*/
 577	SAVE_4GPRS(3, r1)		/* save r3 - r6 in stackframe   */
 578	SAVE_2GPRS(7, r1)		/* save r7, r8 in stackframe	*/
 579	mflr	r9			/* Get LR, later save to stack	*/
 580	ld	r2,PACATOC(r13)		/* get kernel TOC into r2	*/
 581	std	r9,_LINK(r1)
 582	lbz	r10,PACAIRQSOFTMASK(r13)
 583	mfspr	r11,SPRN_XER		/* save XER in stackframe	*/
 584	std	r10,SOFTE(r1)
 585	std	r11,_XER(r1)
 586	li	r9,IVEC
 587	std	r9,_TRAP(r1)		/* set trap number		*/
 588	li	r10,0
 589	ld	r11,exception_marker@toc(r2)
 590	std	r10,RESULT(r1)		/* clear regs->result		*/
 591	std	r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame	*/
 592.endm
 593
 594/*
 595 * On entry r13 points to the paca, r9-r13 are saved in the paca,
 596 * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
 597 * SRR1, and relocation is on.
 598 *
 599 * If stack=0, then the stack is already set in r1, and r1 is saved in r10.
 600 * PPR save and CPU accounting is not done for the !stack case (XXX why not?)
 601 */
 602.macro GEN_COMMON name
 603	__GEN_COMMON_ENTRY \name
 604	__GEN_COMMON_BODY \name
 605.endm
 606
 607.macro SEARCH_RESTART_TABLE
 608#ifdef CONFIG_RELOCATABLE
 609	mr	r12,r2
 610	ld	r2,PACATOC(r13)
 611	LOAD_REG_ADDR(r9, __start___restart_table)
 612	LOAD_REG_ADDR(r10, __stop___restart_table)
 613	mr	r2,r12
 614#else
 615	LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___restart_table)
 616	LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___restart_table)
 617#endif
 618300:
 619	cmpd	r9,r10
 620	beq	302f
 621	ld	r12,0(r9)
 622	cmpld	r11,r12
 623	blt	301f
 624	ld	r12,8(r9)
 625	cmpld	r11,r12
 626	bge	301f
 627	ld	r12,16(r9)
 628	b	303f
 629301:
 630	addi	r9,r9,24
 631	b	300b
 632302:
 633	li	r12,0
 634303:
 635.endm
 636
 637.macro SEARCH_SOFT_MASK_TABLE
 638#ifdef CONFIG_RELOCATABLE
 639	mr	r12,r2
 640	ld	r2,PACATOC(r13)
 641	LOAD_REG_ADDR(r9, __start___soft_mask_table)
 642	LOAD_REG_ADDR(r10, __stop___soft_mask_table)
 643	mr	r2,r12
 644#else
 645	LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___soft_mask_table)
 646	LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___soft_mask_table)
 647#endif
 648300:
 649	cmpd	r9,r10
 650	beq	302f
 651	ld	r12,0(r9)
 652	cmpld	r11,r12
 653	blt	301f
 654	ld	r12,8(r9)
 655	cmpld	r11,r12
 656	bge	301f
 657	li	r12,1
 658	b	303f
 659301:
 660	addi	r9,r9,16
 661	b	300b
 662302:
 663	li	r12,0
 664303:
 665.endm
 666
 667/*
 668 * Restore all registers including H/SRR0/1 saved in a stack frame of a
 669 * standard exception.
 670 */
 671.macro EXCEPTION_RESTORE_REGS hsrr=0
 672	/* Move original SRR0 and SRR1 into the respective regs */
 673	ld	r9,_MSR(r1)
 674	li	r10,0
 
 
 675	.if \hsrr
 676	mtspr	SPRN_HSRR1,r9
 677	stb	r10,PACAHSRR_VALID(r13)
 678	.else
 679	mtspr	SPRN_SRR1,r9
 680	stb	r10,PACASRR_VALID(r13)
 681	.endif
 682	ld	r9,_NIP(r1)
 683	.if \hsrr
 684	mtspr	SPRN_HSRR0,r9
 685	.else
 686	mtspr	SPRN_SRR0,r9
 687	.endif
 688	ld	r9,_CTR(r1)
 689	mtctr	r9
 690	ld	r9,_XER(r1)
 691	mtxer	r9
 692	ld	r9,_LINK(r1)
 693	mtlr	r9
 694	ld	r9,_CCR(r1)
 695	mtcr	r9
 696	REST_8GPRS(2, r1)
 697	REST_4GPRS(10, r1)
 698	REST_GPR(0, r1)
 699	/* restore original r1. */
 700	ld	r1,GPR1(r1)
 701.endm
 702
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 703/*
 704 * There are a few constraints to be concerned with.
 705 * - Real mode exceptions code/data must be located at their physical location.
 706 * - Virtual mode exceptions must be mapped at their 0xc000... location.
 707 * - Fixed location code must not call directly beyond the __end_interrupts
 708 *   area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence
 709 *   must be used.
 710 * - LOAD_HANDLER targets must be within first 64K of physical 0 /
 711 *   virtual 0xc00...
 712 * - Conditional branch targets must be within +/-32K of caller.
 713 *
 714 * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and
 715 * therefore don't have to run in physically located code or rfid to
 716 * virtual mode kernel code. However on relocatable kernels they do have
 717 * to branch to KERNELBASE offset because the rest of the kernel (outside
 718 * the exception vectors) may be located elsewhere.
 719 *
 720 * Virtual exceptions correspond with physical, except their entry points
 721 * are offset by 0xc000000000000000 and also tend to get an added 0x4000
 722 * offset applied. Virtual exceptions are enabled with the Alternate
 723 * Interrupt Location (AIL) bit set in the LPCR. However this does not
 724 * guarantee they will be delivered virtually. Some conditions (see the ISA)
 725 * cause exceptions to be delivered in real mode.
 726 *
 727 * The scv instructions are a special case. They get a 0x3000 offset applied.
 728 * scv exceptions have unique reentrancy properties, see below.
 729 *
 730 * It's impossible to receive interrupts below 0x300 via AIL.
 731 *
 732 * KVM: None of the virtual exceptions are from the guest. Anything that
 733 * escalated to HV=1 from HV=0 is delivered via real mode handlers.
 734 *
 735 *
 736 * We layout physical memory as follows:
 737 * 0x0000 - 0x00ff : Secondary processor spin code
 738 * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
 739 * 0x1900 - 0x2fff : Real mode trampolines
 740 * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
 741 * 0x5900 - 0x6fff : Relon mode trampolines
 742 * 0x7000 - 0x7fff : FWNMI data area
 743 * 0x8000 -   .... : Common interrupt handlers, remaining early
 744 *                   setup code, rest of kernel.
 745 *
 746 * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
 747 * is necessary. Until then it's more consistent to explicitly put VIRT_NONE
 748 * vectors there.
 749 */
 750OPEN_FIXED_SECTION(real_vectors,        0x0100, 0x1900)
 751OPEN_FIXED_SECTION(real_trampolines,    0x1900, 0x3000)
 752OPEN_FIXED_SECTION(virt_vectors,        0x3000, 0x5900)
 753OPEN_FIXED_SECTION(virt_trampolines,    0x5900, 0x7000)
 754
 755#ifdef CONFIG_PPC_POWERNV
 756	.globl start_real_trampolines
 757	.globl end_real_trampolines
 758	.globl start_virt_trampolines
 759	.globl end_virt_trampolines
 760#endif
 761
 762#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
 763/*
 764 * Data area reserved for FWNMI option.
 765 * This address (0x7000) is fixed by the RPA.
 766 * pseries and powernv need to keep the whole page from
 767 * 0x7000 to 0x8000 free for use by the firmware
 768 */
 769ZERO_FIXED_SECTION(fwnmi_page,          0x7000, 0x8000)
 770OPEN_TEXT_SECTION(0x8000)
 771#else
 772OPEN_TEXT_SECTION(0x7000)
 773#endif
 774
 775USE_FIXED_SECTION(real_vectors)
 776
 777/*
 778 * This is the start of the interrupt handlers for pSeries
 779 * This code runs with relocation off.
 780 * Code from here to __end_interrupts gets copied down to real
 781 * address 0x100 when we are running a relocatable kernel.
 782 * Therefore any relative branches in this section must only
 783 * branch to labels in this section.
 784 */
 785	.globl __start_interrupts
 786__start_interrupts:
 787
 788/**
 789 * Interrupt 0x3000 - System Call Vectored Interrupt (syscall).
 790 * This is a synchronous interrupt invoked with the "scv" instruction. The
 791 * system call does not alter the HV bit, so it is directed to the OS.
 792 *
 793 * Handling:
 794 * scv instructions enter the kernel without changing EE, RI, ME, or HV.
 795 * In particular, this means we can take a maskable interrupt at any point
 796 * in the scv handler, which is unlike any other interrupt. This is solved
 797 * by treating the instruction addresses in the handler as being soft-masked,
 798 * by adding a SOFT_MASK_TABLE entry for them.
 799 *
 800 * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and
 801 * ensure scv is never executed with relocation off, which means AIL-0
 802 * should never happen.
 803 *
 804 * Before leaving the following inside-__end_soft_masked text, at least of the
 805 * following must be true:
 806 * - MSR[PR]=1 (i.e., return to userspace)
 807 * - MSR_EE|MSR_RI is clear (no reentrant exceptions)
 808 * - Standard kernel environment is set up (stack, paca, etc)
 809 *
 810 * Call convention:
 811 *
 812 * syscall register convention is in Documentation/powerpc/syscall64-abi.rst
 813 */
 814EXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000)
 815	/* SCV 0 */
 816	mr	r9,r13
 817	GET_PACA(r13)
 818	mflr	r11
 819	mfctr	r12
 820	li	r10,IRQS_ALL_DISABLED
 821	stb	r10,PACAIRQSOFTMASK(r13)
 822#ifdef CONFIG_RELOCATABLE
 823	b	system_call_vectored_tramp
 824#else
 825	b	system_call_vectored_common
 826#endif
 827	nop
 828
 829	/* SCV 1 - 127 */
 830	.rept	127
 831	mr	r9,r13
 832	GET_PACA(r13)
 833	mflr	r11
 834	mfctr	r12
 835	li	r10,IRQS_ALL_DISABLED
 836	stb	r10,PACAIRQSOFTMASK(r13)
 837	li	r0,-1 /* cause failure */
 838#ifdef CONFIG_RELOCATABLE
 839	b	system_call_vectored_sigill_tramp
 840#else
 841	b	system_call_vectored_sigill
 842#endif
 843	.endr
 844EXC_VIRT_END(system_call_vectored, 0x3000, 0x1000)
 845
 846// Treat scv vectors as soft-masked, see comment above.
 847// Use absolute values rather than labels here, so they don't get relocated,
 848// because this code runs unrelocated.
 849SOFT_MASK_TABLE(0xc000000000003000, 0xc000000000004000)
 850
 851#ifdef CONFIG_RELOCATABLE
 852TRAMP_VIRT_BEGIN(system_call_vectored_tramp)
 853	__LOAD_HANDLER(r10, system_call_vectored_common)
 854	mtctr	r10
 855	bctr
 856
 857TRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp)
 858	__LOAD_HANDLER(r10, system_call_vectored_sigill)
 859	mtctr	r10
 860	bctr
 861#endif
 862
 863
 864/* No virt vectors corresponding with 0x0..0x100 */
 865EXC_VIRT_NONE(0x4000, 0x100)
 866
 867
 868/**
 869 * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI).
 870 * This is a non-maskable, asynchronous interrupt always taken in real-mode.
 871 * It is caused by:
 872 * - Wake from power-saving state, on powernv.
 873 * - An NMI from another CPU, triggered by firmware or hypercall.
 874 * - As crash/debug signal injected from BMC, firmware or hypervisor.
 875 *
 876 * Handling:
 877 * Power-save wakeup is the only performance critical path, so this is
 878 * determined quickly as possible first. In this case volatile registers
 879 * can be discarded and SPRs like CFAR don't need to be read.
 880 *
 881 * If not a powersave wakeup, then it's run as a regular interrupt, however
 882 * it uses its own stack and PACA save area to preserve the regular kernel
 883 * environment for debugging.
 884 *
 885 * This interrupt is not maskable, so triggering it when MSR[RI] is clear,
 886 * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely
 887 * correct to switch to virtual mode to run the regular interrupt handler
 888 * because it might be interrupted when the MMU is in a bad state (e.g., SLB
 889 * is clear).
 890 *
 891 * FWNMI:
 892 * PAPR specifies a "fwnmi" facility which sends the sreset to a different
 893 * entry point with a different register set up. Some hypervisors will
 894 * send the sreset to 0x100 in the guest if it is not fwnmi capable.
 895 *
 896 * KVM:
 897 * Unlike most SRR interrupts, this may be taken by the host while executing
 898 * in a guest, so a KVM test is required. KVM will pull the CPU out of guest
 899 * mode and then raise the sreset.
 900 */
 901INT_DEFINE_BEGIN(system_reset)
 902	IVEC=0x100
 903	IAREA=PACA_EXNMI
 904	IVIRT=0 /* no virt entry point */
 905	/*
 906	 * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is
 907	 * being used, so a nested NMI exception would corrupt it.
 908	 */
 909	ISET_RI=0
 910	ISTACK=0
 911	IKVM_REAL=1
 912INT_DEFINE_END(system_reset)
 913
 914EXC_REAL_BEGIN(system_reset, 0x100, 0x100)
 915#ifdef CONFIG_PPC_P7_NAP
 916	/*
 917	 * If running native on arch 2.06 or later, check if we are waking up
 918	 * from nap/sleep/winkle, and branch to idle handler. This tests SRR1
 919	 * bits 46:47. A non-0 value indicates that we are coming from a power
 920	 * saving state. The idle wakeup handler initially runs in real mode,
 921	 * but we branch to the 0xc000... address so we can turn on relocation
 922	 * with mtmsrd later, after SPRs are restored.
 923	 *
 924	 * Careful to minimise cost for the fast path (idle wakeup) while
 925	 * also avoiding clobbering CFAR for the debug path (non-idle).
 926	 *
 927	 * For the idle wake case volatile registers can be clobbered, which
 928	 * is why we use those initially. If it turns out to not be an idle
 929	 * wake, carefully put everything back the way it was, so we can use
 930	 * common exception macros to handle it.
 931	 */
 932BEGIN_FTR_SECTION
 933	SET_SCRATCH0(r13)
 934	GET_PACA(r13)
 935	std	r3,PACA_EXNMI+0*8(r13)
 936	std	r4,PACA_EXNMI+1*8(r13)
 937	std	r5,PACA_EXNMI+2*8(r13)
 938	mfspr	r3,SPRN_SRR1
 939	mfocrf	r4,0x80
 940	rlwinm.	r5,r3,47-31,30,31
 941	bne+	system_reset_idle_wake
 942	/* Not powersave wakeup. Restore regs for regular interrupt handler. */
 943	mtocrf	0x80,r4
 944	ld	r3,PACA_EXNMI+0*8(r13)
 945	ld	r4,PACA_EXNMI+1*8(r13)
 946	ld	r5,PACA_EXNMI+2*8(r13)
 947	GET_SCRATCH0(r13)
 948END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 949#endif
 950
 951	GEN_INT_ENTRY system_reset, virt=0
 952	/*
 
 
 
 953	 * In theory, we should not enable relocation here if it was disabled
 954	 * in SRR1, because the MMU may not be configured to support it (e.g.,
 955	 * SLB may have been cleared). In practice, there should only be a few
 956	 * small windows where that's the case, and sreset is considered to
 957	 * be dangerous anyway.
 958	 */
 959EXC_REAL_END(system_reset, 0x100, 0x100)
 960EXC_VIRT_NONE(0x4100, 0x100)
 
 961
 962#ifdef CONFIG_PPC_P7_NAP
 963TRAMP_REAL_BEGIN(system_reset_idle_wake)
 964	/* We are waking up from idle, so may clobber any volatile register */
 965	cmpwi	cr1,r5,2
 966	bltlr	cr1	/* no state loss, return to idle caller with r3=SRR1 */
 967	BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss))
 968#endif
 969
 970#ifdef CONFIG_PPC_PSERIES
 971/*
 972 * Vectors for the FWNMI option.  Share common code.
 973 */
 974TRAMP_REAL_BEGIN(system_reset_fwnmi)
 975	GEN_INT_ENTRY system_reset, virt=0
 
 976
 977#endif /* CONFIG_PPC_PSERIES */
 978
 979EXC_COMMON_BEGIN(system_reset_common)
 980	__GEN_COMMON_ENTRY system_reset
 981	/*
 982	 * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able
 983	 * to recover, but nested NMI will notice in_nmi and not recover
 984	 * because of the use of the NMI stack. in_nmi reentrancy is tested in
 985	 * system_reset_exception.
 986	 */
 987	lhz	r10,PACA_IN_NMI(r13)
 988	addi	r10,r10,1
 989	sth	r10,PACA_IN_NMI(r13)
 990	li	r10,MSR_RI
 991	mtmsrd 	r10,1
 992
 993	mr	r10,r1
 994	ld	r1,PACA_NMI_EMERG_SP(r13)
 995	subi	r1,r1,INT_FRAME_SIZE
 996	__GEN_COMMON_BODY system_reset
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 997
 998	addi	r3,r1,STACK_FRAME_OVERHEAD
 999	bl	system_reset_exception
1000
1001	/* Clear MSR_RI before setting SRR0 and SRR1. */
1002	li	r9,0
1003	mtmsrd	r9,1
1004
1005	/*
1006	 * MSR_RI is clear, now we can decrement paca->in_nmi.
1007	 */
1008	lhz	r10,PACA_IN_NMI(r13)
1009	subi	r10,r10,1
1010	sth	r10,PACA_IN_NMI(r13)
1011
1012	kuap_kernel_restore r9, r10
1013	EXCEPTION_RESTORE_REGS
 
 
 
 
 
 
 
1014	RFI_TO_USER_OR_KERNEL
1015
1016
1017/**
1018 * Interrupt 0x200 - Machine Check Interrupt (MCE).
1019 * This is a non-maskable interrupt always taken in real-mode. It can be
1020 * synchronous or asynchronous, caused by hardware or software, and it may be
1021 * taken in a power-saving state.
1022 *
1023 * Handling:
1024 * Similarly to system reset, this uses its own stack and PACA save area,
1025 * the difference is re-entrancy is allowed on the machine check stack.
1026 *
1027 * machine_check_early is run in real mode, and carefully decodes the
1028 * machine check and tries to handle it (e.g., flush the SLB if there was an
1029 * error detected there), determines if it was recoverable and logs the
1030 * event.
1031 *
1032 * This early code does not "reconcile" irq soft-mask state like SRESET or
1033 * regular interrupts do, so irqs_disabled() among other things may not work
1034 * properly (irq disable/enable already doesn't work because irq tracing can
1035 * not work in real mode).
1036 *
1037 * Then, depending on the execution context when the interrupt is taken, there
1038 * are 3 main actions:
1039 * - Executing in kernel mode. The event is queued with irq_work, which means
1040 *   it is handled when it is next safe to do so (i.e., the kernel has enabled
1041 *   interrupts), which could be immediately when the interrupt returns. This
1042 *   avoids nasty issues like switching to virtual mode when the MMU is in a
1043 *   bad state, or when executing OPAL code. (SRESET is exposed to such issues,
1044 *   but it has different priorities). Check to see if the CPU was in power
1045 *   save, and return via the wake up code if it was.
1046 *
1047 * - Executing in user mode. machine_check_exception is run like a normal
1048 *   interrupt handler, which processes the data generated by the early handler.
1049 *
1050 * - Executing in guest mode. The interrupt is run with its KVM test, and
1051 *   branches to KVM to deal with. KVM may queue the event for the host
1052 *   to report later.
1053 *
1054 * This interrupt is not maskable, so if it triggers when MSR[RI] is clear,
1055 * or SCRATCH0 is in use, it may cause a crash.
1056 *
1057 * KVM:
1058 * See SRESET.
1059 */
1060INT_DEFINE_BEGIN(machine_check_early)
1061	IVEC=0x200
1062	IAREA=PACA_EXMC
1063	IVIRT=0 /* no virt entry point */
1064	IREALMODE_COMMON=1
1065	/*
1066	 * MSR_RI is not enabled, because PACA_EXMC is being used, so a
1067	 * nested machine check corrupts it. machine_check_common enables
1068	 * MSR_RI.
1069	 */
1070	ISET_RI=0
1071	ISTACK=0
1072	IDAR=1
1073	IDSISR=1
1074	IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
1075INT_DEFINE_END(machine_check_early)
1076
1077INT_DEFINE_BEGIN(machine_check)
1078	IVEC=0x200
1079	IAREA=PACA_EXMC
1080	IVIRT=0 /* no virt entry point */
1081	ISET_RI=0
1082	IDAR=1
1083	IDSISR=1
1084	IKVM_REAL=1
1085INT_DEFINE_END(machine_check)
1086
1087EXC_REAL_BEGIN(machine_check, 0x200, 0x100)
1088	GEN_INT_ENTRY machine_check_early, virt=0
1089EXC_REAL_END(machine_check, 0x200, 0x100)
1090EXC_VIRT_NONE(0x4200, 0x100)
1091
1092#ifdef CONFIG_PPC_PSERIES
1093TRAMP_REAL_BEGIN(machine_check_fwnmi)
1094	/* See comment at machine_check exception, don't turn on RI */
1095	GEN_INT_ENTRY machine_check_early, virt=0
1096#endif
1097
 
 
1098#define MACHINE_CHECK_HANDLER_WINDUP			\
1099	/* Clear MSR_RI before setting SRR0 and SRR1. */\
1100	li	r9,0;					\
1101	mtmsrd	r9,1;		/* Clear MSR_RI */	\
1102	/* Decrement paca->in_mce now RI is clear. */	\
1103	lhz	r12,PACA_IN_MCE(r13);			\
1104	subi	r12,r12,1;				\
1105	sth	r12,PACA_IN_MCE(r13);			\
1106	EXCEPTION_RESTORE_REGS
1107
1108EXC_COMMON_BEGIN(machine_check_early_common)
1109	__GEN_REALMODE_COMMON_ENTRY machine_check_early
 
 
1110
1111	/*
1112	 * Switch to mc_emergency stack and handle re-entrancy (we limit
1113	 * the nested MCE upto level 4 to avoid stack overflow).
1114	 * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
1115	 *
1116	 * We use paca->in_mce to check whether this is the first entry or
1117	 * nested machine check. We increment paca->in_mce to track nested
1118	 * machine checks.
1119	 *
1120	 * If this is the first entry then set stack pointer to
1121	 * paca->mc_emergency_sp, otherwise r1 is already pointing to
1122	 * stack frame on mc_emergency stack.
1123	 *
1124	 * NOTE: We are here with MSR_ME=0 (off), which means we risk a
1125	 * checkstop if we get another machine check exception before we do
1126	 * rfid with MSR_ME=1.
1127	 *
1128	 * This interrupt can wake directly from idle. If that is the case,
1129	 * the machine check is handled then the idle wakeup code is called
1130	 * to restore state.
1131	 */
1132	lhz	r10,PACA_IN_MCE(r13)
1133	cmpwi	r10,0			/* Are we in nested machine check */
1134	cmpwi	cr1,r10,MAX_MCE_DEPTH	/* Are we at maximum nesting */
1135	addi	r10,r10,1		/* increment paca->in_mce */
1136	sth	r10,PACA_IN_MCE(r13)
1137
1138	mr	r10,r1			/* Save r1 */
1139	bne	1f
1140	/* First machine check entry */
1141	ld	r1,PACAMCEMERGSP(r13)	/* Use MC emergency stack */
11421:	/* Limit nested MCE to level 4 to avoid stack overflow */
1143	bgt	cr1,unrecoverable_mce	/* Check if we hit limit of 4 */
1144	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame */
1145
1146	__GEN_COMMON_BODY machine_check_early
 
1147
1148BEGIN_FTR_SECTION
1149	bl	enable_machine_check
1150END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
1151	li	r10,MSR_RI
1152	mtmsrd	r10,1
1153
 
1154	addi	r3,r1,STACK_FRAME_OVERHEAD
1155	bl	machine_check_early
1156	std	r3,RESULT(r1)	/* Save result */
1157	ld	r12,_MSR(r1)
1158
1159#ifdef CONFIG_PPC_P7_NAP
1160	/*
1161	 * Check if thread was in power saving mode. We come here when any
1162	 * of the following is true:
1163	 * a. thread wasn't in power saving mode
1164	 * b. thread was in power saving mode with no state loss,
1165	 *    supervisor state loss or hypervisor state loss.
1166	 *
1167	 * Go back to nap/sleep/winkle mode again if (b) is true.
1168	 */
1169BEGIN_FTR_SECTION
1170	rlwinm.	r11,r12,47-31,30,31
1171	bne	machine_check_idle_common
1172END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
1173#endif
1174
1175#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1176	/*
1177	 * Check if we are coming from guest. If yes, then run the normal
1178	 * exception handler which will take the
1179	 * machine_check_kvm->kvm_interrupt branch to deliver the MC event
1180	 * to guest.
1181	 */
1182	lbz	r11,HSTATE_IN_GUEST(r13)
1183	cmpwi	r11,0			/* Check if coming from guest */
1184	bne	mce_deliver		/* continue if we are. */
1185#endif
1186
1187	/*
1188	 * Check if we are coming from userspace. If yes, then run the normal
1189	 * exception handler which will deliver the MC event to this kernel.
1190	 */
1191	andi.	r11,r12,MSR_PR		/* See if coming from user. */
1192	bne	mce_deliver		/* continue in V mode if we are. */
1193
1194	/*
1195	 * At this point we are coming from kernel context.
1196	 * Queue up the MCE event and return from the interrupt.
1197	 * But before that, check if this is an un-recoverable exception.
1198	 * If yes, then stay on emergency stack and panic.
1199	 */
1200	andi.	r11,r12,MSR_RI
1201	beq	unrecoverable_mce
1202
1203	/*
1204	 * Check if we have successfully handled/recovered from error, if not
1205	 * then stay on emergency stack and panic.
1206	 */
1207	ld	r3,RESULT(r1)	/* Load result */
1208	cmpdi	r3,0		/* see if we handled MCE successfully */
1209	beq	unrecoverable_mce /* if !handled then panic */
1210
1211	/*
1212	 * Return from MC interrupt.
1213	 * Queue up the MCE event so that we can log it later, while
1214	 * returning from kernel or opal call.
1215	 */
1216	bl	machine_check_queue_event
1217	MACHINE_CHECK_HANDLER_WINDUP
1218	RFI_TO_KERNEL
1219
1220mce_deliver:
1221	/*
1222	 * This is a host user or guest MCE. Restore all registers, then
1223	 * run the "late" handler. For host user, this will run the
1224	 * machine_check_exception handler in virtual mode like a normal
1225	 * interrupt handler. For guest, this will trigger the KVM test
1226	 * and branch to the KVM interrupt similarly to other interrupts.
1227	 */
1228BEGIN_FTR_SECTION
1229	ld	r10,ORIG_GPR3(r1)
1230	mtspr	SPRN_CFAR,r10
1231END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1232	MACHINE_CHECK_HANDLER_WINDUP
1233	GEN_INT_ENTRY machine_check, virt=0
 
1234
1235EXC_COMMON_BEGIN(machine_check_common)
1236	/*
1237	 * Machine check is different because we use a different
1238	 * save area: PACA_EXMC instead of PACA_EXGEN.
1239	 */
1240	GEN_COMMON machine_check
1241
1242	/* Enable MSR_RI when finished with PACA_EXMC */
1243	li	r10,MSR_RI
1244	mtmsrd 	r10,1
 
1245	addi	r3,r1,STACK_FRAME_OVERHEAD
1246	bl	machine_check_exception_async
1247	b	interrupt_return_srr
1248
1249
1250#ifdef CONFIG_PPC_P7_NAP
1251/*
1252 * This is an idle wakeup. Low level machine check has already been
1253 * done. Queue the event then call the idle code to do the wake up.
1254 */
1255EXC_COMMON_BEGIN(machine_check_idle_common)
1256	bl	machine_check_queue_event
1257
1258	/*
1259	 * GPR-loss wakeups are relatively straightforward, because the
1260	 * idle sleep code has saved all non-volatile registers on its
1261	 * own stack, and r1 in PACAR1.
 
1262	 *
1263	 * For no-loss wakeups the r1 and lr registers used by the
1264	 * early machine check handler have to be restored first. r2 is
1265	 * the kernel TOC, so no need to restore it.
1266	 *
1267	 * Then decrement MCE nesting after finishing with the stack.
1268	 */
1269	ld	r3,_MSR(r1)
1270	ld	r4,_LINK(r1)
1271	ld	r1,GPR1(r1)
1272
1273	lhz	r11,PACA_IN_MCE(r13)
1274	subi	r11,r11,1
1275	sth	r11,PACA_IN_MCE(r13)
1276
1277	mtlr	r4
1278	rlwinm	r10,r3,47-31,30,31
1279	cmpwi	cr1,r10,2
1280	bltlr	cr1	/* no state loss, return to idle caller with r3=SRR1 */
1281	b	idle_return_gpr_loss
1282#endif
1283
1284EXC_COMMON_BEGIN(unrecoverable_mce)
1285	/*
1286	 * We are going down. But there are chances that we might get hit by
1287	 * another MCE during panic path and we may run into unstable state
1288	 * with no way out. Hence, turn ME bit off while going down, so that
1289	 * when another MCE is hit during panic path, system will checkstop
1290	 * and hypervisor will get restarted cleanly by SP.
1291	 */
1292BEGIN_FTR_SECTION
1293	li	r10,0 /* clear MSR_RI */
1294	mtmsrd	r10,1
1295	bl	disable_machine_check
1296END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
1297	ld	r10,PACAKMSR(r13)
1298	li	r3,MSR_ME
1299	andc	r10,r10,r3
1300	mtmsrd	r10
1301
1302	lhz	r12,PACA_IN_MCE(r13)
1303	subi	r12,r12,1
1304	sth	r12,PACA_IN_MCE(r13)
1305
1306	/*
1307	 * Invoke machine_check_exception to print MCE event and panic.
1308	 * This is the NMI version of the handler because we are called from
1309	 * the early handler which is a true NMI.
1310	 */
1311	addi	r3,r1,STACK_FRAME_OVERHEAD
1312	bl	machine_check_exception
1313
1314	/*
1315	 * We will not reach here. Even if we did, there is no way out.
1316	 * Call unrecoverable_exception and die.
1317	 */
1318	addi	r3,r1,STACK_FRAME_OVERHEAD
1319	bl	unrecoverable_exception
1320	b	.
1321
1322
1323/**
1324 * Interrupt 0x300 - Data Storage Interrupt (DSI).
1325 * This is a synchronous interrupt generated due to a data access exception,
1326 * e.g., a load orstore which does not have a valid page table entry with
1327 * permissions. DAWR matches also fault here, as do RC updates, and minor misc
1328 * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc.
1329 *
1330 * Handling:
1331 * - Hash MMU
1332 *   Go to do_hash_fault, which attempts to fill the HPT from an entry in the
1333 *   Linux page table. Hash faults can hit in kernel mode in a fairly
1334 *   arbitrary state (e.g., interrupts disabled, locks held) when accessing
1335 *   "non-bolted" regions, e.g., vmalloc space. However these should always be
1336 *   backed by Linux page table entries.
1337 *
1338 *   If no entry is found the Linux page fault handler is invoked (by
1339 *   do_hash_fault). Linux page faults can happen in kernel mode due to user
1340 *   copy operations of course.
1341 *
1342 *   KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest
1343 *   MMU context, which may cause a DSI in the host, which must go to the
1344 *   KVM handler. MSR[IR] is not enabled, so the real-mode handler will
1345 *   always be used regardless of AIL setting.
1346 *
1347 * - Radix MMU
1348 *   The hardware loads from the Linux page table directly, so a fault goes
1349 *   immediately to Linux page fault.
1350 *
1351 * Conditions like DAWR match are handled on the way in to Linux page fault.
1352 */
1353INT_DEFINE_BEGIN(data_access)
1354	IVEC=0x300
1355	IDAR=1
1356	IDSISR=1
1357	IKVM_REAL=1
1358INT_DEFINE_END(data_access)
1359
1360EXC_REAL_BEGIN(data_access, 0x300, 0x80)
1361	GEN_INT_ENTRY data_access, virt=0
1362EXC_REAL_END(data_access, 0x300, 0x80)
1363EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
1364	GEN_INT_ENTRY data_access, virt=1
1365EXC_VIRT_END(data_access, 0x4300, 0x80)
 
1366EXC_COMMON_BEGIN(data_access_common)
1367	GEN_COMMON data_access
1368	ld	r4,_DSISR(r1)
1369	addi	r3,r1,STACK_FRAME_OVERHEAD
1370	andis.	r0,r4,DSISR_DABRMATCH@h
1371	bne-	1f
 
 
 
 
1372BEGIN_MMU_FTR_SECTION
1373	bl	do_hash_fault
 
 
1374MMU_FTR_SECTION_ELSE
1375	bl	do_page_fault
1376ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1377	b	interrupt_return_srr
1378
13791:	bl	do_break
1380	/*
1381	 * do_break() may have changed the NV GPRS while handling a breakpoint.
1382	 * If so, we need to restore them with their updated values.
1383	 */
1384	REST_NVGPRS(r1)
1385	b	interrupt_return_srr
1386
1387
1388/**
1389 * Interrupt 0x380 - Data Segment Interrupt (DSLB).
1390 * This is a synchronous interrupt in response to an MMU fault missing SLB
1391 * entry for HPT, or an address outside RPT translation range.
1392 *
1393 * Handling:
1394 * - HPT:
1395 *   This refills the SLB, or reports an access fault similarly to a bad page
1396 *   fault. When coming from user-mode, the SLB handler may access any kernel
1397 *   data, though it may itself take a DSLB. When coming from kernel mode,
1398 *   recursive faults must be avoided so access is restricted to the kernel
1399 *   image text/data, kernel stack, and any data allocated below
1400 *   ppc64_bolted_size (first segment). The kernel handler must avoid stomping
1401 *   on user-handler data structures.
1402 *
1403 *   KVM: Same as 0x300, DSLB must test for KVM guest.
1404 */
1405INT_DEFINE_BEGIN(data_access_slb)
1406	IVEC=0x380
1407	IDAR=1
1408	IKVM_REAL=1
1409INT_DEFINE_END(data_access_slb)
1410
1411EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
1412	GEN_INT_ENTRY data_access_slb, virt=0
1413EXC_REAL_END(data_access_slb, 0x380, 0x80)
1414EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
1415	GEN_INT_ENTRY data_access_slb, virt=1
1416EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
 
1417EXC_COMMON_BEGIN(data_access_slb_common)
1418	GEN_COMMON data_access_slb
 
 
1419BEGIN_MMU_FTR_SECTION
1420	/* HPT case, do SLB fault */
1421	addi	r3,r1,STACK_FRAME_OVERHEAD
1422	bl	do_slb_fault
1423	cmpdi	r3,0
1424	bne-	1f
1425	b	fast_interrupt_return_srr
14261:	/* Error case */
1427MMU_FTR_SECTION_ELSE
1428	/* Radix case, access is outside page table range */
1429	li	r3,-EFAULT
1430ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1431	std	r3,RESULT(r1)
 
 
 
 
1432	addi	r3,r1,STACK_FRAME_OVERHEAD
1433	bl	do_bad_slb_fault
1434	b	interrupt_return_srr
1435
1436
1437/**
1438 * Interrupt 0x400 - Instruction Storage Interrupt (ISI).
1439 * This is a synchronous interrupt in response to an MMU fault due to an
1440 * instruction fetch.
1441 *
1442 * Handling:
1443 * Similar to DSI, though in response to fetch. The faulting address is found
1444 * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR).
1445 */
1446INT_DEFINE_BEGIN(instruction_access)
1447	IVEC=0x400
1448	IISIDE=1
1449	IDAR=1
1450	IDSISR=1
1451#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1452	IKVM_REAL=1
1453#endif
1454INT_DEFINE_END(instruction_access)
1455
1456EXC_REAL_BEGIN(instruction_access, 0x400, 0x80)
1457	GEN_INT_ENTRY instruction_access, virt=0
1458EXC_REAL_END(instruction_access, 0x400, 0x80)
1459EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
1460	GEN_INT_ENTRY instruction_access, virt=1
1461EXC_VIRT_END(instruction_access, 0x4400, 0x80)
 
1462EXC_COMMON_BEGIN(instruction_access_common)
1463	GEN_COMMON instruction_access
1464	addi	r3,r1,STACK_FRAME_OVERHEAD
 
1465BEGIN_MMU_FTR_SECTION
1466	bl	do_hash_fault
 
 
1467MMU_FTR_SECTION_ELSE
1468	bl	do_page_fault
1469ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1470	b	interrupt_return_srr
1471
1472
1473/**
1474 * Interrupt 0x480 - Instruction Segment Interrupt (ISLB).
1475 * This is a synchronous interrupt in response to an MMU fault due to an
1476 * instruction fetch.
1477 *
1478 * Handling:
1479 * Similar to DSLB, though in response to fetch. The faulting address is found
1480 * in SRR0 (rather than DAR).
1481 */
1482INT_DEFINE_BEGIN(instruction_access_slb)
1483	IVEC=0x480
1484	IISIDE=1
1485	IDAR=1
1486#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1487	IKVM_REAL=1
1488#endif
1489INT_DEFINE_END(instruction_access_slb)
1490
1491EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80)
1492	GEN_INT_ENTRY instruction_access_slb, virt=0
1493EXC_REAL_END(instruction_access_slb, 0x480, 0x80)
1494EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
1495	GEN_INT_ENTRY instruction_access_slb, virt=1
1496EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
 
1497EXC_COMMON_BEGIN(instruction_access_slb_common)
1498	GEN_COMMON instruction_access_slb
 
 
1499BEGIN_MMU_FTR_SECTION
1500	/* HPT case, do SLB fault */
1501	addi	r3,r1,STACK_FRAME_OVERHEAD
1502	bl	do_slb_fault
1503	cmpdi	r3,0
1504	bne-	1f
1505	b	fast_interrupt_return_srr
15061:	/* Error case */
1507MMU_FTR_SECTION_ELSE
1508	/* Radix case, access is outside page table range */
1509	li	r3,-EFAULT
1510ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
1511	std	r3,RESULT(r1)
 
 
 
 
1512	addi	r3,r1,STACK_FRAME_OVERHEAD
1513	bl	do_bad_slb_fault
1514	b	interrupt_return_srr
1515
1516
1517/**
1518 * Interrupt 0x500 - External Interrupt.
1519 * This is an asynchronous maskable interrupt in response to an "external
1520 * exception" from the interrupt controller or hypervisor (e.g., device
1521 * interrupt). It is maskable in hardware by clearing MSR[EE], and
1522 * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()).
1523 *
1524 * When running in HV mode, Linux sets up the LPCR[LPES] bit such that
1525 * interrupts are delivered with HSRR registers, guests use SRRs, which
1526 * reqiures IHSRR_IF_HVMODE.
1527 *
1528 * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that
1529 * external interrupts are delivered as Hypervisor Virtualization Interrupts
1530 * rather than External Interrupts.
1531 *
1532 * Handling:
1533 * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead,
1534 * because registers at the time of the interrupt are not so important as it is
1535 * asynchronous.
1536 *
1537 * If soft masked, the masked handler will note the pending interrupt for
1538 * replay, and clear MSR[EE] in the interrupted context.
1539 */
1540INT_DEFINE_BEGIN(hardware_interrupt)
1541	IVEC=0x500
1542	IHSRR_IF_HVMODE=1
1543	IMASK=IRQS_DISABLED
1544	IKVM_REAL=1
1545	IKVM_VIRT=1
1546INT_DEFINE_END(hardware_interrupt)
1547
1548EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
1549	GEN_INT_ENTRY hardware_interrupt, virt=0
1550EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
1551EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
1552	GEN_INT_ENTRY hardware_interrupt, virt=1
1553EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
1554EXC_COMMON_BEGIN(hardware_interrupt_common)
1555	GEN_COMMON hardware_interrupt
1556	addi	r3,r1,STACK_FRAME_OVERHEAD
1557	bl	do_IRQ
1558	BEGIN_FTR_SECTION
1559	b	interrupt_return_hsrr
1560	FTR_SECTION_ELSE
1561	b	interrupt_return_srr
1562	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
1563
1564
1565/**
1566 * Interrupt 0x600 - Alignment Interrupt
1567 * This is a synchronous interrupt in response to data alignment fault.
1568 */
1569INT_DEFINE_BEGIN(alignment)
1570	IVEC=0x600
1571	IDAR=1
1572	IDSISR=1
1573#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1574	IKVM_REAL=1
1575#endif
1576INT_DEFINE_END(alignment)
1577
1578EXC_REAL_BEGIN(alignment, 0x600, 0x100)
1579	GEN_INT_ENTRY alignment, virt=0
1580EXC_REAL_END(alignment, 0x600, 0x100)
1581EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
1582	GEN_INT_ENTRY alignment, virt=1
1583EXC_VIRT_END(alignment, 0x4600, 0x100)
 
1584EXC_COMMON_BEGIN(alignment_common)
1585	GEN_COMMON alignment
 
1586	addi	r3,r1,STACK_FRAME_OVERHEAD
1587	bl	alignment_exception
1588	REST_NVGPRS(r1) /* instruction emulation may change GPRs */
1589	b	interrupt_return_srr
1590
1591
1592/**
1593 * Interrupt 0x700 - Program Interrupt (program check).
1594 * This is a synchronous interrupt in response to various instruction faults:
1595 * traps, privilege errors, TM errors, floating point exceptions.
1596 *
1597 * Handling:
1598 * This interrupt may use the "emergency stack" in some cases when being taken
1599 * from kernel context, which complicates handling.
1600 */
1601INT_DEFINE_BEGIN(program_check)
1602	IVEC=0x700
1603#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1604	IKVM_REAL=1
1605#endif
1606INT_DEFINE_END(program_check)
1607
1608EXC_REAL_BEGIN(program_check, 0x700, 0x100)
1609
1610#ifdef CONFIG_CPU_LITTLE_ENDIAN
1611	/*
1612	 * There's a short window during boot where although the kernel is
1613	 * running little endian, any exceptions will cause the CPU to switch
1614	 * back to big endian. For example a WARN() boils down to a trap
1615	 * instruction, which will cause a program check, and we end up here but
1616	 * with the CPU in big endian mode. The first instruction of the program
1617	 * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when
1618	 * executed in the wrong endian is an lhzu with a ~3GB displacement from
1619	 * r3. The content of r3 is random, so that is a load from some random
1620	 * location, and depending on the system can easily lead to a checkstop,
1621	 * or an infinitely recursive page fault.
1622	 *
1623	 * So to handle that case we have a trampoline here that can detect we
1624	 * are in the wrong endian and flip us back to the correct endian. We
1625	 * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires
1626	 * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as
1627	 * SPRG1 is already used for the paca. SPRG3 is user readable, but this
1628	 * trampoline is only active very early in boot, and SPRG3 will be
1629	 * reinitialised in vdso_getcpu_init() before userspace starts.
1630	 */
1631BEGIN_FTR_SECTION
1632	tdi   0,0,0x48    // Trap never, or in reverse endian: b . + 8
1633	b     1f          // Skip trampoline if endian is correct
1634	.long 0xa643707d  // mtsprg  0, r11      Backup r11
1635	.long 0xa6027a7d  // mfsrr0  r11
1636	.long 0xa643727d  // mtsprg  2, r11      Backup SRR0 in SPRG2
1637	.long 0xa6027b7d  // mfsrr1  r11
1638	.long 0xa643737d  // mtsprg  3, r11      Backup SRR1 in SPRG3
1639	.long 0xa600607d  // mfmsr   r11
1640	.long 0x01006b69  // xori    r11, r11, 1 Invert MSR[LE]
1641	.long 0xa6037b7d  // mtsrr1  r11
1642	.long 0x34076039  // li      r11, 0x734
1643	.long 0xa6037a7d  // mtsrr0  r11
1644	.long 0x2400004c  // rfid
1645	mfsprg r11, 3
1646	mtsrr1 r11        // Restore SRR1
1647	mfsprg r11, 2
1648	mtsrr0 r11        // Restore SRR0
1649	mfsprg r11, 0     // Restore r11
16501:
1651END_FTR_SECTION(0, 1)     // nop out after boot
1652#endif /* CONFIG_CPU_LITTLE_ENDIAN */
1653
1654	GEN_INT_ENTRY program_check, virt=0
1655EXC_REAL_END(program_check, 0x700, 0x100)
1656EXC_VIRT_BEGIN(program_check, 0x4700, 0x100)
1657	GEN_INT_ENTRY program_check, virt=1
1658EXC_VIRT_END(program_check, 0x4700, 0x100)
 
1659EXC_COMMON_BEGIN(program_check_common)
1660	__GEN_COMMON_ENTRY program_check
1661
1662	/*
1663	 * It's possible to receive a TM Bad Thing type program check with
1664	 * userspace register values (in particular r1), but with SRR1 reporting
1665	 * that we came from the kernel. Normally that would confuse the bad
1666	 * stack logic, and we would report a bad kernel stack pointer. Instead
1667	 * we switch to the emergency stack if we're taking a TM Bad Thing from
1668	 * the kernel.
1669	 */
1670
1671	andi.	r10,r12,MSR_PR
1672	bne	.Lnormal_stack		/* If userspace, go normal path */
1673
1674	andis.	r10,r12,(SRR1_PROGTM)@h
1675	bne	.Lemergency_stack	/* If TM, emergency		*/
1676
1677	cmpdi	r1,-INT_FRAME_SIZE	/* check if r1 is in userspace	*/
1678	blt	.Lnormal_stack		/* normal path if not		*/
1679
1680	/* Use the emergency stack					*/
1681.Lemergency_stack:
1682	andi.	r10,r12,MSR_PR		/* Set CR0 correctly for label	*/
1683					/* 3 in EXCEPTION_PROLOG_COMMON	*/
1684	mr	r10,r1			/* Save r1			*/
1685	ld	r1,PACAEMERGSP(r13)	/* Use emergency stack		*/
1686	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame		*/
1687	__ISTACK(program_check)=0
1688	__GEN_COMMON_BODY program_check
1689	b .Ldo_program_check
1690
1691.Lnormal_stack:
1692	__ISTACK(program_check)=1
1693	__GEN_COMMON_BODY program_check
1694
1695.Ldo_program_check:
1696	addi	r3,r1,STACK_FRAME_OVERHEAD
1697	bl	program_check_exception
1698	REST_NVGPRS(r1) /* instruction emulation may change GPRs */
1699	b	interrupt_return_srr
1700
1701
1702/*
1703 * Interrupt 0x800 - Floating-Point Unavailable Interrupt.
1704 * This is a synchronous interrupt in response to executing an fp instruction
1705 * with MSR[FP]=0.
1706 *
1707 * Handling:
1708 * This will load FP registers and enable the FP bit if coming from userspace,
1709 * otherwise report a bad kernel use of FP.
1710 */
1711INT_DEFINE_BEGIN(fp_unavailable)
1712	IVEC=0x800
1713#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1714	IKVM_REAL=1
1715#endif
1716INT_DEFINE_END(fp_unavailable)
1717
1718EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100)
1719	GEN_INT_ENTRY fp_unavailable, virt=0
1720EXC_REAL_END(fp_unavailable, 0x800, 0x100)
1721EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100)
1722	GEN_INT_ENTRY fp_unavailable, virt=1
1723EXC_VIRT_END(fp_unavailable, 0x4800, 0x100)
 
1724EXC_COMMON_BEGIN(fp_unavailable_common)
1725	GEN_COMMON fp_unavailable
1726	bne	1f			/* if from user, just load it up */
 
 
1727	addi	r3,r1,STACK_FRAME_OVERHEAD
1728	bl	kernel_fp_unavailable_exception
17290:	trap
1730	EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0
17311:
1732#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1733BEGIN_FTR_SECTION
1734	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
1735	 * transaction), go do TM stuff
1736	 */
1737	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
1738	bne-	2f
1739END_FTR_SECTION_IFSET(CPU_FTR_TM)
1740#endif
1741	bl	load_up_fpu
1742	b	fast_interrupt_return_srr
1743#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
17442:	/* User process was in a transaction */
 
 
1745	addi	r3,r1,STACK_FRAME_OVERHEAD
1746	bl	fp_unavailable_tm
1747	b	interrupt_return_srr
1748#endif
1749
1750
1751/**
1752 * Interrupt 0x900 - Decrementer Interrupt.
1753 * This is an asynchronous interrupt in response to a decrementer exception
1754 * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing
1755 * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e.,
1756 * local_irq_disable()).
1757 *
1758 * Handling:
1759 * This calls into Linux timer handler. NVGPRs are not saved (see 0x500).
1760 *
1761 * If soft masked, the masked handler will note the pending interrupt for
1762 * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled
1763 * in the interrupted context.
1764 * If PPC_WATCHDOG is configured, the soft masked handler will actually set
1765 * things back up to run soft_nmi_interrupt as a regular interrupt handler
1766 * on the emergency stack.
1767 */
1768INT_DEFINE_BEGIN(decrementer)
1769	IVEC=0x900
1770	IMASK=IRQS_DISABLED
1771#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1772	IKVM_REAL=1
1773#endif
1774INT_DEFINE_END(decrementer)
1775
1776EXC_REAL_BEGIN(decrementer, 0x900, 0x80)
1777	GEN_INT_ENTRY decrementer, virt=0
1778EXC_REAL_END(decrementer, 0x900, 0x80)
1779EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
1780	GEN_INT_ENTRY decrementer, virt=1
1781EXC_VIRT_END(decrementer, 0x4900, 0x80)
1782EXC_COMMON_BEGIN(decrementer_common)
1783	GEN_COMMON decrementer
1784	addi	r3,r1,STACK_FRAME_OVERHEAD
1785	bl	timer_interrupt
1786	b	interrupt_return_srr
1787
1788
1789/**
1790 * Interrupt 0x980 - Hypervisor Decrementer Interrupt.
1791 * This is an asynchronous interrupt, similar to 0x900 but for the HDEC
1792 * register.
1793 *
1794 * Handling:
1795 * Linux does not use this outside KVM where it's used to keep a host timer
1796 * while the guest is given control of DEC. It should normally be caught by
1797 * the KVM test and routed there.
1798 */
1799INT_DEFINE_BEGIN(hdecrementer)
1800	IVEC=0x980
1801	IHSRR=1
1802	ISTACK=0
1803	IKVM_REAL=1
1804	IKVM_VIRT=1
1805INT_DEFINE_END(hdecrementer)
1806
1807EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80)
1808	GEN_INT_ENTRY hdecrementer, virt=0
1809EXC_REAL_END(hdecrementer, 0x980, 0x80)
1810EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80)
1811	GEN_INT_ENTRY hdecrementer, virt=1
1812EXC_VIRT_END(hdecrementer, 0x4980, 0x80)
1813EXC_COMMON_BEGIN(hdecrementer_common)
1814	__GEN_COMMON_ENTRY hdecrementer
1815	/*
1816	 * Hypervisor decrementer interrupts not caught by the KVM test
1817	 * shouldn't occur but are sometimes left pending on exit from a KVM
1818	 * guest.  We don't need to do anything to clear them, as they are
1819	 * edge-triggered.
1820	 *
1821	 * Be careful to avoid touching the kernel stack.
1822	 */
1823	li	r10,0
1824	stb	r10,PACAHSRR_VALID(r13)
1825	ld	r10,PACA_EXGEN+EX_CTR(r13)
1826	mtctr	r10
1827	mtcrf	0x80,r9
1828	ld	r9,PACA_EXGEN+EX_R9(r13)
1829	ld	r10,PACA_EXGEN+EX_R10(r13)
1830	ld	r11,PACA_EXGEN+EX_R11(r13)
1831	ld	r12,PACA_EXGEN+EX_R12(r13)
1832	ld	r13,PACA_EXGEN+EX_R13(r13)
1833	HRFI_TO_KERNEL
1834
1835
1836/**
1837 * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt.
1838 * This is an asynchronous interrupt in response to a msgsndp doorbell.
1839 * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
1840 * IRQS_DISABLED mask (i.e., local_irq_disable()).
1841 *
1842 * Handling:
1843 * Guests may use this for IPIs between threads in a core if the
1844 * hypervisor supports it. NVGPRS are not saved (see 0x500).
1845 *
1846 * If soft masked, the masked handler will note the pending interrupt for
1847 * replay, leaving MSR[EE] enabled in the interrupted context because the
1848 * doorbells are edge triggered.
1849 */
1850INT_DEFINE_BEGIN(doorbell_super)
1851	IVEC=0xa00
1852	IMASK=IRQS_DISABLED
1853#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1854	IKVM_REAL=1
1855#endif
1856INT_DEFINE_END(doorbell_super)
1857
1858EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100)
1859	GEN_INT_ENTRY doorbell_super, virt=0
1860EXC_REAL_END(doorbell_super, 0xa00, 0x100)
1861EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100)
1862	GEN_INT_ENTRY doorbell_super, virt=1
1863EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
1864EXC_COMMON_BEGIN(doorbell_super_common)
1865	GEN_COMMON doorbell_super
1866	addi	r3,r1,STACK_FRAME_OVERHEAD
1867#ifdef CONFIG_PPC_DOORBELL
1868	bl	doorbell_exception
1869#else
1870	bl	unknown_async_exception
1871#endif
1872	b	interrupt_return_srr
1873
1874
1875EXC_REAL_NONE(0xb00, 0x100)
1876EXC_VIRT_NONE(0x4b00, 0x100)
1877
1878/**
1879 * Interrupt 0xc00 - System Call Interrupt (syscall, hcall).
1880 * This is a synchronous interrupt invoked with the "sc" instruction. The
1881 * system call is invoked with "sc 0" and does not alter the HV bit, so it
1882 * is directed to the currently running OS. The hypercall is invoked with
1883 * "sc 1" and it sets HV=1, so it elevates to hypervisor.
1884 *
1885 * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to
1886 * 0x4c00 virtual mode.
1887 *
1888 * Handling:
1889 * If the KVM test fires then it was due to a hypercall and is accordingly
1890 * routed to KVM. Otherwise this executes a normal Linux system call.
1891 *
1892 * Call convention:
1893 *
1894 * syscall and hypercalls register conventions are documented in
1895 * Documentation/powerpc/syscall64-abi.rst and
1896 * Documentation/powerpc/papr_hcalls.rst respectively.
 
 
 
 
 
 
 
 
 
 
 
 
 
1897 *
1898 * The intersection of volatile registers that don't contain possible
1899 * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry
1900 * without saving, though xer is not a good idea to use, as hardware may
1901 * interpret some bits so it may be costly to change them.
1902 */
1903INT_DEFINE_BEGIN(system_call)
1904	IVEC=0xc00
1905	IKVM_REAL=1
1906	IKVM_VIRT=1
1907INT_DEFINE_END(system_call)
1908
1909.macro SYSTEM_CALL virt
1910#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1911	/*
1912	 * There is a little bit of juggling to get syscall and hcall
1913	 * working well. Save r13 in ctr to avoid using SPRG scratch
1914	 * register.
1915	 *
1916	 * Userspace syscalls have already saved the PPR, hcalls must save
1917	 * it before setting HMT_MEDIUM.
1918	 */
1919	mtctr	r13
1920	GET_PACA(r13)
1921	std	r10,PACA_EXGEN+EX_R10(r13)
1922	INTERRUPT_TO_KERNEL
1923	KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */
1924	mfctr	r9
1925#else
1926	mr	r9,r13
1927	GET_PACA(r13)
1928	INTERRUPT_TO_KERNEL
1929#endif
1930
1931#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
1932BEGIN_FTR_SECTION
1933	cmpdi	r0,0x1ebe
1934	beq-	1f
1935END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
1936#endif
1937
1938	/* We reach here with PACA in r13, r13 in r9. */
1939	mfspr	r11,SPRN_SRR0
1940	mfspr	r12,SPRN_SRR1
1941
1942	HMT_MEDIUM
1943
1944	.if ! \virt
1945	__LOAD_HANDLER(r10, system_call_common_real)
1946	mtctr	r10
1947	bctr
 
 
 
1948	.else
 
 
1949#ifdef CONFIG_RELOCATABLE
1950	__LOAD_HANDLER(r10, system_call_common)
1951	mtctr	r10
1952	bctr
1953#else
1954	b	system_call_common
1955#endif
1956	.endif
1957
1958#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
1959	/* Fast LE/BE switch system call */
19601:	mfspr	r12,SPRN_SRR1
1961	xori	r12,r12,MSR_LE
1962	mtspr	SPRN_SRR1,r12
1963	mr	r13,r9
1964	RFI_TO_USER	/* return to userspace */
1965	b	.	/* prevent speculative execution */
1966#endif
1967.endm
1968
1969EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
1970	SYSTEM_CALL 0
1971EXC_REAL_END(system_call, 0xc00, 0x100)
1972EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
1973	SYSTEM_CALL 1
1974EXC_VIRT_END(system_call, 0x4c00, 0x100)
1975
1976#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
1977TRAMP_REAL_BEGIN(kvm_hcall)
1978	std	r9,PACA_EXGEN+EX_R9(r13)
1979	std	r11,PACA_EXGEN+EX_R11(r13)
1980	std	r12,PACA_EXGEN+EX_R12(r13)
1981	mfcr	r9
1982	mfctr	r10
1983	std	r10,PACA_EXGEN+EX_R13(r13)
1984	li	r10,0
1985	std	r10,PACA_EXGEN+EX_CFAR(r13)
1986	std	r10,PACA_EXGEN+EX_CTR(r13)
1987	 /*
1988	  * Save the PPR (on systems that support it) before changing to
1989	  * HMT_MEDIUM. That allows the KVM code to save that value into the
1990	  * guest state (it is the guest's PPR value).
1991	  */
1992BEGIN_FTR_SECTION
1993	mfspr	r10,SPRN_PPR
1994	std	r10,PACA_EXGEN+EX_PPR(r13)
1995END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1996
1997	HMT_MEDIUM
1998
1999#ifdef CONFIG_RELOCATABLE
2000	/*
2001	 * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives
2002	 * outside the head section.
2003	 */
2004	__LOAD_FAR_HANDLER(r10, kvmppc_hcall)
2005	mtctr   r10
2006	bctr
2007#else
2008	b       kvmppc_hcall
2009#endif
2010#endif
2011
2012/**
2013 * Interrupt 0xd00 - Trace Interrupt.
2014 * This is a synchronous interrupt in response to instruction step or
2015 * breakpoint faults.
2016 */
2017INT_DEFINE_BEGIN(single_step)
2018	IVEC=0xd00
2019#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2020	IKVM_REAL=1
2021#endif
2022INT_DEFINE_END(single_step)
2023
2024EXC_REAL_BEGIN(single_step, 0xd00, 0x100)
2025	GEN_INT_ENTRY single_step, virt=0
2026EXC_REAL_END(single_step, 0xd00, 0x100)
2027EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100)
2028	GEN_INT_ENTRY single_step, virt=1
2029EXC_VIRT_END(single_step, 0x4d00, 0x100)
2030EXC_COMMON_BEGIN(single_step_common)
2031	GEN_COMMON single_step
2032	addi	r3,r1,STACK_FRAME_OVERHEAD
2033	bl	single_step_exception
2034	b	interrupt_return_srr
2035
2036
2037/**
2038 * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI).
2039 * This is a synchronous interrupt in response to an MMU fault caused by a
2040 * guest data access.
2041 *
2042 * Handling:
2043 * This should always get routed to KVM. In radix MMU mode, this is caused
2044 * by a guest nested radix access that can't be performed due to the
2045 * partition scope page table. In hash mode, this can be caused by guests
2046 * running with translation disabled (virtual real mode) or with VPM enabled.
2047 * KVM will update the page table structures or disallow the access.
2048 */
2049INT_DEFINE_BEGIN(h_data_storage)
2050	IVEC=0xe00
2051	IHSRR=1
2052	IDAR=1
2053	IDSISR=1
2054	IKVM_REAL=1
2055	IKVM_VIRT=1
2056INT_DEFINE_END(h_data_storage)
2057
2058EXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20)
2059	GEN_INT_ENTRY h_data_storage, virt=0, ool=1
2060EXC_REAL_END(h_data_storage, 0xe00, 0x20)
2061EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20)
2062	GEN_INT_ENTRY h_data_storage, virt=1, ool=1
2063EXC_VIRT_END(h_data_storage, 0x4e00, 0x20)
 
2064EXC_COMMON_BEGIN(h_data_storage_common)
2065	GEN_COMMON h_data_storage
 
2066	addi    r3,r1,STACK_FRAME_OVERHEAD
2067BEGIN_MMU_FTR_SECTION
2068	bl      do_bad_page_fault_segv
 
 
2069MMU_FTR_SECTION_ELSE
2070	bl      unknown_exception
2071ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
2072	b       interrupt_return_hsrr
2073
2074
2075/**
2076 * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI).
2077 * This is a synchronous interrupt in response to an MMU fault caused by a
2078 * guest instruction fetch, similar to HDSI.
2079 */
2080INT_DEFINE_BEGIN(h_instr_storage)
2081	IVEC=0xe20
2082	IHSRR=1
2083	IKVM_REAL=1
2084	IKVM_VIRT=1
2085INT_DEFINE_END(h_instr_storage)
2086
2087EXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20)
2088	GEN_INT_ENTRY h_instr_storage, virt=0, ool=1
2089EXC_REAL_END(h_instr_storage, 0xe20, 0x20)
2090EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20)
2091	GEN_INT_ENTRY h_instr_storage, virt=1, ool=1
2092EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20)
2093EXC_COMMON_BEGIN(h_instr_storage_common)
2094	GEN_COMMON h_instr_storage
2095	addi	r3,r1,STACK_FRAME_OVERHEAD
2096	bl	unknown_exception
2097	b	interrupt_return_hsrr
2098
2099
2100/**
2101 * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt.
2102 */
2103INT_DEFINE_BEGIN(emulation_assist)
2104	IVEC=0xe40
2105	IHSRR=1
2106	IKVM_REAL=1
2107	IKVM_VIRT=1
2108INT_DEFINE_END(emulation_assist)
2109
2110EXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20)
2111	GEN_INT_ENTRY emulation_assist, virt=0, ool=1
2112EXC_REAL_END(emulation_assist, 0xe40, 0x20)
2113EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20)
2114	GEN_INT_ENTRY emulation_assist, virt=1, ool=1
2115EXC_VIRT_END(emulation_assist, 0x4e40, 0x20)
2116EXC_COMMON_BEGIN(emulation_assist_common)
2117	GEN_COMMON emulation_assist
2118	addi	r3,r1,STACK_FRAME_OVERHEAD
2119	bl	emulation_assist_interrupt
2120	REST_NVGPRS(r1) /* instruction emulation may change GPRs */
2121	b	interrupt_return_hsrr
2122
2123
2124/**
2125 * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI).
2126 * This is an asynchronous interrupt caused by a Hypervisor Maintenance
2127 * Exception. It is always taken in real mode but uses HSRR registers
2128 * unlike SRESET and MCE.
2129 *
2130 * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable
2131 * with IRQS_DISABLED mask (i.e., local_irq_disable()).
2132 *
2133 * Handling:
2134 * This is a special case, this is handled similarly to machine checks, with an
2135 * initial real mode handler that is not soft-masked, which attempts to fix the
2136 * problem. Then a regular handler which is soft-maskable and reports the
2137 * problem.
2138 *
2139 * The emergency stack is used for the early real mode handler.
2140 *
2141 * XXX: unclear why MCE and HMI schemes could not be made common, e.g.,
2142 * either use soft-masking for the MCE, or use irq_work for the HMI.
2143 *
2144 * KVM:
2145 * Unlike MCE, this calls into KVM without calling the real mode handler
2146 * first.
2147 */
2148INT_DEFINE_BEGIN(hmi_exception_early)
2149	IVEC=0xe60
2150	IHSRR=1
2151	IREALMODE_COMMON=1
2152	ISTACK=0
2153	IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
2154	IKVM_REAL=1
2155INT_DEFINE_END(hmi_exception_early)
2156
2157INT_DEFINE_BEGIN(hmi_exception)
2158	IVEC=0xe60
2159	IHSRR=1
2160	IMASK=IRQS_DISABLED
2161	IKVM_REAL=1
2162INT_DEFINE_END(hmi_exception)
2163
 
 
 
 
 
2164EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20)
2165	GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1
2166EXC_REAL_END(hmi_exception, 0xe60, 0x20)
2167EXC_VIRT_NONE(0x4e60, 0x20)
2168
2169EXC_COMMON_BEGIN(hmi_exception_early_common)
2170	__GEN_REALMODE_COMMON_ENTRY hmi_exception_early
2171
 
2172	mr	r10,r1			/* Save r1 */
2173	ld	r1,PACAEMERGSP(r13)	/* Use emergency stack for realmode */
2174	subi	r1,r1,INT_FRAME_SIZE	/* alloc stack frame		*/
2175
2176	__GEN_COMMON_BODY hmi_exception_early
 
2177
2178	addi	r3,r1,STACK_FRAME_OVERHEAD
2179	bl	hmi_exception_realmode
2180	cmpdi	cr0,r3,0
2181	bne	1f
2182
2183	EXCEPTION_RESTORE_REGS hsrr=1
2184	HRFI_TO_USER_OR_KERNEL
2185
21861:
2187	/*
2188	 * Go to virtual mode and pull the HMI event information from
2189	 * firmware.
2190	 */
2191	EXCEPTION_RESTORE_REGS hsrr=1
2192	GEN_INT_ENTRY hmi_exception, virt=0
2193
2194EXC_COMMON_BEGIN(hmi_exception_common)
2195	GEN_COMMON hmi_exception
 
 
 
2196	addi	r3,r1,STACK_FRAME_OVERHEAD
2197	bl	handle_hmi_exception
2198	b	interrupt_return_hsrr
2199
2200
2201/**
2202 * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt.
2203 * This is an asynchronous interrupt in response to a msgsnd doorbell.
2204 * Similar to the 0xa00 doorbell but for host rather than guest.
2205 */
2206INT_DEFINE_BEGIN(h_doorbell)
2207	IVEC=0xe80
2208	IHSRR=1
2209	IMASK=IRQS_DISABLED
2210	IKVM_REAL=1
2211	IKVM_VIRT=1
2212INT_DEFINE_END(h_doorbell)
2213
2214EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20)
2215	GEN_INT_ENTRY h_doorbell, virt=0, ool=1
2216EXC_REAL_END(h_doorbell, 0xe80, 0x20)
2217EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20)
2218	GEN_INT_ENTRY h_doorbell, virt=1, ool=1
2219EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
2220EXC_COMMON_BEGIN(h_doorbell_common)
2221	GEN_COMMON h_doorbell
2222	addi	r3,r1,STACK_FRAME_OVERHEAD
2223#ifdef CONFIG_PPC_DOORBELL
2224	bl	doorbell_exception
2225#else
2226	bl	unknown_async_exception
2227#endif
2228	b	interrupt_return_hsrr
2229
2230
2231/**
2232 * Interrupt 0xea0 - Hypervisor Virtualization Interrupt.
2233 * This is an asynchronous interrupt in response to an "external exception".
2234 * Similar to 0x500 but for host only.
2235 */
2236INT_DEFINE_BEGIN(h_virt_irq)
2237	IVEC=0xea0
2238	IHSRR=1
2239	IMASK=IRQS_DISABLED
2240	IKVM_REAL=1
2241	IKVM_VIRT=1
2242INT_DEFINE_END(h_virt_irq)
2243
2244EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20)
2245	GEN_INT_ENTRY h_virt_irq, virt=0, ool=1
2246EXC_REAL_END(h_virt_irq, 0xea0, 0x20)
2247EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20)
2248	GEN_INT_ENTRY h_virt_irq, virt=1, ool=1
2249EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
2250EXC_COMMON_BEGIN(h_virt_irq_common)
2251	GEN_COMMON h_virt_irq
2252	addi	r3,r1,STACK_FRAME_OVERHEAD
2253	bl	do_IRQ
2254	b	interrupt_return_hsrr
2255
2256
2257EXC_REAL_NONE(0xec0, 0x20)
2258EXC_VIRT_NONE(0x4ec0, 0x20)
2259EXC_REAL_NONE(0xee0, 0x20)
2260EXC_VIRT_NONE(0x4ee0, 0x20)
2261
2262
2263/*
2264 * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU).
2265 * This is an asynchronous interrupt in response to a PMU exception.
2266 * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
2267 * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()).
2268 *
2269 * Handling:
2270 * This calls into the perf subsystem.
2271 *
2272 * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it
2273 * runs under local_irq_disable. However it may be soft-masked in
2274 * powerpc-specific code.
2275 *
2276 * If soft masked, the masked handler will note the pending interrupt for
2277 * replay, and clear MSR[EE] in the interrupted context.
2278 */
2279INT_DEFINE_BEGIN(performance_monitor)
2280	IVEC=0xf00
2281	IMASK=IRQS_PMI_DISABLED
2282#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2283	IKVM_REAL=1
2284#endif
2285INT_DEFINE_END(performance_monitor)
2286
2287EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20)
2288	GEN_INT_ENTRY performance_monitor, virt=0, ool=1
2289EXC_REAL_END(performance_monitor, 0xf00, 0x20)
2290EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20)
2291	GEN_INT_ENTRY performance_monitor, virt=1, ool=1
2292EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
2293EXC_COMMON_BEGIN(performance_monitor_common)
2294	GEN_COMMON performance_monitor
2295	addi	r3,r1,STACK_FRAME_OVERHEAD
2296	bl	performance_monitor_exception
2297	b	interrupt_return_srr
2298
2299
2300/**
2301 * Interrupt 0xf20 - Vector Unavailable Interrupt.
2302 * This is a synchronous interrupt in response to
2303 * executing a vector (or altivec) instruction with MSR[VEC]=0.
2304 * Similar to FP unavailable.
2305 */
2306INT_DEFINE_BEGIN(altivec_unavailable)
2307	IVEC=0xf20
2308#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2309	IKVM_REAL=1
2310#endif
2311INT_DEFINE_END(altivec_unavailable)
2312
2313EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20)
2314	GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1
2315EXC_REAL_END(altivec_unavailable, 0xf20, 0x20)
2316EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20)
2317	GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1
2318EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20)
 
2319EXC_COMMON_BEGIN(altivec_unavailable_common)
2320	GEN_COMMON altivec_unavailable
2321#ifdef CONFIG_ALTIVEC
2322BEGIN_FTR_SECTION
2323	beq	1f
2324#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2325  BEGIN_FTR_SECTION_NESTED(69)
2326	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
2327	 * transaction), go do TM stuff
2328	 */
2329	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
2330	bne-	2f
2331  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
2332#endif
2333	bl	load_up_altivec
2334	b	fast_interrupt_return_srr
2335#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
23362:	/* User process was in a transaction */
 
 
2337	addi	r3,r1,STACK_FRAME_OVERHEAD
2338	bl	altivec_unavailable_tm
2339	b	interrupt_return_srr
2340#endif
23411:
2342END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2343#endif
 
 
2344	addi	r3,r1,STACK_FRAME_OVERHEAD
2345	bl	altivec_unavailable_exception
2346	b	interrupt_return_srr
2347
2348
2349/**
2350 * Interrupt 0xf40 - VSX Unavailable Interrupt.
2351 * This is a synchronous interrupt in response to
2352 * executing a VSX instruction with MSR[VSX]=0.
2353 * Similar to FP unavailable.
2354 */
2355INT_DEFINE_BEGIN(vsx_unavailable)
2356	IVEC=0xf40
2357#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2358	IKVM_REAL=1
2359#endif
2360INT_DEFINE_END(vsx_unavailable)
2361
2362EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20)
2363	GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1
2364EXC_REAL_END(vsx_unavailable, 0xf40, 0x20)
2365EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20)
2366	GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1
2367EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20)
 
2368EXC_COMMON_BEGIN(vsx_unavailable_common)
2369	GEN_COMMON vsx_unavailable
2370#ifdef CONFIG_VSX
2371BEGIN_FTR_SECTION
2372	beq	1f
2373#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2374  BEGIN_FTR_SECTION_NESTED(69)
2375	/* Test if 2 TM state bits are zero.  If non-zero (ie. userspace was in
2376	 * transaction), go do TM stuff
2377	 */
2378	rldicl.	r0, r12, (64-MSR_TS_LG), (64-2)
2379	bne-	2f
2380  END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
2381#endif
2382	b	load_up_vsx
2383#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
23842:	/* User process was in a transaction */
 
 
2385	addi	r3,r1,STACK_FRAME_OVERHEAD
2386	bl	vsx_unavailable_tm
2387	b	interrupt_return_srr
2388#endif
23891:
2390END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2391#endif
 
 
2392	addi	r3,r1,STACK_FRAME_OVERHEAD
2393	bl	vsx_unavailable_exception
2394	b	interrupt_return_srr
2395
2396
2397/**
2398 * Interrupt 0xf60 - Facility Unavailable Interrupt.
2399 * This is a synchronous interrupt in response to
2400 * executing an instruction without access to the facility that can be
2401 * resolved by the OS (e.g., FSCR, MSR).
2402 * Similar to FP unavailable.
2403 */
2404INT_DEFINE_BEGIN(facility_unavailable)
2405	IVEC=0xf60
2406#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2407	IKVM_REAL=1
2408#endif
2409INT_DEFINE_END(facility_unavailable)
2410
2411EXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20)
2412	GEN_INT_ENTRY facility_unavailable, virt=0, ool=1
2413EXC_REAL_END(facility_unavailable, 0xf60, 0x20)
2414EXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20)
2415	GEN_INT_ENTRY facility_unavailable, virt=1, ool=1
2416EXC_VIRT_END(facility_unavailable, 0x4f60, 0x20)
2417EXC_COMMON_BEGIN(facility_unavailable_common)
2418	GEN_COMMON facility_unavailable
2419	addi	r3,r1,STACK_FRAME_OVERHEAD
2420	bl	facility_unavailable_exception
2421	REST_NVGPRS(r1) /* instruction emulation may change GPRs */
2422	b	interrupt_return_srr
2423
2424
2425/**
2426 * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt.
2427 * This is a synchronous interrupt in response to
2428 * executing an instruction without access to the facility that can only
2429 * be resolved in HV mode (e.g., HFSCR).
2430 * Similar to FP unavailable.
2431 */
2432INT_DEFINE_BEGIN(h_facility_unavailable)
2433	IVEC=0xf80
2434	IHSRR=1
2435	IKVM_REAL=1
2436	IKVM_VIRT=1
2437INT_DEFINE_END(h_facility_unavailable)
2438
2439EXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20)
2440	GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1
2441EXC_REAL_END(h_facility_unavailable, 0xf80, 0x20)
2442EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20)
2443	GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1
2444EXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20)
2445EXC_COMMON_BEGIN(h_facility_unavailable_common)
2446	GEN_COMMON h_facility_unavailable
2447	addi	r3,r1,STACK_FRAME_OVERHEAD
2448	bl	facility_unavailable_exception
2449	REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
2450	b	interrupt_return_hsrr
2451
2452
2453EXC_REAL_NONE(0xfa0, 0x20)
2454EXC_VIRT_NONE(0x4fa0, 0x20)
2455EXC_REAL_NONE(0xfc0, 0x20)
2456EXC_VIRT_NONE(0x4fc0, 0x20)
2457EXC_REAL_NONE(0xfe0, 0x20)
2458EXC_VIRT_NONE(0x4fe0, 0x20)
2459
2460EXC_REAL_NONE(0x1000, 0x100)
2461EXC_VIRT_NONE(0x5000, 0x100)
2462EXC_REAL_NONE(0x1100, 0x100)
2463EXC_VIRT_NONE(0x5100, 0x100)
2464
2465#ifdef CONFIG_CBE_RAS
2466INT_DEFINE_BEGIN(cbe_system_error)
2467	IVEC=0x1200
2468	IHSRR=1
2469INT_DEFINE_END(cbe_system_error)
2470
2471EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)
2472	GEN_INT_ENTRY cbe_system_error, virt=0
2473EXC_REAL_END(cbe_system_error, 0x1200, 0x100)
2474EXC_VIRT_NONE(0x5200, 0x100)
2475EXC_COMMON_BEGIN(cbe_system_error_common)
2476	GEN_COMMON cbe_system_error
2477	addi	r3,r1,STACK_FRAME_OVERHEAD
2478	bl	cbe_system_error_exception
2479	b	interrupt_return_hsrr
2480
2481#else /* CONFIG_CBE_RAS */
2482EXC_REAL_NONE(0x1200, 0x100)
2483EXC_VIRT_NONE(0x5200, 0x100)
2484#endif
2485
2486/**
2487 * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt.
2488 * This has been removed from the ISA before 2.01, which is the earliest
2489 * 64-bit BookS ISA supported, however the G5 / 970 implements this
2490 * interrupt with a non-architected feature available through the support
2491 * processor interface.
2492 */
2493INT_DEFINE_BEGIN(instruction_breakpoint)
2494	IVEC=0x1300
2495#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2496	IKVM_REAL=1
2497#endif
2498INT_DEFINE_END(instruction_breakpoint)
2499
2500EXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100)
2501	GEN_INT_ENTRY instruction_breakpoint, virt=0
2502EXC_REAL_END(instruction_breakpoint, 0x1300, 0x100)
2503EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100)
2504	GEN_INT_ENTRY instruction_breakpoint, virt=1
2505EXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100)
2506EXC_COMMON_BEGIN(instruction_breakpoint_common)
2507	GEN_COMMON instruction_breakpoint
2508	addi	r3,r1,STACK_FRAME_OVERHEAD
2509	bl	instruction_breakpoint_exception
2510	b	interrupt_return_srr
2511
2512
2513EXC_REAL_NONE(0x1400, 0x100)
2514EXC_VIRT_NONE(0x5400, 0x100)
2515
2516/**
2517 * Interrupt 0x1500 - Soft Patch Interrupt
2518 *
2519 * Handling:
2520 * This is an implementation specific interrupt which can be used for a
2521 * range of exceptions.
2522 *
2523 * This interrupt handler is unique in that it runs the denormal assist
2524 * code even for guests (and even in guest context) without going to KVM,
2525 * for speed. POWER9 does not raise denorm exceptions, so this special case
2526 * could be phased out in future to reduce special cases.
2527 */
2528INT_DEFINE_BEGIN(denorm_exception)
2529	IVEC=0x1500
2530	IHSRR=1
2531	IBRANCH_TO_COMMON=0
2532	IKVM_REAL=1
2533INT_DEFINE_END(denorm_exception)
2534
2535EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100)
2536	GEN_INT_ENTRY denorm_exception, virt=0
2537#ifdef CONFIG_PPC_DENORMALISATION
2538	andis.	r10,r12,(HSRR1_DENORM)@h /* denorm? */
 
2539	bne+	denorm_assist
2540#endif
2541	GEN_BRANCH_TO_COMMON denorm_exception, virt=0
2542EXC_REAL_END(denorm_exception, 0x1500, 0x100)
 
 
2543#ifdef CONFIG_PPC_DENORMALISATION
2544EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100)
2545	GEN_INT_ENTRY denorm_exception, virt=1
2546	andis.	r10,r12,(HSRR1_DENORM)@h /* denorm? */
 
2547	bne+	denorm_assist
2548	GEN_BRANCH_TO_COMMON denorm_exception, virt=1
2549EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
2550#else
2551EXC_VIRT_NONE(0x5500, 0x100)
2552#endif
2553
 
 
2554#ifdef CONFIG_PPC_DENORMALISATION
2555TRAMP_REAL_BEGIN(denorm_assist)
2556BEGIN_FTR_SECTION
2557/*
2558 * To denormalise we need to move a copy of the register to itself.
2559 * For POWER6 do that here for all FP regs.
2560 */
2561	mfmsr	r10
2562	ori	r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
2563	xori	r10,r10,(MSR_FE0|MSR_FE1)
2564	mtmsrd	r10
2565	sync
2566
2567	.Lreg=0
2568	.rept 32
2569	fmr	.Lreg,.Lreg
2570	.Lreg=.Lreg+1
2571	.endr
2572
2573FTR_SECTION_ELSE
2574/*
2575 * To denormalise we need to move a copy of the register to itself.
2576 * For POWER7 do that here for the first 32 VSX registers only.
2577 */
2578	mfmsr	r10
2579	oris	r10,r10,MSR_VSX@h
2580	mtmsrd	r10
2581	sync
2582
2583	.Lreg=0
2584	.rept 32
2585	XVCPSGNDP(.Lreg,.Lreg,.Lreg)
2586	.Lreg=.Lreg+1
2587	.endr
2588
2589ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
2590
2591BEGIN_FTR_SECTION
2592	b	denorm_done
2593END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
2594/*
2595 * To denormalise we need to move a copy of the register to itself.
2596 * For POWER8 we need to do that for all 64 VSX registers
2597 */
2598	.Lreg=32
2599	.rept 32
2600	XVCPSGNDP(.Lreg,.Lreg,.Lreg)
2601	.Lreg=.Lreg+1
2602	.endr
2603
2604denorm_done:
2605	mfspr	r11,SPRN_HSRR0
2606	subi	r11,r11,4
2607	mtspr	SPRN_HSRR0,r11
2608	mtcrf	0x80,r9
2609	ld	r9,PACA_EXGEN+EX_R9(r13)
2610BEGIN_FTR_SECTION
2611	ld	r10,PACA_EXGEN+EX_PPR(r13)
2612	mtspr	SPRN_PPR,r10
2613END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
2614BEGIN_FTR_SECTION
2615	ld	r10,PACA_EXGEN+EX_CFAR(r13)
2616	mtspr	SPRN_CFAR,r10
2617END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
2618	li	r10,0
2619	stb	r10,PACAHSRR_VALID(r13)
2620	ld	r10,PACA_EXGEN+EX_R10(r13)
2621	ld	r11,PACA_EXGEN+EX_R11(r13)
2622	ld	r12,PACA_EXGEN+EX_R12(r13)
2623	ld	r13,PACA_EXGEN+EX_R13(r13)
2624	HRFI_TO_UNKNOWN
2625	b	.
2626#endif
2627
2628EXC_COMMON_BEGIN(denorm_exception_common)
2629	GEN_COMMON denorm_exception
2630	addi	r3,r1,STACK_FRAME_OVERHEAD
2631	bl	unknown_exception
2632	b	interrupt_return_hsrr
2633
2634
2635#ifdef CONFIG_CBE_RAS
2636INT_DEFINE_BEGIN(cbe_maintenance)
2637	IVEC=0x1600
2638	IHSRR=1
2639INT_DEFINE_END(cbe_maintenance)
2640
2641EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)
2642	GEN_INT_ENTRY cbe_maintenance, virt=0
2643EXC_REAL_END(cbe_maintenance, 0x1600, 0x100)
2644EXC_VIRT_NONE(0x5600, 0x100)
2645EXC_COMMON_BEGIN(cbe_maintenance_common)
2646	GEN_COMMON cbe_maintenance
2647	addi	r3,r1,STACK_FRAME_OVERHEAD
2648	bl	cbe_maintenance_exception
2649	b	interrupt_return_hsrr
2650
2651#else /* CONFIG_CBE_RAS */
2652EXC_REAL_NONE(0x1600, 0x100)
2653EXC_VIRT_NONE(0x5600, 0x100)
2654#endif
2655
2656
2657INT_DEFINE_BEGIN(altivec_assist)
2658	IVEC=0x1700
2659#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2660	IKVM_REAL=1
2661#endif
2662INT_DEFINE_END(altivec_assist)
2663
2664EXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100)
2665	GEN_INT_ENTRY altivec_assist, virt=0
2666EXC_REAL_END(altivec_assist, 0x1700, 0x100)
2667EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100)
2668	GEN_INT_ENTRY altivec_assist, virt=1
2669EXC_VIRT_END(altivec_assist, 0x5700, 0x100)
2670EXC_COMMON_BEGIN(altivec_assist_common)
2671	GEN_COMMON altivec_assist
2672	addi	r3,r1,STACK_FRAME_OVERHEAD
2673#ifdef CONFIG_ALTIVEC
2674	bl	altivec_assist_exception
2675	REST_NVGPRS(r1) /* instruction emulation may change GPRs */
2676#else
2677	bl	unknown_exception
2678#endif
2679	b	interrupt_return_srr
2680
2681
2682#ifdef CONFIG_CBE_RAS
2683INT_DEFINE_BEGIN(cbe_thermal)
2684	IVEC=0x1800
2685	IHSRR=1
2686INT_DEFINE_END(cbe_thermal)
2687
2688EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
2689	GEN_INT_ENTRY cbe_thermal, virt=0
2690EXC_REAL_END(cbe_thermal, 0x1800, 0x100)
2691EXC_VIRT_NONE(0x5800, 0x100)
2692EXC_COMMON_BEGIN(cbe_thermal_common)
2693	GEN_COMMON cbe_thermal
2694	addi	r3,r1,STACK_FRAME_OVERHEAD
2695	bl	cbe_thermal_exception
2696	b	interrupt_return_hsrr
2697
2698#else /* CONFIG_CBE_RAS */
2699EXC_REAL_NONE(0x1800, 0x100)
2700EXC_VIRT_NONE(0x5800, 0x100)
2701#endif
2702
2703
2704#ifdef CONFIG_PPC_WATCHDOG
2705
2706INT_DEFINE_BEGIN(soft_nmi)
2707	IVEC=0x900
2708	ISTACK=0
2709INT_DEFINE_END(soft_nmi)
 
 
 
 
2710
2711/*
2712 * Branch to soft_nmi_interrupt using the emergency stack. The emergency
2713 * stack is one that is usable by maskable interrupts so long as MSR_EE
2714 * remains off. It is used for recovery when something has corrupted the
2715 * normal kernel stack, for example. The "soft NMI" must not use the process
2716 * stack because we want irq disabled sections to avoid touching the stack
2717 * at all (other than PMU interrupts), so use the emergency stack for this,
2718 * and run it entirely with interrupts hard disabled.
2719 */
2720EXC_COMMON_BEGIN(soft_nmi_common)
2721	mr	r10,r1
2722	ld	r1,PACAEMERGSP(r13)
2723	subi	r1,r1,INT_FRAME_SIZE
2724	__GEN_COMMON_BODY soft_nmi
2725
2726	addi	r3,r1,STACK_FRAME_OVERHEAD
2727	bl	soft_nmi_interrupt
 
2728
2729	/* Clear MSR_RI before setting SRR0 and SRR1. */
2730	li	r9,0
2731	mtmsrd	r9,1
2732
2733	kuap_kernel_restore r9, r10
2734
2735	EXCEPTION_RESTORE_REGS hsrr=0
2736	RFI_TO_KERNEL
2737
2738#endif /* CONFIG_PPC_WATCHDOG */
2739
2740/*
2741 * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
2742 * - If it was a decrementer interrupt, we bump the dec to max and and return.
2743 * - If it was a doorbell we return immediately since doorbells are edge
2744 *   triggered and won't automatically refire.
2745 * - If it was a HMI we return immediately since we handled it in realmode
2746 *   and it won't refire.
2747 * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
2748 * This is called with r10 containing the value to OR to the paca field.
2749 */
2750.macro MASKED_INTERRUPT hsrr=0
2751	.if \hsrr
2752masked_Hinterrupt:
2753	.else
2754masked_interrupt:
2755	.endif
2756	stw	r9,PACA_EXGEN+EX_CCR(r13)
2757	lbz	r9,PACAIRQHAPPENED(r13)
2758	or	r9,r9,r10
2759	stb	r9,PACAIRQHAPPENED(r13)
2760
2761	.if ! \hsrr
2762	cmpwi	r10,PACA_IRQ_DEC
2763	bne	1f
2764	LOAD_REG_IMMEDIATE(r9, 0x7fffffff)
2765	mtspr	SPRN_DEC,r9
2766#ifdef CONFIG_PPC_WATCHDOG
2767	lwz	r9,PACA_EXGEN+EX_CCR(r13)
2768	b	soft_nmi_common
2769#else
2770	b	2f
2771#endif
2772	.endif
2773
27741:	andi.	r10,r10,PACA_IRQ_MUST_HARD_MASK
2775	beq	2f
2776	xori	r12,r12,MSR_EE	/* clear MSR_EE */
2777	.if \hsrr
2778	mtspr	SPRN_HSRR1,r12
 
 
2779	.else
2780	mtspr	SPRN_SRR1,r12
 
 
2781	.endif
2782	ori	r9,r9,PACA_IRQ_HARD_DIS
2783	stb	r9,PACAIRQHAPPENED(r13)
27842:	/* done */
2785	li	r9,0
2786	.if \hsrr
2787	stb	r9,PACAHSRR_VALID(r13)
2788	.else
2789	stb	r9,PACASRR_VALID(r13)
2790	.endif
2791
2792	SEARCH_RESTART_TABLE
2793	cmpdi	r12,0
2794	beq	3f
2795	.if \hsrr
2796	mtspr	SPRN_HSRR0,r12
2797	.else
2798	mtspr	SPRN_SRR0,r12
2799	.endif
28003:
2801
2802	ld	r9,PACA_EXGEN+EX_CTR(r13)
2803	mtctr	r9
2804	lwz	r9,PACA_EXGEN+EX_CCR(r13)
2805	mtcrf	0x80,r9
2806	std	r1,PACAR1(r13)
2807	ld	r9,PACA_EXGEN+EX_R9(r13)
2808	ld	r10,PACA_EXGEN+EX_R10(r13)
2809	ld	r11,PACA_EXGEN+EX_R11(r13)
2810	ld	r12,PACA_EXGEN+EX_R12(r13)
2811	ld	r13,PACA_EXGEN+EX_R13(r13)
2812	/* May return to masked low address where r13 is not set up */
2813	.if \hsrr
2814	HRFI_TO_KERNEL
2815	.else
2816	RFI_TO_KERNEL
2817	.endif
2818	b	.
 
2819.endm
2820
2821TRAMP_REAL_BEGIN(stf_barrier_fallback)
2822	std	r9,PACA_EXRFI+EX_R9(r13)
2823	std	r10,PACA_EXRFI+EX_R10(r13)
2824	sync
2825	ld	r9,PACA_EXRFI+EX_R9(r13)
2826	ld	r10,PACA_EXRFI+EX_R10(r13)
2827	ori	31,31,0
2828	.rept 14
2829	b	1f
28301:
2831	.endr
2832	blr
2833
2834/* Clobbers r10, r11, ctr */
2835.macro L1D_DISPLACEMENT_FLUSH
 
 
 
 
 
 
 
2836	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
2837	ld	r11,PACA_L1D_FLUSH_SIZE(r13)
2838	srdi	r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
2839	mtctr	r11
2840	DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
2841
2842	/* order ld/st prior to dcbt stop all streams with flushing */
2843	sync
2844
2845	/*
2846	 * The load addresses are at staggered offsets within cachelines,
2847	 * which suits some pipelines better (on others it should not
2848	 * hurt).
2849	 */
28501:
2851	ld	r11,(0x80 + 8)*0(r10)
2852	ld	r11,(0x80 + 8)*1(r10)
2853	ld	r11,(0x80 + 8)*2(r10)
2854	ld	r11,(0x80 + 8)*3(r10)
2855	ld	r11,(0x80 + 8)*4(r10)
2856	ld	r11,(0x80 + 8)*5(r10)
2857	ld	r11,(0x80 + 8)*6(r10)
2858	ld	r11,(0x80 + 8)*7(r10)
2859	addi	r10,r10,0x80*8
2860	bdnz	1b
2861.endm
2862
2863TRAMP_REAL_BEGIN(entry_flush_fallback)
2864	std	r9,PACA_EXRFI+EX_R9(r13)
2865	std	r10,PACA_EXRFI+EX_R10(r13)
2866	std	r11,PACA_EXRFI+EX_R11(r13)
2867	mfctr	r9
2868	L1D_DISPLACEMENT_FLUSH
2869	mtctr	r9
2870	ld	r9,PACA_EXRFI+EX_R9(r13)
2871	ld	r10,PACA_EXRFI+EX_R10(r13)
2872	ld	r11,PACA_EXRFI+EX_R11(r13)
2873	blr
2874
2875/*
2876 * The SCV entry flush happens with interrupts enabled, so it must disable
2877 * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10
2878 * (containing LR) does not need to be preserved here because scv entry
2879 * puts 0 in the pt_regs, CTR can be clobbered for the same reason.
2880 */
2881TRAMP_REAL_BEGIN(scv_entry_flush_fallback)
2882	li	r10,0
2883	mtmsrd	r10,1
2884	lbz	r10,PACAIRQHAPPENED(r13)
2885	ori	r10,r10,PACA_IRQ_HARD_DIS
2886	stb	r10,PACAIRQHAPPENED(r13)
2887	std	r11,PACA_EXRFI+EX_R11(r13)
2888	L1D_DISPLACEMENT_FLUSH
2889	ld	r11,PACA_EXRFI+EX_R11(r13)
2890	li	r10,MSR_RI
2891	mtmsrd	r10,1
2892	blr
2893
2894TRAMP_REAL_BEGIN(rfi_flush_fallback)
2895	SET_SCRATCH0(r13);
2896	GET_PACA(r13);
2897	std	r1,PACA_EXRFI+EX_R12(r13)
2898	ld	r1,PACAKSAVE(r13)
2899	std	r9,PACA_EXRFI+EX_R9(r13)
2900	std	r10,PACA_EXRFI+EX_R10(r13)
2901	std	r11,PACA_EXRFI+EX_R11(r13)
2902	mfctr	r9
2903	L1D_DISPLACEMENT_FLUSH
2904	mtctr	r9
2905	ld	r9,PACA_EXRFI+EX_R9(r13)
2906	ld	r10,PACA_EXRFI+EX_R10(r13)
2907	ld	r11,PACA_EXRFI+EX_R11(r13)
2908	ld	r1,PACA_EXRFI+EX_R12(r13)
2909	GET_SCRATCH0(r13);
2910	rfid
2911
2912TRAMP_REAL_BEGIN(hrfi_flush_fallback)
2913	SET_SCRATCH0(r13);
2914	GET_PACA(r13);
2915	std	r1,PACA_EXRFI+EX_R12(r13)
2916	ld	r1,PACAKSAVE(r13)
2917	std	r9,PACA_EXRFI+EX_R9(r13)
2918	std	r10,PACA_EXRFI+EX_R10(r13)
2919	std	r11,PACA_EXRFI+EX_R11(r13)
2920	mfctr	r9
2921	L1D_DISPLACEMENT_FLUSH
2922	mtctr	r9
2923	ld	r9,PACA_EXRFI+EX_R9(r13)
2924	ld	r10,PACA_EXRFI+EX_R10(r13)
2925	ld	r11,PACA_EXRFI+EX_R11(r13)
2926	ld	r1,PACA_EXRFI+EX_R12(r13)
2927	GET_SCRATCH0(r13);
2928	hrfid
2929
2930TRAMP_REAL_BEGIN(rfscv_flush_fallback)
2931	/* system call volatile */
2932	mr	r7,r13
2933	GET_PACA(r13);
2934	mr	r8,r1
2935	ld	r1,PACAKSAVE(r13)
2936	mfctr	r9
2937	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
2938	ld	r11,PACA_L1D_FLUSH_SIZE(r13)
2939	srdi	r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
2940	mtctr	r11
2941	DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
2942
2943	/* order ld/st prior to dcbt stop all streams with flushing */
2944	sync
2945
2946	/*
2947	 * The load adresses are at staggered offsets within cachelines,
2948	 * which suits some pipelines better (on others it should not
2949	 * hurt).
2950	 */
29511:
2952	ld	r11,(0x80 + 8)*0(r10)
2953	ld	r11,(0x80 + 8)*1(r10)
2954	ld	r11,(0x80 + 8)*2(r10)
2955	ld	r11,(0x80 + 8)*3(r10)
2956	ld	r11,(0x80 + 8)*4(r10)
2957	ld	r11,(0x80 + 8)*5(r10)
2958	ld	r11,(0x80 + 8)*6(r10)
2959	ld	r11,(0x80 + 8)*7(r10)
2960	addi	r10,r10,0x80*8
2961	bdnz	1b
2962
2963	mtctr	r9
2964	li	r9,0
2965	li	r10,0
2966	li	r11,0
2967	mr	r1,r8
2968	mr	r13,r7
2969	RFSCV
2970
2971USE_TEXT_SECTION()
 
 
 
 
 
 
 
2972
2973#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2974kvm_interrupt:
2975	/*
2976	 * The conditional branch in KVMTEST can't reach all the way,
2977	 * make a stub.
2978	 */
2979	b	kvmppc_interrupt
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2980#endif
2981
2982_GLOBAL(do_uaccess_flush)
2983	UACCESS_FLUSH_FIXUP_SECTION
2984	nop
2985	nop
2986	nop
2987	blr
2988	L1D_DISPLACEMENT_FLUSH
2989	blr
2990_ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
2991EXPORT_SYMBOL(do_uaccess_flush)
2992
 
2993
2994MASKED_INTERRUPT
2995MASKED_INTERRUPT hsrr=1
2996
2997	/*
2998	 * Relocation-on interrupts: A subset of the interrupts can be delivered
2999	 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
3000	 * it.  Addresses are the same as the original interrupt addresses, but
3001	 * offset by 0xc000000000004000.
3002	 * It's impossible to receive interrupts below 0x300 via this mechanism.
3003	 * KVM: None of these traps are from the guest ; anything that escalated
3004	 * to HV=1 from HV=0 is delivered via real mode handlers.
3005	 */
3006
3007	/*
3008	 * This uses the standard macro, since the original 0x300 vector
3009	 * only has extra guff for STAB-based processors -- which never
3010	 * come here.
3011	 */
3012
 
 
 
3013USE_FIXED_SECTION(virt_trampolines)
3014	/*
3015	 * All code below __end_soft_masked is treated as soft-masked. If
3016	 * any code runs here with MSR[EE]=1, it must then cope with pending
3017	 * soft interrupt being raised (i.e., by ensuring it is replayed).
3018	 *
3019	 * The __end_interrupts marker must be past the out-of-line (OOL)
3020	 * handlers, so that they are copied to real address 0x100 when running
3021	 * a relocatable kernel. This ensures they can be reached from the short
3022	 * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
3023	 * directly, without using LOAD_HANDLER().
3024	 */
3025	.align	7
3026	.globl	__end_interrupts
3027__end_interrupts:
3028DEFINE_FIXED_SYMBOL(__end_interrupts)
3029
 
 
 
 
 
 
 
 
 
3030CLOSE_FIXED_SECTION(real_vectors);
3031CLOSE_FIXED_SECTION(real_trampolines);
3032CLOSE_FIXED_SECTION(virt_vectors);
3033CLOSE_FIXED_SECTION(virt_trampolines);
3034
3035USE_TEXT_SECTION()
3036
3037/* MSR[RI] should be clear because this uses SRR[01] */
3038enable_machine_check:
3039	mflr	r0
3040	bcl	20,31,$+4
30410:	mflr	r3
3042	addi	r3,r3,(1f - 0b)
3043	mtspr	SPRN_SRR0,r3
3044	mfmsr	r3
3045	ori	r3,r3,MSR_ME
3046	mtspr	SPRN_SRR1,r3
3047	RFI_TO_KERNEL
30481:	mtlr	r0
3049	blr
3050
3051/* MSR[RI] should be clear because this uses SRR[01] */
3052disable_machine_check:
3053	mflr	r0
3054	bcl	20,31,$+4
30550:	mflr	r3
3056	addi	r3,r3,(1f - 0b)
3057	mtspr	SPRN_SRR0,r3
3058	mfmsr	r3
3059	li	r4,MSR_ME
3060	andc	r3,r3,r4
3061	mtspr	SPRN_SRR1,r3
3062	RFI_TO_KERNEL
30631:	mtlr	r0
3064	blr