1/*
   2 *  Boot code and exception vectors for Book3E processors
   3 *
   4 *  Copyright (C) 2007 Ben. Herrenschmidt (benh@kernel.crashing.org), IBM Corp.
   5 *
   6 *  This program is free software; you can redistribute it and/or
   7 *  modify it under the terms of the GNU General Public License
   8 *  as published by the Free Software Foundation; either version
   9 *  2 of the License, or (at your option) any later version.
  10 */
  11
  12#include <linux/threads.h>
  13#include <asm/reg.h>
  14#include <asm/page.h>
  15#include <asm/ppc_asm.h>
  16#include <asm/asm-offsets.h>
  17#include <asm/cputable.h>
  18#include <asm/setup.h>
  19#include <asm/thread_info.h>
  20#include <asm/reg_a2.h>
  21#include <asm/exception-64e.h>
  22#include <asm/bug.h>
  23#include <asm/irqflags.h>
  24#include <asm/ptrace.h>
  25#include <asm/ppc-opcode.h>
  26#include <asm/mmu.h>
  27#include <asm/hw_irq.h>
  28#include <asm/kvm_asm.h>
  29#include <asm/kvm_booke_hv_asm.h>
  30
  31/* XXX This will ultimately add space for a special exception save
  32 *     structure used to save things like SRR0/SRR1, SPRGs, MAS, etc...
  33 *     when taking special interrupts. For now we don't support that,
  34 *     special interrupts from within a non-standard level will probably
  35 *     blow you up
  36 */
  37#define SPECIAL_EXC_SRR0	0
  38#define SPECIAL_EXC_SRR1	1
  39#define SPECIAL_EXC_SPRG_GEN	2
  40#define SPECIAL_EXC_SPRG_TLB	3
  41#define SPECIAL_EXC_MAS0	4
  42#define SPECIAL_EXC_MAS1	5
  43#define SPECIAL_EXC_MAS2	6
  44#define SPECIAL_EXC_MAS3	7
  45#define SPECIAL_EXC_MAS6	8
  46#define SPECIAL_EXC_MAS7	9
  47#define SPECIAL_EXC_MAS5	10	/* E.HV only */
  48#define SPECIAL_EXC_MAS8	11	/* E.HV only */
  49#define SPECIAL_EXC_IRQHAPPENED	12
  50#define SPECIAL_EXC_DEAR	13
  51#define SPECIAL_EXC_ESR		14
  52#define SPECIAL_EXC_SOFTE	15
  53#define SPECIAL_EXC_CSRR0	16
  54#define SPECIAL_EXC_CSRR1	17
  55/* must be even to keep 16-byte stack alignment */
  56#define SPECIAL_EXC_END		18
  57
  58#define SPECIAL_EXC_FRAME_SIZE	(INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
  59#define SPECIAL_EXC_FRAME_OFFS  (INT_FRAME_SIZE - 288)
  60
  61#define SPECIAL_EXC_STORE(reg, name) \
  62	std	reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
  63
  64#define SPECIAL_EXC_LOAD(reg, name) \
  65	ld	reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
  66
  67special_reg_save:
  68	lbz	r9,PACAIRQHAPPENED(r13)
  69	RECONCILE_IRQ_STATE(r3,r4)
  70
  71	/*
  72	 * We only need (or have stack space) to save this stuff if
  73	 * we interrupted the kernel.
  74	 */
  75	ld	r3,_MSR(r1)
  76	andi.	r3,r3,MSR_PR
  77	bnelr
  78
  79	/* Copy info into temporary exception thread info */
  80	ld	r11,PACAKSAVE(r13)
  81	CURRENT_THREAD_INFO(r11, r11)
  82	CURRENT_THREAD_INFO(r12, r1)
  83	ld	r10,TI_FLAGS(r11)
  84	std	r10,TI_FLAGS(r12)
  85	ld	r10,TI_PREEMPT(r11)
  86	std	r10,TI_PREEMPT(r12)
  87	ld	r10,TI_TASK(r11)
  88	std	r10,TI_TASK(r12)
  89
  90	/*
  91	 * Advance to the next TLB exception frame for handler
  92	 * types that don't do it automatically.
  93	 */
  94	LOAD_REG_ADDR(r11,extlb_level_exc)
  95	lwz	r12,0(r11)
  96	mfspr	r10,SPRN_SPRG_TLB_EXFRAME
  97	add	r10,r10,r12
  98	mtspr	SPRN_SPRG_TLB_EXFRAME,r10
  99
 100	/*
 101	 * Save registers needed to allow nesting of certain exceptions
 102	 * (such as TLB misses) inside special exception levels
 103	 */
 104	mfspr	r10,SPRN_SRR0
 105	SPECIAL_EXC_STORE(r10,SRR0)
 106	mfspr	r10,SPRN_SRR1
 107	SPECIAL_EXC_STORE(r10,SRR1)
 108	mfspr	r10,SPRN_SPRG_GEN_SCRATCH
 109	SPECIAL_EXC_STORE(r10,SPRG_GEN)
 110	mfspr	r10,SPRN_SPRG_TLB_SCRATCH
 111	SPECIAL_EXC_STORE(r10,SPRG_TLB)
 112	mfspr	r10,SPRN_MAS0
 113	SPECIAL_EXC_STORE(r10,MAS0)
 114	mfspr	r10,SPRN_MAS1
 115	SPECIAL_EXC_STORE(r10,MAS1)
 116	mfspr	r10,SPRN_MAS2
 117	SPECIAL_EXC_STORE(r10,MAS2)
 118	mfspr	r10,SPRN_MAS3
 119	SPECIAL_EXC_STORE(r10,MAS3)
 120	mfspr	r10,SPRN_MAS6
 121	SPECIAL_EXC_STORE(r10,MAS6)
 122	mfspr	r10,SPRN_MAS7
 123	SPECIAL_EXC_STORE(r10,MAS7)
 124BEGIN_FTR_SECTION
 125	mfspr	r10,SPRN_MAS5
 126	SPECIAL_EXC_STORE(r10,MAS5)
 127	mfspr	r10,SPRN_MAS8
 128	SPECIAL_EXC_STORE(r10,MAS8)
 129
 130	/* MAS5/8 could have inappropriate values if we interrupted KVM code */
 131	li	r10,0
 132	mtspr	SPRN_MAS5,r10
 133	mtspr	SPRN_MAS8,r10
 134END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
 135	SPECIAL_EXC_STORE(r9,IRQHAPPENED)
 136
 137	mfspr	r10,SPRN_DEAR
 138	SPECIAL_EXC_STORE(r10,DEAR)
 139	mfspr	r10,SPRN_ESR
 140	SPECIAL_EXC_STORE(r10,ESR)
 141
 142	lbz	r10,PACAIRQSOFTMASK(r13)
 143	SPECIAL_EXC_STORE(r10,SOFTE)
 144	ld	r10,_NIP(r1)
 145	SPECIAL_EXC_STORE(r10,CSRR0)
 146	ld	r10,_MSR(r1)
 147	SPECIAL_EXC_STORE(r10,CSRR1)
 148
 149	blr
 150
 151ret_from_level_except:
 152	ld	r3,_MSR(r1)
 153	andi.	r3,r3,MSR_PR
 154	beq	1f
 155	b	ret_from_except
 1561:
 157
 158	LOAD_REG_ADDR(r11,extlb_level_exc)
 159	lwz	r12,0(r11)
 160	mfspr	r10,SPRN_SPRG_TLB_EXFRAME
 161	sub	r10,r10,r12
 162	mtspr	SPRN_SPRG_TLB_EXFRAME,r10
 163
 164	/*
 165	 * It's possible that the special level exception interrupted a
 166	 * TLB miss handler, and inserted the same entry that the
 167	 * interrupted handler was about to insert.  On CPUs without TLB
 168	 * write conditional, this can result in a duplicate TLB entry.
 169	 * Wipe all non-bolted entries to be safe.
 170	 *
 171	 * Note that this doesn't protect against any TLB misses
 172	 * we may take accessing the stack from here to the end of
 173	 * the special level exception.  It's not clear how we can
 174	 * reasonably protect against that, but only CPUs with
 175	 * neither TLB write conditional nor bolted kernel memory
 176	 * are affected.  Do any such CPUs even exist?
 177	 */
 178	PPC_TLBILX_ALL(0,R0)
 179
 180	REST_NVGPRS(r1)
 181
 182	SPECIAL_EXC_LOAD(r10,SRR0)
 183	mtspr	SPRN_SRR0,r10
 184	SPECIAL_EXC_LOAD(r10,SRR1)
 185	mtspr	SPRN_SRR1,r10
 186	SPECIAL_EXC_LOAD(r10,SPRG_GEN)
 187	mtspr	SPRN_SPRG_GEN_SCRATCH,r10
 188	SPECIAL_EXC_LOAD(r10,SPRG_TLB)
 189	mtspr	SPRN_SPRG_TLB_SCRATCH,r10
 190	SPECIAL_EXC_LOAD(r10,MAS0)
 191	mtspr	SPRN_MAS0,r10
 192	SPECIAL_EXC_LOAD(r10,MAS1)
 193	mtspr	SPRN_MAS1,r10
 194	SPECIAL_EXC_LOAD(r10,MAS2)
 195	mtspr	SPRN_MAS2,r10
 196	SPECIAL_EXC_LOAD(r10,MAS3)
 197	mtspr	SPRN_MAS3,r10
 198	SPECIAL_EXC_LOAD(r10,MAS6)
 199	mtspr	SPRN_MAS6,r10
 200	SPECIAL_EXC_LOAD(r10,MAS7)
 201	mtspr	SPRN_MAS7,r10
 202BEGIN_FTR_SECTION
 203	SPECIAL_EXC_LOAD(r10,MAS5)
 204	mtspr	SPRN_MAS5,r10
 205	SPECIAL_EXC_LOAD(r10,MAS8)
 206	mtspr	SPRN_MAS8,r10
 207END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
 208
 209	lbz	r6,PACAIRQSOFTMASK(r13)
 210	ld	r5,SOFTE(r1)
 211
 212	/* Interrupts had better not already be enabled... */
 213	tweqi	r6,IRQS_ENABLED
 214
 215	andi.	r6,r5,IRQS_DISABLED
 216	bne	1f
 217
 218	TRACE_ENABLE_INTS
 219	stb	r5,PACAIRQSOFTMASK(r13)
 2201:
 221	/*
 222	 * Restore PACAIRQHAPPENED rather than setting it based on
 223	 * the return MSR[EE], since we could have interrupted
 224	 * __check_irq_replay() or other inconsistent transitory
 225	 * states that must remain that way.
 226	 */
 227	SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
 228	stb	r10,PACAIRQHAPPENED(r13)
 229
 230	SPECIAL_EXC_LOAD(r10,DEAR)
 231	mtspr	SPRN_DEAR,r10
 232	SPECIAL_EXC_LOAD(r10,ESR)
 233	mtspr	SPRN_ESR,r10
 234
 235	stdcx.	r0,0,r1		/* to clear the reservation */
 236
 237	REST_4GPRS(2, r1)
 238	REST_4GPRS(6, r1)
 239
 240	ld	r10,_CTR(r1)
 241	ld	r11,_XER(r1)
 242	mtctr	r10
 243	mtxer	r11
 244
 245	blr
 246
 247.macro ret_from_level srr0 srr1 paca_ex scratch
 248	bl	ret_from_level_except
 249
 250	ld	r10,_LINK(r1)
 251	ld	r11,_CCR(r1)
 252	ld	r0,GPR13(r1)
 253	mtlr	r10
 254	mtcr	r11
 255
 256	ld	r10,GPR10(r1)
 257	ld	r11,GPR11(r1)
 258	ld	r12,GPR12(r1)
 259	mtspr	\scratch,r0
 260
 261	std	r10,\paca_ex+EX_R10(r13);
 262	std	r11,\paca_ex+EX_R11(r13);
 263	ld	r10,_NIP(r1)
 264	ld	r11,_MSR(r1)
 265	ld	r0,GPR0(r1)
 266	ld	r1,GPR1(r1)
 267	mtspr	\srr0,r10
 268	mtspr	\srr1,r11
 269	ld	r10,\paca_ex+EX_R10(r13)
 270	ld	r11,\paca_ex+EX_R11(r13)
 271	mfspr	r13,\scratch
 272.endm
 273
 274ret_from_crit_except:
 275	ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
 276	rfci
 277
 278ret_from_mc_except:
 279	ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
 280	rfmci
 281
 282/* Exception prolog code for all exceptions */
 283#define EXCEPTION_PROLOG(n, intnum, type, addition)	    		    \
 284	mtspr	SPRN_SPRG_##type##_SCRATCH,r13;	/* get spare registers */   \
 285	mfspr	r13,SPRN_SPRG_PACA;	/* get PACA */			    \
 286	std	r10,PACA_EX##type+EX_R10(r13);				    \
 287	std	r11,PACA_EX##type+EX_R11(r13);				    \
 288	mfcr	r10;			/* save CR */			    \
 289	mfspr	r11,SPRN_##type##_SRR1;/* what are we coming from */	    \
 290	DO_KVM	intnum,SPRN_##type##_SRR1;    /* KVM hook */		    \
 291	stw	r10,PACA_EX##type+EX_CR(r13); /* save old CR in the PACA */ \
 292	addition;			/* additional code for that exc. */ \
 293	std	r1,PACA_EX##type+EX_R1(r13); /* save old r1 in the PACA */  \
 294	type##_SET_KSTACK;		/* get special stack if necessary */\
 295	andi.	r10,r11,MSR_PR;		/* save stack pointer */	    \
 296	beq	1f;			/* branch around if supervisor */   \
 297	ld	r1,PACAKSAVE(r13);	/* get kernel stack coming from usr */\
 2981:	cmpdi	cr1,r1,0;		/* check if SP makes sense */	    \
 299	bge-	cr1,exc_##n##_bad_stack;/* bad stack (TODO: out of line) */ \
 300	mfspr	r10,SPRN_##type##_SRR0;	/* read SRR0 before touching stack */
 301
 302/* Exception type-specific macros */
 303#define	GEN_SET_KSTACK							    \
 304	subi	r1,r1,INT_FRAME_SIZE;	/* alloc frame on kernel stack */
 305#define SPRN_GEN_SRR0	SPRN_SRR0
 306#define SPRN_GEN_SRR1	SPRN_SRR1
 307
 308#define	GDBELL_SET_KSTACK	GEN_SET_KSTACK
 309#define SPRN_GDBELL_SRR0	SPRN_GSRR0
 310#define SPRN_GDBELL_SRR1	SPRN_GSRR1
 311
 312#define CRIT_SET_KSTACK						            \
 313	ld	r1,PACA_CRIT_STACK(r13);				    \
 314	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE
 315#define SPRN_CRIT_SRR0	SPRN_CSRR0
 316#define SPRN_CRIT_SRR1	SPRN_CSRR1
 317
 318#define DBG_SET_KSTACK						            \
 319	ld	r1,PACA_DBG_STACK(r13);					    \
 320	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE
 321#define SPRN_DBG_SRR0	SPRN_DSRR0
 322#define SPRN_DBG_SRR1	SPRN_DSRR1
 323
 324#define MC_SET_KSTACK						            \
 325	ld	r1,PACA_MC_STACK(r13);					    \
 326	subi	r1,r1,SPECIAL_EXC_FRAME_SIZE
 327#define SPRN_MC_SRR0	SPRN_MCSRR0
 328#define SPRN_MC_SRR1	SPRN_MCSRR1
 329
 330#define NORMAL_EXCEPTION_PROLOG(n, intnum, addition)			    \
 331	EXCEPTION_PROLOG(n, intnum, GEN, addition##_GEN(n))
 332
 333#define CRIT_EXCEPTION_PROLOG(n, intnum, addition)			    \
 334	EXCEPTION_PROLOG(n, intnum, CRIT, addition##_CRIT(n))
 335
 336#define DBG_EXCEPTION_PROLOG(n, intnum, addition)			    \
 337	EXCEPTION_PROLOG(n, intnum, DBG, addition##_DBG(n))
 338
 339#define MC_EXCEPTION_PROLOG(n, intnum, addition)			    \
 340	EXCEPTION_PROLOG(n, intnum, MC, addition##_MC(n))
 341
 342#define GDBELL_EXCEPTION_PROLOG(n, intnum, addition)			    \
 343	EXCEPTION_PROLOG(n, intnum, GDBELL, addition##_GDBELL(n))
 344
 345/* Variants of the "addition" argument for the prolog
 346 */
 347#define PROLOG_ADDITION_NONE_GEN(n)
 348#define PROLOG_ADDITION_NONE_GDBELL(n)
 349#define PROLOG_ADDITION_NONE_CRIT(n)
 350#define PROLOG_ADDITION_NONE_DBG(n)
 351#define PROLOG_ADDITION_NONE_MC(n)
 352
 353#define PROLOG_ADDITION_MASKABLE_GEN(n)					    \
 354	lbz	r10,PACAIRQSOFTMASK(r13);	/* are irqs soft-masked? */ \
 355	andi.	r10,r10,IRQS_DISABLED;	/* yes -> go out of line */ \
 356	bne	masked_interrupt_book3e_##n
 357
 358#define PROLOG_ADDITION_2REGS_GEN(n)					    \
 359	std	r14,PACA_EXGEN+EX_R14(r13);				    \
 360	std	r15,PACA_EXGEN+EX_R15(r13)
 361
 362#define PROLOG_ADDITION_1REG_GEN(n)					    \
 363	std	r14,PACA_EXGEN+EX_R14(r13);
 364
 365#define PROLOG_ADDITION_2REGS_CRIT(n)					    \
 366	std	r14,PACA_EXCRIT+EX_R14(r13);				    \
 367	std	r15,PACA_EXCRIT+EX_R15(r13)
 368
 369#define PROLOG_ADDITION_2REGS_DBG(n)					    \
 370	std	r14,PACA_EXDBG+EX_R14(r13);				    \
 371	std	r15,PACA_EXDBG+EX_R15(r13)
 372
 373#define PROLOG_ADDITION_2REGS_MC(n)					    \
 374	std	r14,PACA_EXMC+EX_R14(r13);				    \
 375	std	r15,PACA_EXMC+EX_R15(r13)
 376
 377
 378/* Core exception code for all exceptions except TLB misses. */
 379#define EXCEPTION_COMMON_LVL(n, scratch, excf)				    \
 380exc_##n##_common:							    \
 381	std	r0,GPR0(r1);		/* save r0 in stackframe */	    \
 382	std	r2,GPR2(r1);		/* save r2 in stackframe */	    \
 383	SAVE_4GPRS(3, r1);		/* save r3 - r6 in stackframe */    \
 384	SAVE_2GPRS(7, r1);		/* save r7, r8 in stackframe */	    \
 385	std	r9,GPR9(r1);		/* save r9 in stackframe */	    \
 386	std	r10,_NIP(r1);		/* save SRR0 to stackframe */	    \
 387	std	r11,_MSR(r1);		/* save SRR1 to stackframe */	    \
 388	beq	2f;			/* if from kernel mode */	    \
 389	ACCOUNT_CPU_USER_ENTRY(r13,r10,r11);/* accounting (uses cr0+eq) */  \
 3902:	ld	r3,excf+EX_R10(r13);	/* get back r10 */		    \
 391	ld	r4,excf+EX_R11(r13);	/* get back r11 */		    \
 392	mfspr	r5,scratch;		/* get back r13 */		    \
 393	std	r12,GPR12(r1);		/* save r12 in stackframe */	    \
 394	ld	r2,PACATOC(r13);	/* get kernel TOC into r2 */	    \
 395	mflr	r6;			/* save LR in stackframe */	    \
 396	mfctr	r7;			/* save CTR in stackframe */	    \
 397	mfspr	r8,SPRN_XER;		/* save XER in stackframe */	    \
 398	ld	r9,excf+EX_R1(r13);	/* load orig r1 back from PACA */   \
 399	lwz	r10,excf+EX_CR(r13);	/* load orig CR back from PACA	*/  \
 400	lbz	r11,PACAIRQSOFTMASK(r13); /* get current IRQ softe */	    \
 401	ld	r12,exception_marker@toc(r2);				    \
 402	li	r0,0;							    \
 403	std	r3,GPR10(r1);		/* save r10 to stackframe */	    \
 404	std	r4,GPR11(r1);		/* save r11 to stackframe */	    \
 405	std	r5,GPR13(r1);		/* save it to stackframe */	    \
 406	std	r6,_LINK(r1);						    \
 407	std	r7,_CTR(r1);						    \
 408	std	r8,_XER(r1);						    \
 409	li	r3,(n)+1;		/* indicate partial regs in trap */ \
 410	std	r9,0(r1);		/* store stack frame back link */   \
 411	std	r10,_CCR(r1);		/* store orig CR in stackframe */   \
 412	std	r9,GPR1(r1);		/* store stack frame back link */   \
 413	std	r11,SOFTE(r1);		/* and save it to stackframe */     \
 414	std	r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */	    \
 415	std	r3,_TRAP(r1);		/* set trap number		*/  \
 416	std	r0,RESULT(r1);		/* clear regs->result */
 417
 418#define EXCEPTION_COMMON(n) \
 419	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)
 420#define EXCEPTION_COMMON_CRIT(n) \
 421	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_CRIT_SCRATCH, PACA_EXCRIT)
 422#define EXCEPTION_COMMON_MC(n) \
 423	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_MC_SCRATCH, PACA_EXMC)
 424#define EXCEPTION_COMMON_DBG(n) \
 425	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)
 426
 427/*
 428 * This is meant for exceptions that don't immediately hard-enable.  We
 429 * set a bit in paca->irq_happened to ensure that a subsequent call to
 430 * arch_local_irq_restore() will properly hard-enable and avoid the
 431 * fast-path, and then reconcile irq state.
 432 */
 433#define INTS_DISABLE	RECONCILE_IRQ_STATE(r3,r4)
 434
 435/*
 436 * This is called by exceptions that don't use INTS_DISABLE (that did not
 437 * touch irq indicators in the PACA).  This will restore MSR:EE to it's
 438 * previous value
 439 *
 440 * XXX In the long run, we may want to open-code it in order to separate the
 441 *     load from the wrtee, thus limiting the latency caused by the dependency
 442 *     but at this point, I'll favor code clarity until we have a near to final
 443 *     implementation
 444 */
 445#define INTS_RESTORE_HARD						    \
 446	ld	r11,_MSR(r1);						    \
 447	wrtee	r11;
 448
 449/* XXX FIXME: Restore r14/r15 when necessary */
 450#define BAD_STACK_TRAMPOLINE(n)						    \
 451exc_##n##_bad_stack:							    \
 452	li	r1,(n);			/* get exception number */	    \
 453	sth	r1,PACA_TRAP_SAVE(r13);	/* store trap */		    \
 454	b	bad_stack_book3e;	/* bad stack error */
 455
 456/* WARNING: If you change the layout of this stub, make sure you check
 457	*   the debug exception handler which handles single stepping
 458	*   into exceptions from userspace, and the MM code in
 459	*   arch/powerpc/mm/tlb_nohash.c which patches the branch here
 460	*   and would need to be updated if that branch is moved
 461	*/
 462#define	EXCEPTION_STUB(loc, label)					\
 463	. = interrupt_base_book3e + loc;				\
 464	nop;	/* To make debug interrupts happy */			\
 465	b	exc_##label##_book3e;
 466
 467#define ACK_NONE(r)
 468#define ACK_DEC(r)							\
 469	lis	r,TSR_DIS@h;						\
 470	mtspr	SPRN_TSR,r
 471#define ACK_FIT(r)							\
 472	lis	r,TSR_FIS@h;						\
 473	mtspr	SPRN_TSR,r
 474
 475/* Used by asynchronous interrupt that may happen in the idle loop.
 476 *
 477 * This check if the thread was in the idle loop, and if yes, returns
 478 * to the caller rather than the PC. This is to avoid a race if
 479 * interrupts happen before the wait instruction.
 480 */
 481#define CHECK_NAPPING()							\
 482	CURRENT_THREAD_INFO(r11, r1);					\
 483	ld	r10,TI_LOCAL_FLAGS(r11);				\
 484	andi.	r9,r10,_TLF_NAPPING;					\
 485	beq+	1f;							\
 486	ld	r8,_LINK(r1);						\
 487	rlwinm	r7,r10,0,~_TLF_NAPPING;					\
 488	std	r8,_NIP(r1);						\
 489	std	r7,TI_LOCAL_FLAGS(r11);					\
 4901:
 491
 492
 493#define MASKABLE_EXCEPTION(trapnum, intnum, label, hdlr, ack)		\
 494	START_EXCEPTION(label);						\
 495	NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\
 496	EXCEPTION_COMMON(trapnum)					\
 497	INTS_DISABLE;							\
 498	ack(r8);							\
 499	CHECK_NAPPING();						\
 500	addi	r3,r1,STACK_FRAME_OVERHEAD;				\
 501	bl	hdlr;							\
 502	b	ret_from_except_lite;
 503
 504/* This value is used to mark exception frames on the stack. */
 505	.section	".toc","aw"
 506exception_marker:
 507	.tc	ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
 508
 509
 510/*
 511 * And here we have the exception vectors !
 512 */
 513
 514	.text
 515	.balign	0x1000
 516	.globl interrupt_base_book3e
 517interrupt_base_book3e:					/* fake trap */
 518	EXCEPTION_STUB(0x000, machine_check)
 519	EXCEPTION_STUB(0x020, critical_input)		/* 0x0100 */
 520	EXCEPTION_STUB(0x040, debug_crit)		/* 0x0d00 */
 521	EXCEPTION_STUB(0x060, data_storage)		/* 0x0300 */
 522	EXCEPTION_STUB(0x080, instruction_storage)	/* 0x0400 */
 523	EXCEPTION_STUB(0x0a0, external_input)		/* 0x0500 */
 524	EXCEPTION_STUB(0x0c0, alignment)		/* 0x0600 */
 525	EXCEPTION_STUB(0x0e0, program)			/* 0x0700 */
 526	EXCEPTION_STUB(0x100, fp_unavailable)		/* 0x0800 */
 527	EXCEPTION_STUB(0x120, system_call)		/* 0x0c00 */
 528	EXCEPTION_STUB(0x140, ap_unavailable)		/* 0x0f20 */
 529	EXCEPTION_STUB(0x160, decrementer)		/* 0x0900 */
 530	EXCEPTION_STUB(0x180, fixed_interval)		/* 0x0980 */
 531	EXCEPTION_STUB(0x1a0, watchdog)			/* 0x09f0 */
 532	EXCEPTION_STUB(0x1c0, data_tlb_miss)
 533	EXCEPTION_STUB(0x1e0, instruction_tlb_miss)
 534	EXCEPTION_STUB(0x200, altivec_unavailable)
 535	EXCEPTION_STUB(0x220, altivec_assist)
 536	EXCEPTION_STUB(0x260, perfmon)
 537	EXCEPTION_STUB(0x280, doorbell)
 538	EXCEPTION_STUB(0x2a0, doorbell_crit)
 539	EXCEPTION_STUB(0x2c0, guest_doorbell)
 540	EXCEPTION_STUB(0x2e0, guest_doorbell_crit)
 541	EXCEPTION_STUB(0x300, hypercall)
 542	EXCEPTION_STUB(0x320, ehpriv)
 543	EXCEPTION_STUB(0x340, lrat_error)
 544
 545	.globl __end_interrupts
 546__end_interrupts:
 547
 548/* Critical Input Interrupt */
 549	START_EXCEPTION(critical_input);
 550	CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
 551			      PROLOG_ADDITION_NONE)
 552	EXCEPTION_COMMON_CRIT(0x100)
 553	bl	save_nvgprs
 554	bl	special_reg_save
 555	CHECK_NAPPING();
 556	addi	r3,r1,STACK_FRAME_OVERHEAD
 557	bl	unknown_exception
 558	b	ret_from_crit_except
 559
 560/* Machine Check Interrupt */
 561	START_EXCEPTION(machine_check);
 562	MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
 563			    PROLOG_ADDITION_NONE)
 564	EXCEPTION_COMMON_MC(0x000)
 565	bl	save_nvgprs
 566	bl	special_reg_save
 567	CHECK_NAPPING();
 568	addi	r3,r1,STACK_FRAME_OVERHEAD
 569	bl	machine_check_exception
 570	b	ret_from_mc_except
 571
 572/* Data Storage Interrupt */
 573	START_EXCEPTION(data_storage)
 574	NORMAL_EXCEPTION_PROLOG(0x300, BOOKE_INTERRUPT_DATA_STORAGE,
 575				PROLOG_ADDITION_2REGS)
 576	mfspr	r14,SPRN_DEAR
 577	mfspr	r15,SPRN_ESR
 578	EXCEPTION_COMMON(0x300)
 579	INTS_DISABLE
 580	b	storage_fault_common
 581
 582/* Instruction Storage Interrupt */
 583	START_EXCEPTION(instruction_storage);
 584	NORMAL_EXCEPTION_PROLOG(0x400, BOOKE_INTERRUPT_INST_STORAGE,
 585				PROLOG_ADDITION_2REGS)
 586	li	r15,0
 587	mr	r14,r10
 588	EXCEPTION_COMMON(0x400)
 589	INTS_DISABLE
 590	b	storage_fault_common
 591
 592/* External Input Interrupt */
 593	MASKABLE_EXCEPTION(0x500, BOOKE_INTERRUPT_EXTERNAL,
 594			   external_input, do_IRQ, ACK_NONE)
 595
 596/* Alignment */
 597	START_EXCEPTION(alignment);
 598	NORMAL_EXCEPTION_PROLOG(0x600, BOOKE_INTERRUPT_ALIGNMENT,
 599				PROLOG_ADDITION_2REGS)
 600	mfspr	r14,SPRN_DEAR
 601	mfspr	r15,SPRN_ESR
 602	EXCEPTION_COMMON(0x600)
 603	b	alignment_more	/* no room, go out of line */
 604
 605/* Program Interrupt */
 606	START_EXCEPTION(program);
 607	NORMAL_EXCEPTION_PROLOG(0x700, BOOKE_INTERRUPT_PROGRAM,
 608				PROLOG_ADDITION_1REG)
 609	mfspr	r14,SPRN_ESR
 610	EXCEPTION_COMMON(0x700)
 611	INTS_DISABLE
 612	std	r14,_DSISR(r1)
 613	addi	r3,r1,STACK_FRAME_OVERHEAD
 614	ld	r14,PACA_EXGEN+EX_R14(r13)
 615	bl	save_nvgprs
 616	bl	program_check_exception
 617	b	ret_from_except
 618
 619/* Floating Point Unavailable Interrupt */
 620	START_EXCEPTION(fp_unavailable);
 621	NORMAL_EXCEPTION_PROLOG(0x800, BOOKE_INTERRUPT_FP_UNAVAIL,
 622				PROLOG_ADDITION_NONE)
 623	/* we can probably do a shorter exception entry for that one... */
 624	EXCEPTION_COMMON(0x800)
 625	ld	r12,_MSR(r1)
 626	andi.	r0,r12,MSR_PR;
 627	beq-	1f
 628	bl	load_up_fpu
 629	b	fast_exception_return
 6301:	INTS_DISABLE
 631	bl	save_nvgprs
 632	addi	r3,r1,STACK_FRAME_OVERHEAD
 633	bl	kernel_fp_unavailable_exception
 634	b	ret_from_except
 635
 636/* Altivec Unavailable Interrupt */
 637	START_EXCEPTION(altivec_unavailable);
 638	NORMAL_EXCEPTION_PROLOG(0x200, BOOKE_INTERRUPT_ALTIVEC_UNAVAIL,
 639				PROLOG_ADDITION_NONE)
 640	/* we can probably do a shorter exception entry for that one... */
 641	EXCEPTION_COMMON(0x200)
 642#ifdef CONFIG_ALTIVEC
 643BEGIN_FTR_SECTION
 644	ld	r12,_MSR(r1)
 645	andi.	r0,r12,MSR_PR;
 646	beq-	1f
 647	bl	load_up_altivec
 648	b	fast_exception_return
 6491:
 650END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 651#endif
 652	INTS_DISABLE
 653	bl	save_nvgprs
 654	addi	r3,r1,STACK_FRAME_OVERHEAD
 655	bl	altivec_unavailable_exception
 656	b	ret_from_except
 657
 658/* AltiVec Assist */
 659	START_EXCEPTION(altivec_assist);
 660	NORMAL_EXCEPTION_PROLOG(0x220,
 661				BOOKE_INTERRUPT_ALTIVEC_ASSIST,
 662				PROLOG_ADDITION_NONE)
 663	EXCEPTION_COMMON(0x220)
 664	INTS_DISABLE
 665	bl	save_nvgprs
 666	addi	r3,r1,STACK_FRAME_OVERHEAD
 667#ifdef CONFIG_ALTIVEC
 668BEGIN_FTR_SECTION
 669	bl	altivec_assist_exception
 670END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 671#else
 672	bl	unknown_exception
 673#endif
 674	b	ret_from_except
 675
 676
 677/* Decrementer Interrupt */
 678	MASKABLE_EXCEPTION(0x900, BOOKE_INTERRUPT_DECREMENTER,
 679			   decrementer, timer_interrupt, ACK_DEC)
 680
 681/* Fixed Interval Timer Interrupt */
 682	MASKABLE_EXCEPTION(0x980, BOOKE_INTERRUPT_FIT,
 683			   fixed_interval, unknown_exception, ACK_FIT)
 684
 685/* Watchdog Timer Interrupt */
 686	START_EXCEPTION(watchdog);
 687	CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
 688			      PROLOG_ADDITION_NONE)
 689	EXCEPTION_COMMON_CRIT(0x9f0)
 690	bl	save_nvgprs
 691	bl	special_reg_save
 692	CHECK_NAPPING();
 693	addi	r3,r1,STACK_FRAME_OVERHEAD
 694#ifdef CONFIG_BOOKE_WDT
 695	bl	WatchdogException
 696#else
 697	bl	unknown_exception
 698#endif
 699	b	ret_from_crit_except
 700
 701/* System Call Interrupt */
 702	START_EXCEPTION(system_call)
 703	mr	r9,r13			/* keep a copy of userland r13 */
 704	mfspr	r11,SPRN_SRR0		/* get return address */
 705	mfspr	r12,SPRN_SRR1		/* get previous MSR */
 706	mfspr	r13,SPRN_SPRG_PACA	/* get our PACA */
 707	b	system_call_common
 708
 709/* Auxiliary Processor Unavailable Interrupt */
 710	START_EXCEPTION(ap_unavailable);
 711	NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,
 712				PROLOG_ADDITION_NONE)
 713	EXCEPTION_COMMON(0xf20)
 714	INTS_DISABLE
 715	bl	save_nvgprs
 716	addi	r3,r1,STACK_FRAME_OVERHEAD
 717	bl	unknown_exception
 718	b	ret_from_except
 719
 720/* Debug exception as a critical interrupt*/
 721	START_EXCEPTION(debug_crit);
 722	CRIT_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG,
 723			      PROLOG_ADDITION_2REGS)
 724
 725	/*
 726	 * If there is a single step or branch-taken exception in an
 727	 * exception entry sequence, it was probably meant to apply to
 728	 * the code where the exception occurred (since exception entry
 729	 * doesn't turn off DE automatically).  We simulate the effect
 730	 * of turning off DE on entry to an exception handler by turning
 731	 * off DE in the CSRR1 value and clearing the debug status.
 732	 */
 733
 734	mfspr	r14,SPRN_DBSR		/* check single-step/branch taken */
 735	andis.	r15,r14,(DBSR_IC|DBSR_BT)@h
 736	beq+	1f
 737
 738#ifdef CONFIG_RELOCATABLE
 739	ld	r15,PACATOC(r13)
 740	ld	r14,interrupt_base_book3e@got(r15)
 741	ld	r15,__end_interrupts@got(r15)
 742#else
 743	LOAD_REG_IMMEDIATE(r14,interrupt_base_book3e)
 744	LOAD_REG_IMMEDIATE(r15,__end_interrupts)
 745#endif
 746	cmpld	cr0,r10,r14
 747	cmpld	cr1,r10,r15
 748	blt+	cr0,1f
 749	bge+	cr1,1f
 750
 751	/* here it looks like we got an inappropriate debug exception. */
 752	lis	r14,(DBSR_IC|DBSR_BT)@h		/* clear the event */
 753	rlwinm	r11,r11,0,~MSR_DE	/* clear DE in the CSRR1 value */
 754	mtspr	SPRN_DBSR,r14
 755	mtspr	SPRN_CSRR1,r11
 756	lwz	r10,PACA_EXCRIT+EX_CR(r13)	/* restore registers */
 757	ld	r1,PACA_EXCRIT+EX_R1(r13)
 758	ld	r14,PACA_EXCRIT+EX_R14(r13)
 759	ld	r15,PACA_EXCRIT+EX_R15(r13)
 760	mtcr	r10
 761	ld	r10,PACA_EXCRIT+EX_R10(r13)	/* restore registers */
 762	ld	r11,PACA_EXCRIT+EX_R11(r13)
 763	mfspr	r13,SPRN_SPRG_CRIT_SCRATCH
 764	rfci
 765
 766	/* Normal debug exception */
 767	/* XXX We only handle coming from userspace for now since we can't
 768	 *     quite save properly an interrupted kernel state yet
 769	 */
 7701:	andi.	r14,r11,MSR_PR;		/* check for userspace again */
 771	beq	kernel_dbg_exc;		/* if from kernel mode */
 772
 773	/* Now we mash up things to make it look like we are coming on a
 774	 * normal exception
 775	 */
 776	mfspr	r14,SPRN_DBSR
 777	EXCEPTION_COMMON_CRIT(0xd00)
 778	std	r14,_DSISR(r1)
 779	addi	r3,r1,STACK_FRAME_OVERHEAD
 780	mr	r4,r14
 781	ld	r14,PACA_EXCRIT+EX_R14(r13)
 782	ld	r15,PACA_EXCRIT+EX_R15(r13)
 783	bl	save_nvgprs
 784	bl	DebugException
 785	b	ret_from_except
 786
 787kernel_dbg_exc:
 788	b	.	/* NYI */
 789
 790/* Debug exception as a debug interrupt*/
 791	START_EXCEPTION(debug_debug);
 792	DBG_EXCEPTION_PROLOG(0xd00, BOOKE_INTERRUPT_DEBUG,
 793						 PROLOG_ADDITION_2REGS)
 794
 795	/*
 796	 * If there is a single step or branch-taken exception in an
 797	 * exception entry sequence, it was probably meant to apply to
 798	 * the code where the exception occurred (since exception entry
 799	 * doesn't turn off DE automatically).  We simulate the effect
 800	 * of turning off DE on entry to an exception handler by turning
 801	 * off DE in the DSRR1 value and clearing the debug status.
 802	 */
 803
 804	mfspr	r14,SPRN_DBSR		/* check single-step/branch taken */
 805	andis.	r15,r14,(DBSR_IC|DBSR_BT)@h
 806	beq+	1f
 807
 808#ifdef CONFIG_RELOCATABLE
 809	ld	r15,PACATOC(r13)
 810	ld	r14,interrupt_base_book3e@got(r15)
 811	ld	r15,__end_interrupts@got(r15)
 812#else
 813	LOAD_REG_IMMEDIATE(r14,interrupt_base_book3e)
 814	LOAD_REG_IMMEDIATE(r15,__end_interrupts)
 815#endif
 816	cmpld	cr0,r10,r14
 817	cmpld	cr1,r10,r15
 818	blt+	cr0,1f
 819	bge+	cr1,1f
 820
 821	/* here it looks like we got an inappropriate debug exception. */
 822	lis	r14,(DBSR_IC|DBSR_BT)@h		/* clear the event */
 823	rlwinm	r11,r11,0,~MSR_DE	/* clear DE in the DSRR1 value */
 824	mtspr	SPRN_DBSR,r14
 825	mtspr	SPRN_DSRR1,r11
 826	lwz	r10,PACA_EXDBG+EX_CR(r13)	/* restore registers */
 827	ld	r1,PACA_EXDBG+EX_R1(r13)
 828	ld	r14,PACA_EXDBG+EX_R14(r13)
 829	ld	r15,PACA_EXDBG+EX_R15(r13)
 830	mtcr	r10
 831	ld	r10,PACA_EXDBG+EX_R10(r13)	/* restore registers */
 832	ld	r11,PACA_EXDBG+EX_R11(r13)
 833	mfspr	r13,SPRN_SPRG_DBG_SCRATCH
 834	rfdi
 835
 836	/* Normal debug exception */
 837	/* XXX We only handle coming from userspace for now since we can't
 838	 *     quite save properly an interrupted kernel state yet
 839	 */
 8401:	andi.	r14,r11,MSR_PR;		/* check for userspace again */
 841	beq	kernel_dbg_exc;		/* if from kernel mode */
 842
 843	/* Now we mash up things to make it look like we are coming on a
 844	 * normal exception
 845	 */
 846	mfspr	r14,SPRN_DBSR
 847	EXCEPTION_COMMON_DBG(0xd08)
 848	INTS_DISABLE
 849	std	r14,_DSISR(r1)
 850	addi	r3,r1,STACK_FRAME_OVERHEAD
 851	mr	r4,r14
 852	ld	r14,PACA_EXDBG+EX_R14(r13)
 853	ld	r15,PACA_EXDBG+EX_R15(r13)
 854	bl	save_nvgprs
 855	bl	DebugException
 856	b	ret_from_except
 857
 858	START_EXCEPTION(perfmon);
 859	NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,
 860				PROLOG_ADDITION_NONE)
 861	EXCEPTION_COMMON(0x260)
 862	INTS_DISABLE
 863	CHECK_NAPPING()
 864	addi	r3,r1,STACK_FRAME_OVERHEAD
 865	bl	performance_monitor_exception
 866	b	ret_from_except_lite
 867
 868/* Doorbell interrupt */
 869	MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL,
 870			   doorbell, doorbell_exception, ACK_NONE)
 871
 872/* Doorbell critical Interrupt */
 873	START_EXCEPTION(doorbell_crit);
 874	CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
 875			      PROLOG_ADDITION_NONE)
 876	EXCEPTION_COMMON_CRIT(0x2a0)
 877	bl	save_nvgprs
 878	bl	special_reg_save
 879	CHECK_NAPPING();
 880	addi	r3,r1,STACK_FRAME_OVERHEAD
 881	bl	unknown_exception
 882	b	ret_from_crit_except
 883
 884/*
 885 *	Guest doorbell interrupt
 886 *	This general exception use GSRRx save/restore registers
 887 */
 888	START_EXCEPTION(guest_doorbell);
 889	GDBELL_EXCEPTION_PROLOG(0x2c0, BOOKE_INTERRUPT_GUEST_DBELL,
 890			        PROLOG_ADDITION_NONE)
 891	EXCEPTION_COMMON(0x2c0)
 892	addi	r3,r1,STACK_FRAME_OVERHEAD
 893	bl	save_nvgprs
 894	INTS_RESTORE_HARD
 895	bl	unknown_exception
 896	b	ret_from_except
 897
 898/* Guest Doorbell critical Interrupt */
 899	START_EXCEPTION(guest_doorbell_crit);
 900	CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
 901			      PROLOG_ADDITION_NONE)
 902	EXCEPTION_COMMON_CRIT(0x2e0)
 903	bl	save_nvgprs
 904	bl	special_reg_save
 905	CHECK_NAPPING();
 906	addi	r3,r1,STACK_FRAME_OVERHEAD
 907	bl	unknown_exception
 908	b	ret_from_crit_except
 909
 910/* Hypervisor call */
 911	START_EXCEPTION(hypercall);
 912	NORMAL_EXCEPTION_PROLOG(0x310, BOOKE_INTERRUPT_HV_SYSCALL,
 913			        PROLOG_ADDITION_NONE)
 914	EXCEPTION_COMMON(0x310)
 915	addi	r3,r1,STACK_FRAME_OVERHEAD
 916	bl	save_nvgprs
 917	INTS_RESTORE_HARD
 918	bl	unknown_exception
 919	b	ret_from_except
 920
 921/* Embedded Hypervisor priviledged  */
 922	START_EXCEPTION(ehpriv);
 923	NORMAL_EXCEPTION_PROLOG(0x320, BOOKE_INTERRUPT_HV_PRIV,
 924			        PROLOG_ADDITION_NONE)
 925	EXCEPTION_COMMON(0x320)
 926	addi	r3,r1,STACK_FRAME_OVERHEAD
 927	bl	save_nvgprs
 928	INTS_RESTORE_HARD
 929	bl	unknown_exception
 930	b	ret_from_except
 931
 932/* LRAT Error interrupt */
 933	START_EXCEPTION(lrat_error);
 934	NORMAL_EXCEPTION_PROLOG(0x340, BOOKE_INTERRUPT_LRAT_ERROR,
 935			        PROLOG_ADDITION_NONE)
 936	EXCEPTION_COMMON(0x340)
 937	addi	r3,r1,STACK_FRAME_OVERHEAD
 938	bl	save_nvgprs
 939	INTS_RESTORE_HARD
 940	bl	unknown_exception
 941	b	ret_from_except
 942
 943/*
 944 * An interrupt came in while soft-disabled; We mark paca->irq_happened
 945 * accordingly and if the interrupt is level sensitive, we hard disable
 946 * hard disable (full_mask) corresponds to PACA_IRQ_MUST_HARD_MASK, so
 947 * keep these in synch.
 948 */
 949
 950.macro masked_interrupt_book3e paca_irq full_mask
 951	lbz	r10,PACAIRQHAPPENED(r13)
 952	ori	r10,r10,\paca_irq
 953	stb	r10,PACAIRQHAPPENED(r13)
 954
 955	.if \full_mask == 1
 956	rldicl	r10,r11,48,1		/* clear MSR_EE */
 957	rotldi	r11,r10,16
 958	mtspr	SPRN_SRR1,r11
 959	.endif
 960
 961	lwz	r11,PACA_EXGEN+EX_CR(r13)
 962	mtcr	r11
 963	ld	r10,PACA_EXGEN+EX_R10(r13)
 964	ld	r11,PACA_EXGEN+EX_R11(r13)
 965	mfspr	r13,SPRN_SPRG_GEN_SCRATCH
 966	rfi
 967	b	.
 968.endm
 969
 970masked_interrupt_book3e_0x500:
 971	// XXX When adding support for EPR, use PACA_IRQ_EE_EDGE
 972	masked_interrupt_book3e PACA_IRQ_EE 1
 973
 974masked_interrupt_book3e_0x900:
 975	ACK_DEC(r10);
 976	masked_interrupt_book3e PACA_IRQ_DEC 0
 977
 978masked_interrupt_book3e_0x980:
 979	ACK_FIT(r10);
 980	masked_interrupt_book3e PACA_IRQ_DEC 0
 981
 982masked_interrupt_book3e_0x280:
 983masked_interrupt_book3e_0x2c0:
 984	masked_interrupt_book3e PACA_IRQ_DBELL 0
 985
 986/*
 987 * Called from arch_local_irq_enable when an interrupt needs
 988 * to be resent. r3 contains either 0x500,0x900,0x260 or 0x280
 989 * to indicate the kind of interrupt. MSR:EE is already off.
 990 * We generate a stackframe like if a real interrupt had happened.
 991 *
 992 * Note: While MSR:EE is off, we need to make sure that _MSR
 993 * in the generated frame has EE set to 1 or the exception
 994 * handler will not properly re-enable them.
 995 */
 996_GLOBAL(__replay_interrupt)
 997	/* We are going to jump to the exception common code which
 998	 * will retrieve various register values from the PACA which
 999	 * we don't give a damn about.
1000	 */
1001	mflr	r10
1002	mfmsr	r11
1003	mfcr	r4
1004	mtspr	SPRN_SPRG_GEN_SCRATCH,r13;
1005	std	r1,PACA_EXGEN+EX_R1(r13);
1006	stw	r4,PACA_EXGEN+EX_CR(r13);
1007	ori	r11,r11,MSR_EE
1008	subi	r1,r1,INT_FRAME_SIZE;
1009	cmpwi	cr0,r3,0x500
1010	beq	exc_0x500_common
1011	cmpwi	cr0,r3,0x900
1012	beq	exc_0x900_common
1013	cmpwi	cr0,r3,0x280
1014	beq	exc_0x280_common
1015	blr
1016
1017
1018/*
1019 * This is called from 0x300 and 0x400 handlers after the prologs with
1020 * r14 and r15 containing the fault address and error code, with the
1021 * original values stashed away in the PACA
1022 */
1023storage_fault_common:
1024	std	r14,_DAR(r1)
1025	std	r15,_DSISR(r1)
1026	addi	r3,r1,STACK_FRAME_OVERHEAD
1027	mr	r4,r14
1028	mr	r5,r15
1029	ld	r14,PACA_EXGEN+EX_R14(r13)
1030	ld	r15,PACA_EXGEN+EX_R15(r13)
1031	bl	do_page_fault
1032	cmpdi	r3,0
1033	bne-	1f
1034	b	ret_from_except_lite
10351:	bl	save_nvgprs
1036	mr	r5,r3
1037	addi	r3,r1,STACK_FRAME_OVERHEAD
1038	ld	r4,_DAR(r1)
1039	bl	bad_page_fault
1040	b	ret_from_except
1041
1042/*
1043 * Alignment exception doesn't fit entirely in the 0x100 bytes so it
1044 * continues here.
1045 */
1046alignment_more:
1047	std	r14,_DAR(r1)
1048	std	r15,_DSISR(r1)
1049	addi	r3,r1,STACK_FRAME_OVERHEAD
1050	ld	r14,PACA_EXGEN+EX_R14(r13)
1051	ld	r15,PACA_EXGEN+EX_R15(r13)
1052	bl	save_nvgprs
1053	INTS_RESTORE_HARD
1054	bl	alignment_exception
1055	b	ret_from_except
1056
1057/*
1058 * We branch here from entry_64.S for the last stage of the exception
1059 * return code path. MSR:EE is expected to be off at that point
1060 */
1061_GLOBAL(exception_return_book3e)
1062	b	1f
1063
1064/* This is the return from load_up_fpu fast path which could do with
1065 * less GPR restores in fact, but for now we have a single return path
1066 */
1067	.globl fast_exception_return
1068fast_exception_return:
1069	wrteei	0
10701:	mr	r0,r13
1071	ld	r10,_MSR(r1)
1072	REST_4GPRS(2, r1)
1073	andi.	r6,r10,MSR_PR
1074	REST_2GPRS(6, r1)
1075	beq	1f
1076	ACCOUNT_CPU_USER_EXIT(r13, r10, r11)
1077	ld	r0,GPR13(r1)
1078
10791:	stdcx.	r0,0,r1		/* to clear the reservation */
1080
1081	ld	r8,_CCR(r1)
1082	ld	r9,_LINK(r1)
1083	ld	r10,_CTR(r1)
1084	ld	r11,_XER(r1)
1085	mtcr	r8
1086	mtlr	r9
1087	mtctr	r10
1088	mtxer	r11
1089	REST_2GPRS(8, r1)
1090	ld	r10,GPR10(r1)
1091	ld	r11,GPR11(r1)
1092	ld	r12,GPR12(r1)
1093	mtspr	SPRN_SPRG_GEN_SCRATCH,r0
1094
1095	std	r10,PACA_EXGEN+EX_R10(r13);
1096	std	r11,PACA_EXGEN+EX_R11(r13);
1097	ld	r10,_NIP(r1)
1098	ld	r11,_MSR(r1)
1099	ld	r0,GPR0(r1)
1100	ld	r1,GPR1(r1)
1101	mtspr	SPRN_SRR0,r10
1102	mtspr	SPRN_SRR1,r11
1103	ld	r10,PACA_EXGEN+EX_R10(r13)
1104	ld	r11,PACA_EXGEN+EX_R11(r13)
1105	mfspr	r13,SPRN_SPRG_GEN_SCRATCH
1106	rfi
1107
1108/*
1109 * Trampolines used when spotting a bad kernel stack pointer in
1110 * the exception entry code.
1111 *
1112 * TODO: move some bits like SRR0 read to trampoline, pass PACA
1113 * index around, etc... to handle crit & mcheck
1114 */
1115BAD_STACK_TRAMPOLINE(0x000)
1116BAD_STACK_TRAMPOLINE(0x100)
1117BAD_STACK_TRAMPOLINE(0x200)
1118BAD_STACK_TRAMPOLINE(0x220)
1119BAD_STACK_TRAMPOLINE(0x260)
1120BAD_STACK_TRAMPOLINE(0x280)
1121BAD_STACK_TRAMPOLINE(0x2a0)
1122BAD_STACK_TRAMPOLINE(0x2c0)
1123BAD_STACK_TRAMPOLINE(0x2e0)
1124BAD_STACK_TRAMPOLINE(0x300)
1125BAD_STACK_TRAMPOLINE(0x310)
1126BAD_STACK_TRAMPOLINE(0x320)
1127BAD_STACK_TRAMPOLINE(0x340)
1128BAD_STACK_TRAMPOLINE(0x400)
1129BAD_STACK_TRAMPOLINE(0x500)
1130BAD_STACK_TRAMPOLINE(0x600)
1131BAD_STACK_TRAMPOLINE(0x700)
1132BAD_STACK_TRAMPOLINE(0x800)
1133BAD_STACK_TRAMPOLINE(0x900)
1134BAD_STACK_TRAMPOLINE(0x980)
1135BAD_STACK_TRAMPOLINE(0x9f0)
1136BAD_STACK_TRAMPOLINE(0xa00)
1137BAD_STACK_TRAMPOLINE(0xb00)
1138BAD_STACK_TRAMPOLINE(0xc00)
1139BAD_STACK_TRAMPOLINE(0xd00)
1140BAD_STACK_TRAMPOLINE(0xd08)
1141BAD_STACK_TRAMPOLINE(0xe00)
1142BAD_STACK_TRAMPOLINE(0xf00)
1143BAD_STACK_TRAMPOLINE(0xf20)
1144
1145	.globl	bad_stack_book3e
1146bad_stack_book3e:
1147	/* XXX: Needs to make SPRN_SPRG_GEN depend on exception type */
1148	mfspr	r10,SPRN_SRR0;		  /* read SRR0 before touching stack */
1149	ld	r1,PACAEMERGSP(r13)
1150	subi	r1,r1,64+INT_FRAME_SIZE
1151	std	r10,_NIP(r1)
1152	std	r11,_MSR(r1)
1153	ld	r10,PACA_EXGEN+EX_R1(r13) /* FIXME for crit & mcheck */
1154	lwz	r11,PACA_EXGEN+EX_CR(r13) /* FIXME for crit & mcheck */
1155	std	r10,GPR1(r1)
1156	std	r11,_CCR(r1)
1157	mfspr	r10,SPRN_DEAR
1158	mfspr	r11,SPRN_ESR
1159	std	r10,_DAR(r1)
1160	std	r11,_DSISR(r1)
1161	std	r0,GPR0(r1);		/* save r0 in stackframe */	    \
1162	std	r2,GPR2(r1);		/* save r2 in stackframe */	    \
1163	SAVE_4GPRS(3, r1);		/* save r3 - r6 in stackframe */    \
1164	SAVE_2GPRS(7, r1);		/* save r7, r8 in stackframe */	    \
1165	std	r9,GPR9(r1);		/* save r9 in stackframe */	    \
1166	ld	r3,PACA_EXGEN+EX_R10(r13);/* get back r10 */		    \
1167	ld	r4,PACA_EXGEN+EX_R11(r13);/* get back r11 */		    \
1168	mfspr	r5,SPRN_SPRG_GEN_SCRATCH;/* get back r13 XXX can be wrong */ \
1169	std	r3,GPR10(r1);		/* save r10 to stackframe */	    \
1170	std	r4,GPR11(r1);		/* save r11 to stackframe */	    \
1171	std	r12,GPR12(r1);		/* save r12 in stackframe */	    \
1172	std	r5,GPR13(r1);		/* save it to stackframe */	    \
1173	mflr	r10
1174	mfctr	r11
1175	mfxer	r12
1176	std	r10,_LINK(r1)
1177	std	r11,_CTR(r1)
1178	std	r12,_XER(r1)
1179	SAVE_10GPRS(14,r1)
1180	SAVE_8GPRS(24,r1)
1181	lhz	r12,PACA_TRAP_SAVE(r13)
1182	std	r12,_TRAP(r1)
1183	addi	r11,r1,INT_FRAME_SIZE
1184	std	r11,0(r1)
1185	li	r12,0
1186	std	r12,0(r11)
1187	ld	r2,PACATOC(r13)
11881:	addi	r3,r1,STACK_FRAME_OVERHEAD
1189	bl	kernel_bad_stack
1190	b	1b
1191
1192/*
1193 * Setup the initial TLB for a core. This current implementation
1194 * assume that whatever we are running off will not conflict with
1195 * the new mapping at PAGE_OFFSET.
1196 */
1197_GLOBAL(initial_tlb_book3e)
1198
1199	/* Look for the first TLB with IPROT set */
1200	mfspr	r4,SPRN_TLB0CFG
1201	andi.	r3,r4,TLBnCFG_IPROT
1202	lis	r3,MAS0_TLBSEL(0)@h
1203	bne	found_iprot
1204
1205	mfspr	r4,SPRN_TLB1CFG
1206	andi.	r3,r4,TLBnCFG_IPROT
1207	lis	r3,MAS0_TLBSEL(1)@h
1208	bne	found_iprot
1209
1210	mfspr	r4,SPRN_TLB2CFG
1211	andi.	r3,r4,TLBnCFG_IPROT
1212	lis	r3,MAS0_TLBSEL(2)@h
1213	bne	found_iprot
1214
1215	lis	r3,MAS0_TLBSEL(3)@h
1216	mfspr	r4,SPRN_TLB3CFG
1217	/* fall through */
1218
1219found_iprot:
1220	andi.	r5,r4,TLBnCFG_HES
1221	bne	have_hes
1222
1223	mflr	r8				/* save LR */
1224/* 1. Find the index of the entry we're executing in
1225 *
1226 * r3 = MAS0_TLBSEL (for the iprot array)
1227 * r4 = SPRN_TLBnCFG
1228 */
1229	bl	invstr				/* Find our address */
1230invstr:	mflr	r6				/* Make it accessible */
1231	mfmsr	r7
1232	rlwinm	r5,r7,27,31,31			/* extract MSR[IS] */
1233	mfspr	r7,SPRN_PID
1234	slwi	r7,r7,16
1235	or	r7,r7,r5
1236	mtspr	SPRN_MAS6,r7
1237	tlbsx	0,r6				/* search MSR[IS], SPID=PID */
1238
1239	mfspr	r3,SPRN_MAS0
1240	rlwinm	r5,r3,16,20,31			/* Extract MAS0(Entry) */
1241
1242	mfspr	r7,SPRN_MAS1			/* Insure IPROT set */
1243	oris	r7,r7,MAS1_IPROT@h
1244	mtspr	SPRN_MAS1,r7
1245	tlbwe
1246
1247/* 2. Invalidate all entries except the entry we're executing in
1248 *
1249 * r3 = MAS0 w/TLBSEL & ESEL for the entry we are running in
1250 * r4 = SPRN_TLBnCFG
1251 * r5 = ESEL of entry we are running in
1252 */
1253	andi.	r4,r4,TLBnCFG_N_ENTRY		/* Extract # entries */
1254	li	r6,0				/* Set Entry counter to 0 */
12551:	mr	r7,r3				/* Set MAS0(TLBSEL) */
1256	rlwimi	r7,r6,16,4,15			/* Setup MAS0 = TLBSEL | ESEL(r6) */
1257	mtspr	SPRN_MAS0,r7
1258	tlbre
1259	mfspr	r7,SPRN_MAS1
1260	rlwinm	r7,r7,0,2,31			/* Clear MAS1 Valid and IPROT */
1261	cmpw	r5,r6
1262	beq	skpinv				/* Dont update the current execution TLB */
1263	mtspr	SPRN_MAS1,r7
1264	tlbwe
1265	isync
1266skpinv:	addi	r6,r6,1				/* Increment */
1267	cmpw	r6,r4				/* Are we done? */
1268	bne	1b				/* If not, repeat */
1269
1270	/* Invalidate all TLBs */
1271	PPC_TLBILX_ALL(0,R0)
1272	sync
1273	isync
1274
1275/* 3. Setup a temp mapping and jump to it
1276 *
1277 * r3 = MAS0 w/TLBSEL & ESEL for the entry we are running in
1278 * r5 = ESEL of entry we are running in
1279 */
1280	andi.	r7,r5,0x1	/* Find an entry not used and is non-zero */
1281	addi	r7,r7,0x1
1282	mr	r4,r3		/* Set MAS0(TLBSEL) = 1 */
1283	mtspr	SPRN_MAS0,r4
1284	tlbre
1285
1286	rlwimi	r4,r7,16,4,15	/* Setup MAS0 = TLBSEL | ESEL(r7) */
1287	mtspr	SPRN_MAS0,r4
1288
1289	mfspr	r7,SPRN_MAS1
1290	xori	r6,r7,MAS1_TS		/* Setup TMP mapping in the other Address space */
1291	mtspr	SPRN_MAS1,r6
1292
1293	tlbwe
1294
1295	mfmsr	r6
1296	xori	r6,r6,MSR_IS
1297	mtspr	SPRN_SRR1,r6
1298	bl	1f		/* Find our address */
12991:	mflr	r6
1300	addi	r6,r6,(2f - 1b)
1301	mtspr	SPRN_SRR0,r6
1302	rfi
13032:
1304
1305/* 4. Clear out PIDs & Search info
1306 *
1307 * r3 = MAS0 w/TLBSEL & ESEL for the entry we started in
1308 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping
1309 * r5 = MAS3
1310 */
1311	li	r6,0
1312	mtspr   SPRN_MAS6,r6
1313	mtspr	SPRN_PID,r6
1314
1315/* 5. Invalidate mapping we started in
1316 *
1317 * r3 = MAS0 w/TLBSEL & ESEL for the entry we started in
1318 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping
1319 * r5 = MAS3
1320 */
1321	mtspr	SPRN_MAS0,r3
1322	tlbre
1323	mfspr	r6,SPRN_MAS1
1324	rlwinm	r6,r6,0,2,31	/* clear IPROT and VALID */
1325	mtspr	SPRN_MAS1,r6
1326	tlbwe
1327	sync
1328	isync
1329
1330/*
1331 * The mapping only needs to be cache-coherent on SMP, except on
1332 * Freescale e500mc derivatives where it's also needed for coherent DMA.
1333 */
1334#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC)
1335#define M_IF_NEEDED	MAS2_M
1336#else
1337#define M_IF_NEEDED	0
1338#endif
1339
1340/* 6. Setup KERNELBASE mapping in TLB[0]
1341 *
1342 * r3 = MAS0 w/TLBSEL & ESEL for the entry we started in
1343 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping
1344 * r5 = MAS3
1345 */
1346	rlwinm	r3,r3,0,16,3	/* clear ESEL */
1347	mtspr	SPRN_MAS0,r3
1348	lis	r6,(MAS1_VALID|MAS1_IPROT)@h
1349	ori	r6,r6,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
1350	mtspr	SPRN_MAS1,r6
1351
1352	LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET | M_IF_NEEDED)
1353	mtspr	SPRN_MAS2,r6
1354
1355	rlwinm	r5,r5,0,0,25
1356	ori	r5,r5,MAS3_SR | MAS3_SW | MAS3_SX
1357	mtspr	SPRN_MAS3,r5
1358	li	r5,-1
1359	rlwinm	r5,r5,0,0,25
1360
1361	tlbwe
1362
1363/* 7. Jump to KERNELBASE mapping
1364 *
1365 * r4 = MAS0 w/TLBSEL & ESEL for the temp mapping
1366 */
1367	/* Now we branch the new virtual address mapped by this entry */
1368	bl	1f		/* Find our address */
13691:	mflr	r6
1370	addi	r6,r6,(2f - 1b)
1371	tovirt(r6,r6)
1372	lis	r7,MSR_KERNEL@h
1373	ori	r7,r7,MSR_KERNEL@l
1374	mtspr	SPRN_SRR0,r6
1375	mtspr	SPRN_SRR1,r7
1376	rfi				/* start execution out of TLB1[0] entry */
13772:
1378
1379/* 8. Clear out the temp mapping
1380 *
1381 * r4 = MAS0 w/TLBSEL & ESEL for the entry we are running in
1382 */
1383	mtspr	SPRN_MAS0,r4
1384	tlbre
1385	mfspr	r5,SPRN_MAS1
1386	rlwinm	r5,r5,0,2,31	/* clear IPROT and VALID */
1387	mtspr	SPRN_MAS1,r5
1388	tlbwe
1389	sync
1390	isync
1391
1392	/* We translate LR and return */
1393	tovirt(r8,r8)
1394	mtlr	r8
1395	blr
1396
1397have_hes:
1398	/* Setup MAS 0,1,2,3 and 7 for tlbwe of a 1G entry that maps the
1399	 * kernel linear mapping. We also set MAS8 once for all here though
1400	 * that will have to be made dependent on whether we are running under
1401	 * a hypervisor I suppose.
1402	 */
1403
1404	/* BEWARE, MAGIC
1405	 * This code is called as an ordinary function on the boot CPU. But to
1406	 * avoid duplication, this code is also used in SCOM bringup of
1407	 * secondary CPUs. We read the code between the initial_tlb_code_start
1408	 * and initial_tlb_code_end labels one instruction at a time and RAM it
1409	 * into the new core via SCOM. That doesn't process branches, so there
1410	 * must be none between those two labels. It also means if this code
1411	 * ever takes any parameters, the SCOM code must also be updated to
1412	 * provide them.
1413	 */
1414	.globl a2_tlbinit_code_start
1415a2_tlbinit_code_start:
1416
1417	ori	r11,r3,MAS0_WQ_ALLWAYS
1418	oris	r11,r11,MAS0_ESEL(3)@h /* Use way 3: workaround A2 erratum 376 */
1419	mtspr	SPRN_MAS0,r11
1420	lis	r3,(MAS1_VALID | MAS1_IPROT)@h
1421	ori	r3,r3,BOOK3E_PAGESZ_1GB << MAS1_TSIZE_SHIFT
1422	mtspr	SPRN_MAS1,r3
1423	LOAD_REG_IMMEDIATE(r3, PAGE_OFFSET | MAS2_M)
1424	mtspr	SPRN_MAS2,r3
1425	li	r3,MAS3_SR | MAS3_SW | MAS3_SX
1426	mtspr	SPRN_MAS7_MAS3,r3
1427	li	r3,0
1428	mtspr	SPRN_MAS8,r3
1429
1430	/* Write the TLB entry */
1431	tlbwe
1432
1433	.globl a2_tlbinit_after_linear_map
1434a2_tlbinit_after_linear_map:
1435
1436	/* Now we branch the new virtual address mapped by this entry */
1437	LOAD_REG_IMMEDIATE(r3,1f)
1438	mtctr	r3
1439	bctr
1440
14411:	/* We are now running at PAGE_OFFSET, clean the TLB of everything
1442	 * else (including IPROTed things left by firmware)
1443	 * r4 = TLBnCFG
1444	 * r3 = current address (more or less)
1445	 */
1446
1447	li	r5,0
1448	mtspr	SPRN_MAS6,r5
1449	tlbsx	0,r3
1450
1451	rlwinm	r9,r4,0,TLBnCFG_N_ENTRY
1452	rlwinm	r10,r4,8,0xff
1453	addi	r10,r10,-1	/* Get inner loop mask */
1454
1455	li	r3,1
1456
1457	mfspr	r5,SPRN_MAS1
1458	rlwinm	r5,r5,0,(~(MAS1_VALID|MAS1_IPROT))
1459
1460	mfspr	r6,SPRN_MAS2
1461	rldicr	r6,r6,0,51		/* Extract EPN */
1462
1463	mfspr	r7,SPRN_MAS0
1464	rlwinm	r7,r7,0,0xffff0fff	/* Clear HES and WQ */
1465
1466	rlwinm	r8,r7,16,0xfff		/* Extract ESEL */
1467
14682:	add	r4,r3,r8
1469	and	r4,r4,r10
1470
1471	rlwimi	r7,r4,16,MAS0_ESEL_MASK
1472
1473	mtspr	SPRN_MAS0,r7
1474	mtspr	SPRN_MAS1,r5
1475	mtspr	SPRN_MAS2,r6
1476	tlbwe
1477
1478	addi	r3,r3,1
1479	and.	r4,r3,r10
1480
1481	bne	3f
1482	addis	r6,r6,(1<<30)@h
14833:
1484	cmpw	r3,r9
1485	blt	2b
1486
1487	.globl  a2_tlbinit_after_iprot_flush
1488a2_tlbinit_after_iprot_flush:
1489
1490	PPC_TLBILX(0,0,R0)
1491	sync
1492	isync
1493
1494	.globl a2_tlbinit_code_end
1495a2_tlbinit_code_end:
1496
1497	/* We translate LR and return */
1498	mflr	r3
1499	tovirt(r3,r3)
1500	mtlr	r3
1501	blr
1502
1503/*
1504 * Main entry (boot CPU, thread 0)
1505 *
1506 * We enter here from head_64.S, possibly after the prom_init trampoline
1507 * with r3 and r4 already saved to r31 and 30 respectively and in 64 bits
1508 * mode. Anything else is as it was left by the bootloader
1509 *
1510 * Initial requirements of this port:
1511 *
1512 * - Kernel loaded at 0 physical
1513 * - A good lump of memory mapped 0:0 by UTLB entry 0
1514 * - MSR:IS & MSR:DS set to 0
1515 *
1516 * Note that some of the above requirements will be relaxed in the future
1517 * as the kernel becomes smarter at dealing with different initial conditions
1518 * but for now you have to be careful
1519 */
1520_GLOBAL(start_initialization_book3e)
1521	mflr	r28
1522
1523	/* First, we need to setup some initial TLBs to map the kernel
1524	 * text, data and bss at PAGE_OFFSET. We don't have a real mode
1525	 * and always use AS 0, so we just set it up to match our link
1526	 * address and never use 0 based addresses.
1527	 */
1528	bl	initial_tlb_book3e
1529
1530	/* Init global core bits */
1531	bl	init_core_book3e
1532
1533	/* Init per-thread bits */
1534	bl	init_thread_book3e
1535
1536	/* Return to common init code */
1537	tovirt(r28,r28)
1538	mtlr	r28
1539	blr
1540
1541
1542/*
1543 * Secondary core/processor entry
1544 *
1545 * This is entered for thread 0 of a secondary core, all other threads
1546 * are expected to be stopped. It's similar to start_initialization_book3e
1547 * except that it's generally entered from the holding loop in head_64.S
1548 * after CPUs have been gathered by Open Firmware.
1549 *
1550 * We assume we are in 32 bits mode running with whatever TLB entry was
1551 * set for us by the firmware or POR engine.
1552 */
1553_GLOBAL(book3e_secondary_core_init_tlb_set)
1554	li	r4,1
1555	b	generic_secondary_smp_init
1556
1557_GLOBAL(book3e_secondary_core_init)
1558	mflr	r28
1559
1560	/* Do we need to setup initial TLB entry ? */
1561	cmplwi	r4,0
1562	bne	2f
1563
1564	/* Setup TLB for this core */
1565	bl	initial_tlb_book3e
1566
1567	/* We can return from the above running at a different
1568	 * address, so recalculate r2 (TOC)
1569	 */
1570	bl	relative_toc
1571
1572	/* Init global core bits */
15732:	bl	init_core_book3e
1574
1575	/* Init per-thread bits */
15763:	bl	init_thread_book3e
1577
1578	/* Return to common init code at proper virtual address.
1579	 *
1580	 * Due to various previous assumptions, we know we entered this
1581	 * function at either the final PAGE_OFFSET mapping or using a
1582	 * 1:1 mapping at 0, so we don't bother doing a complicated check
1583	 * here, we just ensure the return address has the right top bits.
1584	 *
1585	 * Note that if we ever want to be smarter about where we can be
1586	 * started from, we have to be careful that by the time we reach
1587	 * the code below we may already be running at a different location
1588	 * than the one we were called from since initial_tlb_book3e can
1589	 * have moved us already.
1590	 */
1591	cmpdi	cr0,r28,0
1592	blt	1f
1593	lis	r3,PAGE_OFFSET@highest
1594	sldi	r3,r3,32
1595	or	r28,r28,r3
15961:	mtlr	r28
1597	blr
1598
1599_GLOBAL(book3e_secondary_thread_init)
1600	mflr	r28
1601	b	3b
1602
1603	.globl init_core_book3e
1604init_core_book3e:
1605	/* Establish the interrupt vector base */
1606	tovirt(r2,r2)
1607	LOAD_REG_ADDR(r3, interrupt_base_book3e)
1608	mtspr	SPRN_IVPR,r3
1609	sync
1610	blr
1611
1612init_thread_book3e:
1613	lis	r3,(SPRN_EPCR_ICM | SPRN_EPCR_GICM)@h
1614	mtspr	SPRN_EPCR,r3
1615
1616	/* Make sure interrupts are off */
1617	wrteei	0
1618
1619	/* disable all timers and clear out status */
1620	li	r3,0
1621	mtspr	SPRN_TCR,r3
1622	mfspr	r3,SPRN_TSR
1623	mtspr	SPRN_TSR,r3
1624
1625	blr
1626
1627_GLOBAL(__setup_base_ivors)
1628	SET_IVOR(0, 0x020) /* Critical Input */
1629	SET_IVOR(1, 0x000) /* Machine Check */
1630	SET_IVOR(2, 0x060) /* Data Storage */ 
1631	SET_IVOR(3, 0x080) /* Instruction Storage */
1632	SET_IVOR(4, 0x0a0) /* External Input */ 
1633	SET_IVOR(5, 0x0c0) /* Alignment */ 
1634	SET_IVOR(6, 0x0e0) /* Program */ 
1635	SET_IVOR(7, 0x100) /* FP Unavailable */ 
1636	SET_IVOR(8, 0x120) /* System Call */ 
1637	SET_IVOR(9, 0x140) /* Auxiliary Processor Unavailable */ 
1638	SET_IVOR(10, 0x160) /* Decrementer */ 
1639	SET_IVOR(11, 0x180) /* Fixed Interval Timer */ 
1640	SET_IVOR(12, 0x1a0) /* Watchdog Timer */ 
1641	SET_IVOR(13, 0x1c0) /* Data TLB Error */ 
1642	SET_IVOR(14, 0x1e0) /* Instruction TLB Error */
1643	SET_IVOR(15, 0x040) /* Debug */
1644
1645	sync
1646
1647	blr
1648
1649_GLOBAL(setup_altivec_ivors)
1650	SET_IVOR(32, 0x200) /* AltiVec Unavailable */
1651	SET_IVOR(33, 0x220) /* AltiVec Assist */
1652	blr
1653
1654_GLOBAL(setup_perfmon_ivor)
1655	SET_IVOR(35, 0x260) /* Performance Monitor */
1656	blr
1657
1658_GLOBAL(setup_doorbell_ivors)
1659	SET_IVOR(36, 0x280) /* Processor Doorbell */
1660	SET_IVOR(37, 0x2a0) /* Processor Doorbell Crit */
1661	blr
1662
1663_GLOBAL(setup_ehv_ivors)
1664	SET_IVOR(40, 0x300) /* Embedded Hypervisor System Call */
1665	SET_IVOR(41, 0x320) /* Embedded Hypervisor Privilege */
1666	SET_IVOR(38, 0x2c0) /* Guest Processor Doorbell */
1667	SET_IVOR(39, 0x2e0) /* Guest Processor Doorbell Crit/MC */
1668	blr
1669
1670_GLOBAL(setup_lrat_ivor)
1671	SET_IVOR(42, 0x340) /* LRAT Error */
1672	blr