1/*
   2 *  PowerPC version
   3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   4 *  Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
   5 *    Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
   6 *  Adapted for Power Macintosh by Paul Mackerras.
   7 *  Low-level exception handlers and MMU support
   8 *  rewritten by Paul Mackerras.
   9 *    Copyright (C) 1996 Paul Mackerras.
  10 *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
  11 *
  12 *  This file contains the system call entry code, context switch
  13 *  code, and exception/interrupt return code for PowerPC.
  14 *
  15 *  This program is free software; you can redistribute it and/or
  16 *  modify it under the terms of the GNU General Public License
  17 *  as published by the Free Software Foundation; either version
  18 *  2 of the License, or (at your option) any later version.
  19 *
  20 */
  21
  22#include <linux/errno.h>
  23#include <linux/err.h>
  24#include <linux/sys.h>
  25#include <linux/threads.h>
 
 
  26#include <asm/reg.h>
  27#include <asm/page.h>
  28#include <asm/mmu.h>
  29#include <asm/cputable.h>
  30#include <asm/thread_info.h>
  31#include <asm/ppc_asm.h>
  32#include <asm/asm-offsets.h>
  33#include <asm/unistd.h>
  34#include <asm/ptrace.h>
  35#include <asm/export.h>
 
 
 
 
 
 
  36
  37/*
  38 * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE.
 
 
  39 */
  40#if MSR_KERNEL >= 0x10000
  41#define LOAD_MSR_KERNEL(r, x)	lis r,(x)@h; ori r,r,(x)@l
  42#else
  43#define LOAD_MSR_KERNEL(r, x)	li r,(x)
  44#endif
  45
  46/*
  47 * Align to 4k in order to ensure that all functions modyfing srr0/srr1
  48 * fit into one page in order to not encounter a TLB miss between the
  49 * modification of srr0/srr1 and the associated rfi.
  50 */
  51	.align	12
  52
  53#ifdef CONFIG_BOOKE
  54	.globl	mcheck_transfer_to_handler
  55mcheck_transfer_to_handler:
  56	mfspr	r0,SPRN_DSRR0
  57	stw	r0,_DSRR0(r11)
  58	mfspr	r0,SPRN_DSRR1
  59	stw	r0,_DSRR1(r11)
  60	/* fall through */
  61
  62	.globl	debug_transfer_to_handler
  63debug_transfer_to_handler:
  64	mfspr	r0,SPRN_CSRR0
  65	stw	r0,_CSRR0(r11)
  66	mfspr	r0,SPRN_CSRR1
  67	stw	r0,_CSRR1(r11)
  68	/* fall through */
  69
  70	.globl	crit_transfer_to_handler
  71crit_transfer_to_handler:
  72#ifdef CONFIG_PPC_BOOK3E_MMU
  73	mfspr	r0,SPRN_MAS0
  74	stw	r0,MAS0(r11)
  75	mfspr	r0,SPRN_MAS1
  76	stw	r0,MAS1(r11)
  77	mfspr	r0,SPRN_MAS2
  78	stw	r0,MAS2(r11)
  79	mfspr	r0,SPRN_MAS3
  80	stw	r0,MAS3(r11)
  81	mfspr	r0,SPRN_MAS6
  82	stw	r0,MAS6(r11)
  83#ifdef CONFIG_PHYS_64BIT
  84	mfspr	r0,SPRN_MAS7
  85	stw	r0,MAS7(r11)
  86#endif /* CONFIG_PHYS_64BIT */
  87#endif /* CONFIG_PPC_BOOK3E_MMU */
  88#ifdef CONFIG_44x
  89	mfspr	r0,SPRN_MMUCR
  90	stw	r0,MMUCR(r11)
  91#endif
  92	mfspr	r0,SPRN_SRR0
  93	stw	r0,_SRR0(r11)
  94	mfspr	r0,SPRN_SRR1
  95	stw	r0,_SRR1(r11)
  96
  97	/* set the stack limit to the current stack
  98	 * and set the limit to protect the thread_info
  99	 * struct
 100	 */
 101	mfspr	r8,SPRN_SPRG_THREAD
 102	lwz	r0,KSP_LIMIT(r8)
 103	stw	r0,SAVED_KSP_LIMIT(r11)
 104	rlwimi	r0,r1,0,0,(31-THREAD_SHIFT)
 105	stw	r0,KSP_LIMIT(r8)
 106	/* fall through */
 107#endif
 108
 109#ifdef CONFIG_40x
 110	.globl	crit_transfer_to_handler
 111crit_transfer_to_handler:
 112	lwz	r0,crit_r10@l(0)
 113	stw	r0,GPR10(r11)
 114	lwz	r0,crit_r11@l(0)
 115	stw	r0,GPR11(r11)
 116	mfspr	r0,SPRN_SRR0
 117	stw	r0,crit_srr0@l(0)
 118	mfspr	r0,SPRN_SRR1
 119	stw	r0,crit_srr1@l(0)
 120
 121	/* set the stack limit to the current stack
 122	 * and set the limit to protect the thread_info
 123	 * struct
 124	 */
 125	mfspr	r8,SPRN_SPRG_THREAD
 126	lwz	r0,KSP_LIMIT(r8)
 127	stw	r0,saved_ksp_limit@l(0)
 128	rlwimi	r0,r1,0,0,(31-THREAD_SHIFT)
 129	stw	r0,KSP_LIMIT(r8)
 130	/* fall through */
 131#endif
 132
 133/*
 134 * This code finishes saving the registers to the exception frame
 135 * and jumps to the appropriate handler for the exception, turning
 136 * on address translation.
 137 * Note that we rely on the caller having set cr0.eq iff the exception
 138 * occurred in kernel mode (i.e. MSR:PR = 0).
 139 */
 140	.globl	transfer_to_handler_full
 141transfer_to_handler_full:
 142	SAVE_NVGPRS(r11)
 143	/* fall through */
 144
 145	.globl	transfer_to_handler
 146transfer_to_handler:
 147	stw	r2,GPR2(r11)
 148	stw	r12,_NIP(r11)
 149	stw	r9,_MSR(r11)
 150	andi.	r2,r9,MSR_PR
 151	mfctr	r12
 152	mfspr	r2,SPRN_XER
 153	stw	r12,_CTR(r11)
 154	stw	r2,_XER(r11)
 155	mfspr	r12,SPRN_SPRG_THREAD
 156	addi	r2,r12,-THREAD
 157	tovirt(r2,r2)			/* set r2 to current */
 158	beq	2f			/* if from user, fix up THREAD.regs */
 159	addi	r11,r1,STACK_FRAME_OVERHEAD
 160	stw	r11,PT_REGS(r12)
 161#if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
 162	/* Check to see if the dbcr0 register is set up to debug.  Use the
 163	   internal debug mode bit to do this. */
 164	lwz	r12,THREAD_DBCR0(r12)
 165	andis.	r12,r12,DBCR0_IDM@h
 166	beq+	3f
 167	/* From user and task is ptraced - load up global dbcr0 */
 168	li	r12,-1			/* clear all pending debug events */
 169	mtspr	SPRN_DBSR,r12
 170	lis	r11,global_dbcr0@ha
 171	tophys(r11,r11)
 172	addi	r11,r11,global_dbcr0@l
 173#ifdef CONFIG_SMP
 174	CURRENT_THREAD_INFO(r9, r1)
 175	lwz	r9,TI_CPU(r9)
 176	slwi	r9,r9,3
 177	add	r11,r11,r9
 178#endif
 179	lwz	r12,0(r11)
 180	mtspr	SPRN_DBCR0,r12
 181	lwz	r12,4(r11)
 182	addi	r12,r12,-1
 183	stw	r12,4(r11)
 184#endif
 185#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 186	CURRENT_THREAD_INFO(r9, r1)
 187	tophys(r9, r9)
 188	ACCOUNT_CPU_USER_ENTRY(r9, r11, r12)
 189#endif
 190
 191	b	3f
 192
 1932:	/* if from kernel, check interrupted DOZE/NAP mode and
 194         * check for stack overflow
 195         */
 196	lwz	r9,KSP_LIMIT(r12)
 197	cmplw	r1,r9			/* if r1 <= ksp_limit */
 198	ble-	stack_ovf		/* then the kernel stack overflowed */
 1995:
 200#if defined(CONFIG_6xx) || defined(CONFIG_E500)
 201	CURRENT_THREAD_INFO(r9, r1)
 202	tophys(r9,r9)			/* check local flags */
 203	lwz	r12,TI_LOCAL_FLAGS(r9)
 204	mtcrf	0x01,r12
 205	bt-	31-TLF_NAPPING,4f
 206	bt-	31-TLF_SLEEPING,7f
 207#endif /* CONFIG_6xx || CONFIG_E500 */
 208	.globl transfer_to_handler_cont
 209transfer_to_handler_cont:
 2103:
 211	mflr	r9
 212	lwz	r11,0(r9)		/* virtual address of handler */
 213	lwz	r9,4(r9)		/* where to go when done */
 214#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 215	mtspr	SPRN_NRI, r0
 216#endif
 217#ifdef CONFIG_TRACE_IRQFLAGS
 218	lis	r12,reenable_mmu@h
 219	ori	r12,r12,reenable_mmu@l
 220	mtspr	SPRN_SRR0,r12
 221	mtspr	SPRN_SRR1,r10
 222	SYNC
 223	RFI
 224reenable_mmu:				/* re-enable mmu so we can */
 225	mfmsr	r10
 226	lwz	r12,_MSR(r1)
 227	xor	r10,r10,r12
 228	andi.	r10,r10,MSR_EE		/* Did EE change? */
 229	beq	1f
 230
 231	/*
 232	 * The trace_hardirqs_off will use CALLER_ADDR0 and CALLER_ADDR1.
 233	 * If from user mode there is only one stack frame on the stack, and
 234	 * accessing CALLER_ADDR1 will cause oops. So we need create a dummy
 235	 * stack frame to make trace_hardirqs_off happy.
 236	 *
 237	 * This is handy because we also need to save a bunch of GPRs,
 238	 * r3 can be different from GPR3(r1) at this point, r9 and r11
 239	 * contains the old MSR and handler address respectively,
 240	 * r4 & r5 can contain page fault arguments that need to be passed
 241	 * along as well. r12, CCR, CTR, XER etc... are left clobbered as
 242	 * they aren't useful past this point (aren't syscall arguments),
 243	 * the rest is restored from the exception frame.
 244	 */
 245	stwu	r1,-32(r1)
 246	stw	r9,8(r1)
 247	stw	r11,12(r1)
 248	stw	r3,16(r1)
 249	stw	r4,20(r1)
 250	stw	r5,24(r1)
 251	bl	trace_hardirqs_off
 252	lwz	r5,24(r1)
 253	lwz	r4,20(r1)
 254	lwz	r3,16(r1)
 255	lwz	r11,12(r1)
 256	lwz	r9,8(r1)
 257	addi	r1,r1,32
 258	lwz	r0,GPR0(r1)
 259	lwz	r6,GPR6(r1)
 260	lwz	r7,GPR7(r1)
 261	lwz	r8,GPR8(r1)
 2621:	mtctr	r11
 263	mtlr	r9
 264	bctr				/* jump to handler */
 265#else /* CONFIG_TRACE_IRQFLAGS */
 266	mtspr	SPRN_SRR0,r11
 267	mtspr	SPRN_SRR1,r10
 268	mtlr	r9
 269	SYNC
 270	RFI				/* jump to handler, enable MMU */
 271#endif /* CONFIG_TRACE_IRQFLAGS */
 272
 273#if defined (CONFIG_6xx) || defined(CONFIG_E500)
 2744:	rlwinm	r12,r12,0,~_TLF_NAPPING
 275	stw	r12,TI_LOCAL_FLAGS(r9)
 276	b	power_save_ppc32_restore
 277
 2787:	rlwinm	r12,r12,0,~_TLF_SLEEPING
 279	stw	r12,TI_LOCAL_FLAGS(r9)
 280	lwz	r9,_MSR(r11)		/* if sleeping, clear MSR.EE */
 281	rlwinm	r9,r9,0,~MSR_EE
 282	lwz	r12,_LINK(r11)		/* and return to address in LR */
 
 283	b	fast_exception_return
 284#endif
 
 285
 286/*
 287 * On kernel stack overflow, load up an initial stack pointer
 288 * and call StackOverflow(regs), which should not return.
 289 */
 290stack_ovf:
 291	/* sometimes we use a statically-allocated stack, which is OK. */
 292	lis	r12,_end@h
 293	ori	r12,r12,_end@l
 294	cmplw	r1,r12
 295	ble	5b			/* r1 <= &_end is OK */
 296	SAVE_NVGPRS(r11)
 297	addi	r3,r1,STACK_FRAME_OVERHEAD
 298	lis	r1,init_thread_union@ha
 299	addi	r1,r1,init_thread_union@l
 300	addi	r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
 301	lis	r9,StackOverflow@ha
 302	addi	r9,r9,StackOverflow@l
 303	LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 304#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 305	mtspr	SPRN_NRI, r0
 306#endif
 307	mtspr	SPRN_SRR0,r9
 308	mtspr	SPRN_SRR1,r10
 309	SYNC
 310	RFI
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 311
 312/*
 313 * Handle a system call.
 314 */
 315	.stabs	"arch/powerpc/kernel/",N_SO,0,0,0f
 316	.stabs	"entry_32.S",N_SO,0,0,0f
 3170:
 318
 319_GLOBAL(DoSyscall)
 320	stw	r3,ORIG_GPR3(r1)
 321	li	r12,0
 322	stw	r12,RESULT(r1)
 323	lwz	r11,_CCR(r1)	/* Clear SO bit in CR */
 324	rlwinm	r11,r11,0,4,2
 325	stw	r11,_CCR(r1)
 326#ifdef CONFIG_TRACE_IRQFLAGS
 327	/* Return from syscalls can (and generally will) hard enable
 328	 * interrupts. You aren't supposed to call a syscall with
 329	 * interrupts disabled in the first place. However, to ensure
 330	 * that we get it right vs. lockdep if it happens, we force
 331	 * that hard enable here with appropriate tracing if we see
 332	 * that we have been called with interrupts off
 333	 */
 334	mfmsr	r11
 335	andi.	r12,r11,MSR_EE
 336	bne+	1f
 337	/* We came in with interrupts disabled, we enable them now */
 338	bl	trace_hardirqs_on
 339	mfmsr	r11
 340	lwz	r0,GPR0(r1)
 341	lwz	r3,GPR3(r1)
 342	lwz	r4,GPR4(r1)
 343	ori	r11,r11,MSR_EE
 344	lwz	r5,GPR5(r1)
 345	lwz	r6,GPR6(r1)
 346	lwz	r7,GPR7(r1)
 347	lwz	r8,GPR8(r1)
 348	mtmsr	r11
 3491:
 350#endif /* CONFIG_TRACE_IRQFLAGS */
 351	CURRENT_THREAD_INFO(r10, r1)
 352	lwz	r11,TI_FLAGS(r10)
 353	andi.	r11,r11,_TIF_SYSCALL_DOTRACE
 354	bne-	syscall_dotrace
 355syscall_dotrace_cont:
 356	cmplwi	0,r0,NR_syscalls
 357	lis	r10,sys_call_table@h
 358	ori	r10,r10,sys_call_table@l
 359	slwi	r0,r0,2
 360	bge-	66f
 361	lwzx	r10,r10,r0	/* Fetch system call handler [ptr] */
 362	mtlr	r10
 363	addi	r9,r1,STACK_FRAME_OVERHEAD
 364	PPC440EP_ERR42
 365	blrl			/* Call handler */
 366	.globl	ret_from_syscall
 367ret_from_syscall:
 368	mr	r6,r3
 369	CURRENT_THREAD_INFO(r12, r1)
 370	/* disable interrupts so current_thread_info()->flags can't change */
 371	LOAD_MSR_KERNEL(r10,MSR_KERNEL)	/* doesn't include MSR_EE */
 372	/* Note: We don't bother telling lockdep about it */
 373	SYNC
 374	MTMSRD(r10)
 375	lwz	r9,TI_FLAGS(r12)
 376	li	r8,-MAX_ERRNO
 377	andi.	r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
 378	bne-	syscall_exit_work
 379	cmplw	0,r3,r8
 380	blt+	syscall_exit_cont
 381	lwz	r11,_CCR(r1)			/* Load CR */
 382	neg	r3,r3
 383	oris	r11,r11,0x1000	/* Set SO bit in CR */
 384	stw	r11,_CCR(r1)
 385syscall_exit_cont:
 386	lwz	r8,_MSR(r1)
 387#ifdef CONFIG_TRACE_IRQFLAGS
 388	/* If we are going to return from the syscall with interrupts
 389	 * off, we trace that here. It shouldn't happen though but we
 390	 * want to catch the bugger if it does right ?
 391	 */
 392	andi.	r10,r8,MSR_EE
 393	bne+	1f
 394	stw	r3,GPR3(r1)
 395	bl      trace_hardirqs_off
 396	lwz	r3,GPR3(r1)
 3971:
 398#endif /* CONFIG_TRACE_IRQFLAGS */
 399#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
 400	/* If the process has its own DBCR0 value, load it up.  The internal
 401	   debug mode bit tells us that dbcr0 should be loaded. */
 402	lwz	r0,THREAD+THREAD_DBCR0(r2)
 403	andis.	r10,r0,DBCR0_IDM@h
 404	bnel-	load_dbcr0
 405#endif
 406#ifdef CONFIG_44x
 407BEGIN_MMU_FTR_SECTION
 408	lis	r4,icache_44x_need_flush@ha
 409	lwz	r5,icache_44x_need_flush@l(r4)
 410	cmplwi	cr0,r5,0
 411	bne-	2f
 4121:
 413END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
 414#endif /* CONFIG_44x */
 415BEGIN_FTR_SECTION
 416	lwarx	r7,0,r1
 417END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
 418	stwcx.	r0,0,r1			/* to clear the reservation */
 419#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 420	andi.	r4,r8,MSR_PR
 421	beq	3f
 422	CURRENT_THREAD_INFO(r4, r1)
 423	ACCOUNT_CPU_USER_EXIT(r4, r5, r7)
 4243:
 425#endif
 426	lwz	r4,_LINK(r1)
 427	lwz	r5,_CCR(r1)
 428	mtlr	r4
 429	mtcr	r5
 430	lwz	r7,_NIP(r1)
 431	lwz	r2,GPR2(r1)
 432	lwz	r1,GPR1(r1)
 433#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 434	mtspr	SPRN_NRI, r0
 435#endif
 436	mtspr	SPRN_SRR0,r7
 437	mtspr	SPRN_SRR1,r8
 438	SYNC
 439	RFI
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 440#ifdef CONFIG_44x
 4412:	li	r7,0
 
 442	iccci	r0,r0
 443	stw	r7,icache_44x_need_flush@l(r4)
 444	b	1b
 445#endif  /* CONFIG_44x */
 446
 44766:	li	r3,-ENOSYS
 448	b	ret_from_syscall
 449
 450	.globl	ret_from_fork
 451ret_from_fork:
 452	REST_NVGPRS(r1)
 453	bl	schedule_tail
 454	li	r3,0
 455	b	ret_from_syscall
 456
 457	.globl	ret_from_kernel_thread
 458ret_from_kernel_thread:
 459	REST_NVGPRS(r1)
 460	bl	schedule_tail
 461	mtlr	r14
 462	mr	r3,r15
 463	PPC440EP_ERR42
 464	blrl
 465	li	r3,0
 466	b	ret_from_syscall
 467
 468/* Traced system call support */
 469syscall_dotrace:
 470	SAVE_NVGPRS(r1)
 471	li	r0,0xc00
 472	stw	r0,_TRAP(r1)
 473	addi	r3,r1,STACK_FRAME_OVERHEAD
 474	bl	do_syscall_trace_enter
 475	/*
 476	 * Restore argument registers possibly just changed.
 477	 * We use the return value of do_syscall_trace_enter
 478	 * for call number to look up in the table (r0).
 479	 */
 480	mr	r0,r3
 481	lwz	r3,GPR3(r1)
 482	lwz	r4,GPR4(r1)
 483	lwz	r5,GPR5(r1)
 484	lwz	r6,GPR6(r1)
 485	lwz	r7,GPR7(r1)
 486	lwz	r8,GPR8(r1)
 487	REST_NVGPRS(r1)
 488
 489	cmplwi	r0,NR_syscalls
 490	/* Return code is already in r3 thanks to do_syscall_trace_enter() */
 491	bge-	ret_from_syscall
 492	b	syscall_dotrace_cont
 493
 494syscall_exit_work:
 495	andi.	r0,r9,_TIF_RESTOREALL
 496	beq+	0f
 497	REST_NVGPRS(r1)
 498	b	2f
 4990:	cmplw	0,r3,r8
 500	blt+	1f
 501	andi.	r0,r9,_TIF_NOERROR
 502	bne-	1f
 503	lwz	r11,_CCR(r1)			/* Load CR */
 504	neg	r3,r3
 505	oris	r11,r11,0x1000	/* Set SO bit in CR */
 506	stw	r11,_CCR(r1)
 507
 5081:	stw	r6,RESULT(r1)	/* Save result */
 509	stw	r3,GPR3(r1)	/* Update return value */
 5102:	andi.	r0,r9,(_TIF_PERSYSCALL_MASK)
 511	beq	4f
 512
 513	/* Clear per-syscall TIF flags if any are set.  */
 514
 515	li	r11,_TIF_PERSYSCALL_MASK
 516	addi	r12,r12,TI_FLAGS
 5173:	lwarx	r8,0,r12
 518	andc	r8,r8,r11
 519#ifdef CONFIG_IBM405_ERR77
 520	dcbt	0,r12
 521#endif
 522	stwcx.	r8,0,r12
 523	bne-	3b
 524	subi	r12,r12,TI_FLAGS
 525	
 5264:	/* Anything which requires enabling interrupts? */
 527	andi.	r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
 528	beq	ret_from_except
 529
 530	/* Re-enable interrupts. There is no need to trace that with
 531	 * lockdep as we are supposed to have IRQs on at this point
 532	 */
 533	ori	r10,r10,MSR_EE
 534	SYNC
 535	MTMSRD(r10)
 536
 537	/* Save NVGPRS if they're not saved already */
 538	lwz	r4,_TRAP(r1)
 539	andi.	r4,r4,1
 540	beq	5f
 541	SAVE_NVGPRS(r1)
 542	li	r4,0xc00
 543	stw	r4,_TRAP(r1)
 5445:
 545	addi	r3,r1,STACK_FRAME_OVERHEAD
 546	bl	do_syscall_trace_leave
 547	b	ret_from_except_full
 548
 549/*
 550 * The fork/clone functions need to copy the full register set into
 551 * the child process. Therefore we need to save all the nonvolatile
 552 * registers (r13 - r31) before calling the C code.
 553 */
 554	.globl	ppc_fork
 555ppc_fork:
 556	SAVE_NVGPRS(r1)
 557	lwz	r0,_TRAP(r1)
 558	rlwinm	r0,r0,0,0,30		/* clear LSB to indicate full */
 559	stw	r0,_TRAP(r1)		/* register set saved */
 560	b	sys_fork
 561
 562	.globl	ppc_vfork
 563ppc_vfork:
 564	SAVE_NVGPRS(r1)
 565	lwz	r0,_TRAP(r1)
 566	rlwinm	r0,r0,0,0,30		/* clear LSB to indicate full */
 567	stw	r0,_TRAP(r1)		/* register set saved */
 568	b	sys_vfork
 569
 570	.globl	ppc_clone
 571ppc_clone:
 572	SAVE_NVGPRS(r1)
 573	lwz	r0,_TRAP(r1)
 574	rlwinm	r0,r0,0,0,30		/* clear LSB to indicate full */
 575	stw	r0,_TRAP(r1)		/* register set saved */
 576	b	sys_clone
 577
 578	.globl	ppc_swapcontext
 579ppc_swapcontext:
 580	SAVE_NVGPRS(r1)
 581	lwz	r0,_TRAP(r1)
 582	rlwinm	r0,r0,0,0,30		/* clear LSB to indicate full */
 583	stw	r0,_TRAP(r1)		/* register set saved */
 584	b	sys_swapcontext
 585
 586/*
 587 * Top-level page fault handling.
 588 * This is in assembler because if do_page_fault tells us that
 589 * it is a bad kernel page fault, we want to save the non-volatile
 590 * registers before calling bad_page_fault.
 591 */
 592	.globl	handle_page_fault
 593handle_page_fault:
 594	stw	r4,_DAR(r1)
 595	addi	r3,r1,STACK_FRAME_OVERHEAD
 596#ifdef CONFIG_6xx
 597	andis.  r0,r5,DSISR_DABRMATCH@h
 598	bne-    handle_dabr_fault
 599#endif
 600	bl	do_page_fault
 601	cmpwi	r3,0
 602	beq+	ret_from_except
 603	SAVE_NVGPRS(r1)
 604	lwz	r0,_TRAP(r1)
 605	clrrwi	r0,r0,1
 606	stw	r0,_TRAP(r1)
 607	mr	r5,r3
 608	addi	r3,r1,STACK_FRAME_OVERHEAD
 609	lwz	r4,_DAR(r1)
 610	bl	bad_page_fault
 611	b	ret_from_except_full
 612
 613#ifdef CONFIG_6xx
 614	/* We have a data breakpoint exception - handle it */
 615handle_dabr_fault:
 616	SAVE_NVGPRS(r1)
 617	lwz	r0,_TRAP(r1)
 618	clrrwi	r0,r0,1
 619	stw	r0,_TRAP(r1)
 620	bl      do_break
 621	b	ret_from_except_full
 622#endif
 623
 624/*
 625 * This routine switches between two different tasks.  The process
 626 * state of one is saved on its kernel stack.  Then the state
 627 * of the other is restored from its kernel stack.  The memory
 628 * management hardware is updated to the second process's state.
 629 * Finally, we can return to the second process.
 630 * On entry, r3 points to the THREAD for the current task, r4
 631 * points to the THREAD for the new task.
 632 *
 633 * This routine is always called with interrupts disabled.
 634 *
 635 * Note: there are two ways to get to the "going out" portion
 636 * of this code; either by coming in via the entry (_switch)
 637 * or via "fork" which must set up an environment equivalent
 638 * to the "_switch" path.  If you change this , you'll have to
 639 * change the fork code also.
 640 *
 641 * The code which creates the new task context is in 'copy_thread'
 642 * in arch/ppc/kernel/process.c
 643 */
 644_GLOBAL(_switch)
 645	stwu	r1,-INT_FRAME_SIZE(r1)
 646	mflr	r0
 647	stw	r0,INT_FRAME_SIZE+4(r1)
 648	/* r3-r12 are caller saved -- Cort */
 649	SAVE_NVGPRS(r1)
 650	stw	r0,_NIP(r1)	/* Return to switch caller */
 651	mfmsr	r11
 652	li	r0,MSR_FP	/* Disable floating-point */
 653#ifdef CONFIG_ALTIVEC
 654BEGIN_FTR_SECTION
 655	oris	r0,r0,MSR_VEC@h	/* Disable altivec */
 656	mfspr	r12,SPRN_VRSAVE	/* save vrsave register value */
 657	stw	r12,THREAD+THREAD_VRSAVE(r2)
 658END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 659#endif /* CONFIG_ALTIVEC */
 660#ifdef CONFIG_SPE
 661BEGIN_FTR_SECTION
 662	oris	r0,r0,MSR_SPE@h	 /* Disable SPE */
 663	mfspr	r12,SPRN_SPEFSCR /* save spefscr register value */
 664	stw	r12,THREAD+THREAD_SPEFSCR(r2)
 665END_FTR_SECTION_IFSET(CPU_FTR_SPE)
 666#endif /* CONFIG_SPE */
 667	and.	r0,r0,r11	/* FP or altivec or SPE enabled? */
 668	beq+	1f
 669	andc	r11,r11,r0
 670	MTMSRD(r11)
 671	isync
 6721:	stw	r11,_MSR(r1)
 673	mfcr	r10
 674	stw	r10,_CCR(r1)
 675	stw	r1,KSP(r3)	/* Set old stack pointer */
 676
 677#ifdef CONFIG_SMP
 678	/* We need a sync somewhere here to make sure that if the
 679	 * previous task gets rescheduled on another CPU, it sees all
 680	 * stores it has performed on this one.
 681	 */
 682	sync
 683#endif /* CONFIG_SMP */
 684
 685	tophys(r0,r4)
 686	mtspr	SPRN_SPRG_THREAD,r0	/* Update current THREAD phys addr */
 687	lwz	r1,KSP(r4)	/* Load new stack pointer */
 688
 689	/* save the old current 'last' for return value */
 690	mr	r3,r2
 691	addi	r2,r4,-THREAD	/* Update current */
 692
 693#ifdef CONFIG_ALTIVEC
 694BEGIN_FTR_SECTION
 695	lwz	r0,THREAD+THREAD_VRSAVE(r2)
 696	mtspr	SPRN_VRSAVE,r0		/* if G4, restore VRSAVE reg */
 697END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 698#endif /* CONFIG_ALTIVEC */
 699#ifdef CONFIG_SPE
 700BEGIN_FTR_SECTION
 701	lwz	r0,THREAD+THREAD_SPEFSCR(r2)
 702	mtspr	SPRN_SPEFSCR,r0		/* restore SPEFSCR reg */
 703END_FTR_SECTION_IFSET(CPU_FTR_SPE)
 704#endif /* CONFIG_SPE */
 705
 706	lwz	r0,_CCR(r1)
 707	mtcrf	0xFF,r0
 708	/* r3-r12 are destroyed -- Cort */
 709	REST_NVGPRS(r1)
 710
 711	lwz	r4,_NIP(r1)	/* Return to _switch caller in new task */
 712	mtlr	r4
 713	addi	r1,r1,INT_FRAME_SIZE
 714	blr
 715
 716	.globl	fast_exception_return
 717fast_exception_return:
 718#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
 719	andi.	r10,r9,MSR_RI		/* check for recoverable interrupt */
 720	beq	1f			/* if not, we've got problems */
 721#endif
 722
 7232:	REST_4GPRS(3, r11)
 724	lwz	r10,_CCR(r11)
 725	REST_GPR(1, r11)
 726	mtcr	r10
 727	lwz	r10,_LINK(r11)
 728	mtlr	r10
 
 
 
 729	REST_GPR(10, r11)
 730#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 731	mtspr	SPRN_NRI, r0
 732#endif
 733	mtspr	SPRN_SRR1,r9
 734	mtspr	SPRN_SRR0,r12
 735	REST_GPR(9, r11)
 736	REST_GPR(12, r11)
 737	lwz	r11,GPR11(r11)
 738	SYNC
 739	RFI
 740
 741#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
 742/* check if the exception happened in a restartable section */
 7431:	lis	r3,exc_exit_restart_end@ha
 744	addi	r3,r3,exc_exit_restart_end@l
 745	cmplw	r12,r3
 746	bge	3f
 747	lis	r4,exc_exit_restart@ha
 748	addi	r4,r4,exc_exit_restart@l
 749	cmplw	r12,r4
 750	blt	3f
 751	lis	r3,fee_restarts@ha
 752	tophys(r3,r3)
 753	lwz	r5,fee_restarts@l(r3)
 754	addi	r5,r5,1
 755	stw	r5,fee_restarts@l(r3)
 756	mr	r12,r4		/* restart at exc_exit_restart */
 757	b	2b
 758
 759	.section .bss
 760	.align	2
 761fee_restarts:
 762	.space	4
 763	.previous
 764
 765/* aargh, a nonrecoverable interrupt, panic */
 766/* aargh, we don't know which trap this is */
 767/* but the 601 doesn't implement the RI bit, so assume it's OK */
 7683:
 769BEGIN_FTR_SECTION
 770	b	2b
 771END_FTR_SECTION_IFSET(CPU_FTR_601)
 772	li	r10,-1
 773	stw	r10,_TRAP(r11)
 774	addi	r3,r1,STACK_FRAME_OVERHEAD
 775	lis	r10,MSR_KERNEL@h
 776	ori	r10,r10,MSR_KERNEL@l
 777	bl	transfer_to_handler_full
 778	.long	nonrecoverable_exception
 779	.long	ret_from_except
 780#endif
 
 
 
 
 
 
 
 781
 782	.globl	ret_from_except_full
 783ret_from_except_full:
 784	REST_NVGPRS(r1)
 785	/* fall through */
 
 786
 787	.globl	ret_from_except
 788ret_from_except:
 789	/* Hard-disable interrupts so that current_thread_info()->flags
 790	 * can't change between when we test it and when we return
 791	 * from the interrupt. */
 792	/* Note: We don't bother telling lockdep about it */
 793	LOAD_MSR_KERNEL(r10,MSR_KERNEL)
 794	SYNC			/* Some chip revs have problems here... */
 795	MTMSRD(r10)		/* disable interrupts */
 796
 797	lwz	r3,_MSR(r1)	/* Returning to user mode? */
 798	andi.	r0,r3,MSR_PR
 799	beq	resume_kernel
 800
 801user_exc_return:		/* r10 contains MSR_KERNEL here */
 802	/* Check current_thread_info()->flags */
 803	CURRENT_THREAD_INFO(r9, r1)
 804	lwz	r9,TI_FLAGS(r9)
 805	andi.	r0,r9,_TIF_USER_WORK_MASK
 806	bne	do_work
 807
 808restore_user:
 809#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
 810	/* Check whether this process has its own DBCR0 value.  The internal
 811	   debug mode bit tells us that dbcr0 should be loaded. */
 812	lwz	r0,THREAD+THREAD_DBCR0(r2)
 813	andis.	r10,r0,DBCR0_IDM@h
 814	bnel-	load_dbcr0
 815#endif
 816#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 817	CURRENT_THREAD_INFO(r9, r1)
 818	ACCOUNT_CPU_USER_EXIT(r9, r10, r11)
 819#endif
 820
 821	b	restore
 
 
 
 
 
 822
 823/* N.B. the only way to get here is from the beq following ret_from_except. */
 824resume_kernel:
 825	/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
 826	CURRENT_THREAD_INFO(r9, r1)
 827	lwz	r8,TI_FLAGS(r9)
 828	andis.	r0,r8,_TIF_EMULATE_STACK_STORE@h
 829	beq+	1f
 830
 831	addi	r8,r1,INT_FRAME_SIZE	/* Get the kprobed function entry */
 832
 833	lwz	r3,GPR1(r1)
 834	subi	r3,r3,INT_FRAME_SIZE	/* dst: Allocate a trampoline exception frame */
 835	mr	r4,r1			/* src:  current exception frame */
 836	mr	r1,r3			/* Reroute the trampoline frame to r1 */
 837
 838	/* Copy from the original to the trampoline. */
 839	li	r5,INT_FRAME_SIZE/4	/* size: INT_FRAME_SIZE */
 840	li	r6,0			/* start offset: 0 */
 841	mtctr	r5
 8422:	lwzx	r0,r6,r4
 843	stwx	r0,r6,r3
 844	addi	r6,r6,4
 845	bdnz	2b
 846
 847	/* Do real store operation to complete stwu */
 848	lwz	r5,GPR1(r1)
 849	stw	r8,0(r5)
 850
 851	/* Clear _TIF_EMULATE_STACK_STORE flag */
 852	lis	r11,_TIF_EMULATE_STACK_STORE@h
 853	addi	r5,r9,TI_FLAGS
 8540:	lwarx	r8,0,r5
 855	andc	r8,r8,r11
 856#ifdef CONFIG_IBM405_ERR77
 857	dcbt	0,r5
 858#endif
 859	stwcx.	r8,0,r5
 860	bne-	0b
 8611:
 862
 863#ifdef CONFIG_PREEMPT
 864	/* check current_thread_info->preempt_count */
 865	lwz	r0,TI_PREEMPT(r9)
 866	cmpwi	0,r0,0		/* if non-zero, just restore regs and return */
 867	bne	restore
 868	andi.	r8,r8,_TIF_NEED_RESCHED
 869	beq+	restore
 870	lwz	r3,_MSR(r1)
 871	andi.	r0,r3,MSR_EE	/* interrupts off? */
 872	beq	restore		/* don't schedule if so */
 873#ifdef CONFIG_TRACE_IRQFLAGS
 874	/* Lockdep thinks irqs are enabled, we need to call
 875	 * preempt_schedule_irq with IRQs off, so we inform lockdep
 876	 * now that we -did- turn them off already
 877	 */
 878	bl	trace_hardirqs_off
 879#endif
 8801:	bl	preempt_schedule_irq
 881	CURRENT_THREAD_INFO(r9, r1)
 882	lwz	r3,TI_FLAGS(r9)
 883	andi.	r0,r3,_TIF_NEED_RESCHED
 884	bne-	1b
 885#ifdef CONFIG_TRACE_IRQFLAGS
 886	/* And now, to properly rebalance the above, we tell lockdep they
 887	 * are being turned back on, which will happen when we return
 888	 */
 889	bl	trace_hardirqs_on
 890#endif
 891#endif /* CONFIG_PREEMPT */
 892
 893	/* interrupts are hard-disabled at this point */
 894restore:
 895#ifdef CONFIG_44x
 896BEGIN_MMU_FTR_SECTION
 897	b	1f
 898END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
 899	lis	r4,icache_44x_need_flush@ha
 900	lwz	r5,icache_44x_need_flush@l(r4)
 901	cmplwi	cr0,r5,0
 902	beq+	1f
 903	li	r6,0
 904	iccci	r0,r0
 905	stw	r6,icache_44x_need_flush@l(r4)
 9061:
 907#endif  /* CONFIG_44x */
 908
 909	lwz	r9,_MSR(r1)
 910#ifdef CONFIG_TRACE_IRQFLAGS
 911	/* Lockdep doesn't know about the fact that IRQs are temporarily turned
 912	 * off in this assembly code while peeking at TI_FLAGS() and such. However
 913	 * we need to inform it if the exception turned interrupts off, and we
 914	 * are about to trun them back on.
 915	 *
 916	 * The problem here sadly is that we don't know whether the exceptions was
 917	 * one that turned interrupts off or not. So we always tell lockdep about
 918	 * turning them on here when we go back to wherever we came from with EE
 919	 * on, even if that may meen some redudant calls being tracked. Maybe later
 920	 * we could encode what the exception did somewhere or test the exception
 921	 * type in the pt_regs but that sounds overkill
 922	 */
 923	andi.	r10,r9,MSR_EE
 924	beq	1f
 925	/*
 926	 * Since the ftrace irqsoff latency trace checks CALLER_ADDR1,
 927	 * which is the stack frame here, we need to force a stack frame
 928	 * in case we came from user space.
 929	 */
 930	stwu	r1,-32(r1)
 931	mflr	r0
 932	stw	r0,4(r1)
 933	stwu	r1,-32(r1)
 934	bl	trace_hardirqs_on
 935	lwz	r1,0(r1)
 936	lwz	r1,0(r1)
 937	lwz	r9,_MSR(r1)
 9381:
 939#endif /* CONFIG_TRACE_IRQFLAGS */
 940
 941	lwz	r0,GPR0(r1)
 942	lwz	r2,GPR2(r1)
 943	REST_4GPRS(3, r1)
 944	REST_2GPRS(7, r1)
 945
 946	lwz	r10,_XER(r1)
 947	lwz	r11,_CTR(r1)
 948	mtspr	SPRN_XER,r10
 949	mtctr	r11
 950
 951	PPC405_ERR77(0,r1)
 952BEGIN_FTR_SECTION
 953	lwarx	r11,0,r1
 954END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
 955	stwcx.	r0,0,r1			/* to clear the reservation */
 
 
 
 
 
 
 
 956
 957#if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
 958	andi.	r10,r9,MSR_RI		/* check if this exception occurred */
 959	beql	nonrecoverable		/* at a bad place (MSR:RI = 0) */
 960
 961	lwz	r10,_CCR(r1)
 962	lwz	r11,_LINK(r1)
 963	mtcrf	0xFF,r10
 964	mtlr	r11
 965
 966	/*
 967	 * Once we put values in SRR0 and SRR1, we are in a state
 968	 * where exceptions are not recoverable, since taking an
 969	 * exception will trash SRR0 and SRR1.  Therefore we clear the
 970	 * MSR:RI bit to indicate this.  If we do take an exception,
 971	 * we can't return to the point of the exception but we
 972	 * can restart the exception exit path at the label
 973	 * exc_exit_restart below.  -- paulus
 974	 */
 975	LOAD_MSR_KERNEL(r10,MSR_KERNEL & ~MSR_RI)
 976	SYNC
 977	MTMSRD(r10)		/* clear the RI bit */
 978	.globl exc_exit_restart
 979exc_exit_restart:
 980	lwz	r12,_NIP(r1)
 981#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 982	mtspr	SPRN_NRI, r0
 
 
 
 
 983#endif
 984	mtspr	SPRN_SRR0,r12
 985	mtspr	SPRN_SRR1,r9
 986	REST_4GPRS(9, r1)
 987	lwz	r1,GPR1(r1)
 988	.globl exc_exit_restart_end
 989exc_exit_restart_end:
 990	SYNC
 991	RFI
 992
 993#else /* !(CONFIG_4xx || CONFIG_BOOKE) */
 994	/*
 995	 * This is a bit different on 4xx/Book-E because it doesn't have
 996	 * the RI bit in the MSR.
 997	 * The TLB miss handler checks if we have interrupted
 998	 * the exception exit path and restarts it if so
 999	 * (well maybe one day it will... :).
1000	 */
1001	lwz	r11,_LINK(r1)
1002	mtlr	r11
1003	lwz	r10,_CCR(r1)
1004	mtcrf	0xff,r10
1005	REST_2GPRS(9, r1)
1006	.globl exc_exit_restart
1007exc_exit_restart:
1008	lwz	r11,_NIP(r1)
1009	lwz	r12,_MSR(r1)
1010exc_exit_start:
1011	mtspr	SPRN_SRR0,r11
1012	mtspr	SPRN_SRR1,r12
1013	REST_2GPRS(11, r1)
1014	lwz	r1,GPR1(r1)
1015	.globl exc_exit_restart_end
1016exc_exit_restart_end:
1017	PPC405_ERR77_SYNC
1018	rfi
1019	b	.			/* prevent prefetch past rfi */
 
 
 
 
 
1020
1021/*
1022 * Returning from a critical interrupt in user mode doesn't need
1023 * to be any different from a normal exception.  For a critical
1024 * interrupt in the kernel, we just return (without checking for
1025 * preemption) since the interrupt may have happened at some crucial
1026 * place (e.g. inside the TLB miss handler), and because we will be
1027 * running with r1 pointing into critical_stack, not the current
1028 * process's kernel stack (and therefore current_thread_info() will
1029 * give the wrong answer).
1030 * We have to restore various SPRs that may have been in use at the
1031 * time of the critical interrupt.
1032 *
1033 */
1034#ifdef CONFIG_40x
1035#define PPC_40x_TURN_OFF_MSR_DR						    \
1036	/* avoid any possible TLB misses here by turning off MSR.DR, we	    \
1037	 * assume the instructions here are mapped by a pinned TLB entry */ \
1038	li	r10,MSR_IR;						    \
1039	mtmsr	r10;							    \
1040	isync;								    \
1041	tophys(r1, r1);
1042#else
1043#define PPC_40x_TURN_OFF_MSR_DR
1044#endif
1045
1046#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi)	\
1047	REST_NVGPRS(r1);						\
1048	lwz	r3,_MSR(r1);						\
1049	andi.	r3,r3,MSR_PR;						\
1050	LOAD_MSR_KERNEL(r10,MSR_KERNEL);				\
1051	bne	user_exc_return;					\
1052	lwz	r0,GPR0(r1);						\
1053	lwz	r2,GPR2(r1);						\
1054	REST_4GPRS(3, r1);						\
1055	REST_2GPRS(7, r1);						\
1056	lwz	r10,_XER(r1);						\
1057	lwz	r11,_CTR(r1);						\
1058	mtspr	SPRN_XER,r10;						\
1059	mtctr	r11;							\
1060	PPC405_ERR77(0,r1);						\
1061	stwcx.	r0,0,r1;		/* to clear the reservation */	\
1062	lwz	r11,_LINK(r1);						\
1063	mtlr	r11;							\
1064	lwz	r10,_CCR(r1);						\
1065	mtcrf	0xff,r10;						\
1066	PPC_40x_TURN_OFF_MSR_DR;					\
1067	lwz	r9,_DEAR(r1);						\
1068	lwz	r10,_ESR(r1);						\
1069	mtspr	SPRN_DEAR,r9;						\
1070	mtspr	SPRN_ESR,r10;						\
1071	lwz	r11,_NIP(r1);						\
1072	lwz	r12,_MSR(r1);						\
1073	mtspr	exc_lvl_srr0,r11;					\
1074	mtspr	exc_lvl_srr1,r12;					\
1075	lwz	r9,GPR9(r1);						\
1076	lwz	r12,GPR12(r1);						\
1077	lwz	r10,GPR10(r1);						\
1078	lwz	r11,GPR11(r1);						\
1079	lwz	r1,GPR1(r1);						\
1080	PPC405_ERR77_SYNC;						\
1081	exc_lvl_rfi;							\
1082	b	.;		/* prevent prefetch past exc_lvl_rfi */
1083
1084#define	RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1)			\
1085	lwz	r9,_##exc_lvl_srr0(r1);					\
1086	lwz	r10,_##exc_lvl_srr1(r1);				\
1087	mtspr	SPRN_##exc_lvl_srr0,r9;					\
1088	mtspr	SPRN_##exc_lvl_srr1,r10;
1089
1090#if defined(CONFIG_PPC_BOOK3E_MMU)
1091#ifdef CONFIG_PHYS_64BIT
1092#define	RESTORE_MAS7							\
1093	lwz	r11,MAS7(r1);						\
1094	mtspr	SPRN_MAS7,r11;
1095#else
1096#define	RESTORE_MAS7
1097#endif /* CONFIG_PHYS_64BIT */
1098#define RESTORE_MMU_REGS						\
1099	lwz	r9,MAS0(r1);						\
1100	lwz	r10,MAS1(r1);						\
1101	lwz	r11,MAS2(r1);						\
1102	mtspr	SPRN_MAS0,r9;						\
1103	lwz	r9,MAS3(r1);						\
1104	mtspr	SPRN_MAS1,r10;						\
1105	lwz	r10,MAS6(r1);						\
1106	mtspr	SPRN_MAS2,r11;						\
1107	mtspr	SPRN_MAS3,r9;						\
1108	mtspr	SPRN_MAS6,r10;						\
1109	RESTORE_MAS7;
1110#elif defined(CONFIG_44x)
1111#define RESTORE_MMU_REGS						\
1112	lwz	r9,MMUCR(r1);						\
1113	mtspr	SPRN_MMUCR,r9;
1114#else
1115#define RESTORE_MMU_REGS
1116#endif
1117
1118#ifdef CONFIG_40x
1119	.globl	ret_from_crit_exc
1120ret_from_crit_exc:
1121	mfspr	r9,SPRN_SPRG_THREAD
1122	lis	r10,saved_ksp_limit@ha;
1123	lwz	r10,saved_ksp_limit@l(r10);
1124	tovirt(r9,r9);
1125	stw	r10,KSP_LIMIT(r9)
1126	lis	r9,crit_srr0@ha;
1127	lwz	r9,crit_srr0@l(r9);
1128	lis	r10,crit_srr1@ha;
1129	lwz	r10,crit_srr1@l(r10);
1130	mtspr	SPRN_SRR0,r9;
1131	mtspr	SPRN_SRR1,r10;
1132	RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
 
1133#endif /* CONFIG_40x */
1134
1135#ifdef CONFIG_BOOKE
1136	.globl	ret_from_crit_exc
1137ret_from_crit_exc:
1138	mfspr	r9,SPRN_SPRG_THREAD
1139	lwz	r10,SAVED_KSP_LIMIT(r1)
1140	stw	r10,KSP_LIMIT(r9)
1141	RESTORE_xSRR(SRR0,SRR1);
1142	RESTORE_MMU_REGS;
1143	RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
 
1144
1145	.globl	ret_from_debug_exc
1146ret_from_debug_exc:
1147	mfspr	r9,SPRN_SPRG_THREAD
1148	lwz	r10,SAVED_KSP_LIMIT(r1)
1149	stw	r10,KSP_LIMIT(r9)
1150	lwz	r9,THREAD_INFO-THREAD(r9)
1151	CURRENT_THREAD_INFO(r10, r1)
1152	lwz	r10,TI_PREEMPT(r10)
1153	stw	r10,TI_PREEMPT(r9)
1154	RESTORE_xSRR(SRR0,SRR1);
1155	RESTORE_xSRR(CSRR0,CSRR1);
1156	RESTORE_MMU_REGS;
1157	RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
 
1158
1159	.globl	ret_from_mcheck_exc
1160ret_from_mcheck_exc:
1161	mfspr	r9,SPRN_SPRG_THREAD
1162	lwz	r10,SAVED_KSP_LIMIT(r1)
1163	stw	r10,KSP_LIMIT(r9)
1164	RESTORE_xSRR(SRR0,SRR1);
1165	RESTORE_xSRR(CSRR0,CSRR1);
1166	RESTORE_xSRR(DSRR0,DSRR1);
1167	RESTORE_MMU_REGS;
1168	RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
 
1169#endif /* CONFIG_BOOKE */
1170
1171/*
1172 * Load the DBCR0 value for a task that is being ptraced,
1173 * having first saved away the global DBCR0.  Note that r0
1174 * has the dbcr0 value to set upon entry to this.
1175 */
1176load_dbcr0:
1177	mfmsr	r10		/* first disable debug exceptions */
1178	rlwinm	r10,r10,0,~MSR_DE
1179	mtmsr	r10
1180	isync
1181	mfspr	r10,SPRN_DBCR0
1182	lis	r11,global_dbcr0@ha
1183	addi	r11,r11,global_dbcr0@l
1184#ifdef CONFIG_SMP
1185	CURRENT_THREAD_INFO(r9, r1)
1186	lwz	r9,TI_CPU(r9)
1187	slwi	r9,r9,3
1188	add	r11,r11,r9
1189#endif
1190	stw	r10,0(r11)
1191	mtspr	SPRN_DBCR0,r0
1192	lwz	r10,4(r11)
1193	addi	r10,r10,1
1194	stw	r10,4(r11)
1195	li	r11,-1
1196	mtspr	SPRN_DBSR,r11	/* clear all pending debug events */
1197	blr
1198
1199	.section .bss
1200	.align	4
1201global_dbcr0:
1202	.space	8*NR_CPUS
1203	.previous
1204#endif /* !(CONFIG_4xx || CONFIG_BOOKE) */
1205
1206do_work:			/* r10 contains MSR_KERNEL here */
1207	andi.	r0,r9,_TIF_NEED_RESCHED
1208	beq	do_user_signal
1209
1210do_resched:			/* r10 contains MSR_KERNEL here */
1211	/* Note: We don't need to inform lockdep that we are enabling
1212	 * interrupts here. As far as it knows, they are already enabled
1213	 */
1214	ori	r10,r10,MSR_EE
1215	SYNC
1216	MTMSRD(r10)		/* hard-enable interrupts */
1217	bl	schedule
1218recheck:
1219	/* Note: And we don't tell it we are disabling them again
1220	 * neither. Those disable/enable cycles used to peek at
1221	 * TI_FLAGS aren't advertised.
1222	 */
1223	LOAD_MSR_KERNEL(r10,MSR_KERNEL)
1224	SYNC
1225	MTMSRD(r10)		/* disable interrupts */
1226	CURRENT_THREAD_INFO(r9, r1)
1227	lwz	r9,TI_FLAGS(r9)
1228	andi.	r0,r9,_TIF_NEED_RESCHED
1229	bne-	do_resched
1230	andi.	r0,r9,_TIF_USER_WORK_MASK
1231	beq	restore_user
1232do_user_signal:			/* r10 contains MSR_KERNEL here */
1233	ori	r10,r10,MSR_EE
1234	SYNC
1235	MTMSRD(r10)		/* hard-enable interrupts */
1236	/* save r13-r31 in the exception frame, if not already done */
1237	lwz	r3,_TRAP(r1)
1238	andi.	r0,r3,1
1239	beq	2f
1240	SAVE_NVGPRS(r1)
1241	rlwinm	r3,r3,0,0,30
1242	stw	r3,_TRAP(r1)
12432:	addi	r3,r1,STACK_FRAME_OVERHEAD
1244	mr	r4,r9
1245	bl	do_notify_resume
1246	REST_NVGPRS(r1)
1247	b	recheck
1248
1249/*
1250 * We come here when we are at the end of handling an exception
1251 * that occurred at a place where taking an exception will lose
1252 * state information, such as the contents of SRR0 and SRR1.
1253 */
1254nonrecoverable:
1255	lis	r10,exc_exit_restart_end@ha
1256	addi	r10,r10,exc_exit_restart_end@l
1257	cmplw	r12,r10
1258	bge	3f
1259	lis	r11,exc_exit_restart@ha
1260	addi	r11,r11,exc_exit_restart@l
1261	cmplw	r12,r11
1262	blt	3f
1263	lis	r10,ee_restarts@ha
1264	lwz	r12,ee_restarts@l(r10)
1265	addi	r12,r12,1
1266	stw	r12,ee_restarts@l(r10)
1267	mr	r12,r11		/* restart at exc_exit_restart */
1268	blr
12693:	/* OK, we can't recover, kill this process */
1270	/* but the 601 doesn't implement the RI bit, so assume it's OK */
1271BEGIN_FTR_SECTION
1272	blr
1273END_FTR_SECTION_IFSET(CPU_FTR_601)
1274	lwz	r3,_TRAP(r1)
1275	andi.	r0,r3,1
1276	beq	4f
1277	SAVE_NVGPRS(r1)
1278	rlwinm	r3,r3,0,0,30
1279	stw	r3,_TRAP(r1)
12804:	addi	r3,r1,STACK_FRAME_OVERHEAD
1281	bl	nonrecoverable_exception
1282	/* shouldn't return */
1283	b	4b
1284
1285	.section .bss
1286	.align	2
1287ee_restarts:
1288	.space	4
1289	.previous
1290
1291/*
1292 * PROM code for specific machines follows.  Put it
1293 * here so it's easy to add arch-specific sections later.
1294 * -- Cort
1295 */
1296#ifdef CONFIG_PPC_RTAS
1297/*
1298 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
1299 * called with the MMU off.
1300 */
1301_GLOBAL(enter_rtas)
1302	stwu	r1,-INT_FRAME_SIZE(r1)
1303	mflr	r0
1304	stw	r0,INT_FRAME_SIZE+4(r1)
1305	LOAD_REG_ADDR(r4, rtas)
1306	lis	r6,1f@ha	/* physical return address for rtas */
1307	addi	r6,r6,1f@l
1308	tophys(r6,r6)
1309	tophys(r7,r1)
1310	lwz	r8,RTASENTRY(r4)
1311	lwz	r4,RTASBASE(r4)
1312	mfmsr	r9
1313	stw	r9,8(r1)
1314	LOAD_MSR_KERNEL(r0,MSR_KERNEL)
1315	SYNC			/* disable interrupts so SRR0/1 */
1316	MTMSRD(r0)		/* don't get trashed */
1317	li	r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1318	mtlr	r6
1319	mtspr	SPRN_SPRG_RTAS,r7
1320	mtspr	SPRN_SRR0,r8
1321	mtspr	SPRN_SRR1,r9
1322	RFI
13231:	tophys(r9,r1)
1324	lwz	r8,INT_FRAME_SIZE+4(r9)	/* get return address */
1325	lwz	r9,8(r9)	/* original msr value */
1326	addi	r1,r1,INT_FRAME_SIZE
1327	li	r0,0
1328	mtspr	SPRN_SPRG_RTAS,r0
1329	mtspr	SPRN_SRR0,r8
1330	mtspr	SPRN_SRR1,r9
1331	RFI			/* return to caller */
1332
1333	.globl	machine_check_in_rtas
1334machine_check_in_rtas:
1335	twi	31,0,0
1336	/* XXX load up BATs and panic */
1337
1338#endif /* CONFIG_PPC_RTAS */