1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 *  PowerPC version
   4 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   5 *
   6 *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
   7 *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
   8 *  Adapted for Power Macintosh by Paul Mackerras.
   9 *  Low-level exception handlers and MMU support
  10 *  rewritten by Paul Mackerras.
  11 *    Copyright (C) 1996 Paul Mackerras.
  12 *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
  13 *
  14 *  This file contains the low-level support and setup for the
  15 *  PowerPC platform, including trap and interrupt dispatch.
  16 *  (The PPC 8xx embedded CPUs use head_8xx.S instead.)
  17 */
  18
  19#include <linux/init.h>
  20#include <linux/pgtable.h>
  21#include <linux/linkage.h>
  22
  23#include <asm/reg.h>
  24#include <asm/page.h>
  25#include <asm/mmu.h>
  26#include <asm/cputable.h>
  27#include <asm/cache.h>
  28#include <asm/thread_info.h>
  29#include <asm/ppc_asm.h>
  30#include <asm/asm-offsets.h>
  31#include <asm/ptrace.h>
  32#include <asm/bug.h>
  33#include <asm/kvm_book3s_asm.h>
  34#include <asm/feature-fixups.h>
  35#include <asm/interrupt.h>
  36
  37#include "head_32.h"
  38
  39#define LOAD_BAT(n, reg, RA, RB)	\
  40	/* see the comment for clear_bats() -- Cort */ \
  41	li	RA,0;			\
  42	mtspr	SPRN_IBAT##n##U,RA;	\
  43	mtspr	SPRN_DBAT##n##U,RA;	\
  44	lwz	RA,(n*16)+0(reg);	\
  45	lwz	RB,(n*16)+4(reg);	\
  46	mtspr	SPRN_IBAT##n##U,RA;	\
  47	mtspr	SPRN_IBAT##n##L,RB;	\
  48	lwz	RA,(n*16)+8(reg);	\
  49	lwz	RB,(n*16)+12(reg);	\
  50	mtspr	SPRN_DBAT##n##U,RA;	\
  51	mtspr	SPRN_DBAT##n##L,RB
  52
  53	__HEAD
  54_GLOBAL(_stext);
  55
  56/*
  57 * _start is defined this way because the XCOFF loader in the OpenFirmware
  58 * on the powermac expects the entry point to be a procedure descriptor.
  59 */
  60_GLOBAL(_start);
  61	/*
  62	 * These are here for legacy reasons, the kernel used to
  63	 * need to look like a coff function entry for the pmac
  64	 * but we're always started by some kind of bootloader now.
  65	 *  -- Cort
  66	 */
  67	nop	/* used by __secondary_hold on prep (mtx) and chrp smp */
  68	nop	/* used by __secondary_hold on prep (mtx) and chrp smp */
  69	nop
  70
  71/* PMAC
  72 * Enter here with the kernel text, data and bss loaded starting at
  73 * 0, running with virtual == physical mapping.
  74 * r5 points to the prom entry point (the client interface handler
  75 * address).  Address translation is turned on, with the prom
  76 * managing the hash table.  Interrupts are disabled.  The stack
  77 * pointer (r1) points to just below the end of the half-meg region
  78 * from 0x380000 - 0x400000, which is mapped in already.
  79 *
  80 * If we are booted from MacOS via BootX, we enter with the kernel
  81 * image loaded somewhere, and the following values in registers:
  82 *  r3: 'BooX' (0x426f6f58)
  83 *  r4: virtual address of boot_infos_t
  84 *  r5: 0
  85 *
  86 * PREP
  87 * This is jumped to on prep systems right after the kernel is relocated
  88 * to its proper place in memory by the boot loader.  The expected layout
  89 * of the regs is:
  90 *   r3: ptr to residual data
  91 *   r4: initrd_start or if no initrd then 0
  92 *   r5: initrd_end - unused if r4 is 0
  93 *   r6: Start of command line string
  94 *   r7: End of command line string
  95 *
  96 * This just gets a minimal mmu environment setup so we can call
  97 * start_here() to do the real work.
  98 * -- Cort
  99 */
 100
 101	.globl	__start
 102__start:
 103/*
 104 * We have to do any OF calls before we map ourselves to KERNELBASE,
 105 * because OF may have I/O devices mapped into that area
 106 * (particularly on CHRP).
 107 */
 108	cmpwi	0,r5,0
 109	beq	1f
 110
 111#ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
 112	/* find out where we are now */
 113	bcl	20,31,$+4
 1140:	mflr	r8			/* r8 = runtime addr here */
 115	addis	r8,r8,(_stext - 0b)@ha
 116	addi	r8,r8,(_stext - 0b)@l	/* current runtime base addr */
 117	bl	prom_init
 118#endif /* CONFIG_PPC_OF_BOOT_TRAMPOLINE */
 119
 120	/* We never return. We also hit that trap if trying to boot
 121	 * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */
 122	trap
 123
 124/*
 125 * Check for BootX signature when supporting PowerMac and branch to
 126 * appropriate trampoline if it's present
 127 */
 128#ifdef CONFIG_PPC_PMAC
 1291:	lis	r31,0x426f
 130	ori	r31,r31,0x6f58
 131	cmpw	0,r3,r31
 132	bne	1f
 133	bl	bootx_init
 134	trap
 135#endif /* CONFIG_PPC_PMAC */
 136
 1371:	mr	r31,r3			/* save device tree ptr */
 138	li	r24,0			/* cpu # */
 139
 140/*
 141 * early_init() does the early machine identification and does
 142 * the necessary low-level setup and clears the BSS
 143 *  -- Cort <cort@fsmlabs.com>
 144 */
 145	bl	early_init
 146
 147/* Switch MMU off, clear BATs and flush TLB. At this point, r3 contains
 148 * the physical address we are running at, returned by early_init()
 149 */
 150 	bl	mmu_off
 151__after_mmu_off:
 152	bl	clear_bats
 153	bl	flush_tlbs
 154
 155	bl	initial_bats
 156	bl	load_segment_registers
 157	bl	reloc_offset
 158	bl	early_hash_table
 159#if defined(CONFIG_BOOTX_TEXT)
 160	bl	setup_disp_bat
 161#endif
 162#ifdef CONFIG_PPC_EARLY_DEBUG_CPM
 163	bl	setup_cpm_bat
 164#endif
 165#ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
 166	bl	setup_usbgecko_bat
 167#endif
 168
 169/*
 170 * Call setup_cpu for CPU 0 and initialize 6xx Idle
 171 */
 172	bl	reloc_offset
 173	li	r24,0			/* cpu# */
 174	bl	call_setup_cpu		/* Call setup_cpu for this CPU */
 175	bl	reloc_offset
 176	bl	init_idle_6xx
 177
 178
 179/*
 180 * We need to run with _start at physical address 0.
 181 * On CHRP, we are loaded at 0x10000 since OF on CHRP uses
 182 * the exception vectors at 0 (and therefore this copy
 183 * overwrites OF's exception vectors with our own).
 184 * The MMU is off at this point.
 185 */
 186	bl	reloc_offset
 187	mr	r26,r3
 188	addis	r4,r3,KERNELBASE@h	/* current address of _start */
 189	lis	r5,PHYSICAL_START@h
 190	cmplw	0,r4,r5			/* already running at PHYSICAL_START? */
 191	bne	relocate_kernel
 192/*
 193 * we now have the 1st 16M of ram mapped with the bats.
 194 * prep needs the mmu to be turned on here, but pmac already has it on.
 195 * this shouldn't bother the pmac since it just gets turned on again
 196 * as we jump to our code at KERNELBASE. -- Cort
 197 * Actually no, pmac doesn't have it on any more. BootX enters with MMU
 198 * off, and in other cases, we now turn it off before changing BATs above.
 199 */
 200turn_on_mmu:
 201	mfmsr	r0
 202	ori	r0,r0,MSR_DR|MSR_IR|MSR_RI
 203	mtspr	SPRN_SRR1,r0
 204	lis	r0,start_here@h
 205	ori	r0,r0,start_here@l
 206	mtspr	SPRN_SRR0,r0
 207	rfi				/* enables MMU */
 208
 209/*
 210 * We need __secondary_hold as a place to hold the other cpus on
 211 * an SMP machine, even when we are running a UP kernel.
 212 */
 213	. = 0xc0			/* for prep bootloader */
 214	li	r3,1			/* MTX only has 1 cpu */
 215	.globl	__secondary_hold
 216__secondary_hold:
 217	/* tell the master we're here */
 218	stw	r3,__secondary_hold_acknowledge@l(0)
 219#ifdef CONFIG_SMP
 220100:	lwz	r4,0(0)
 221	/* wait until we're told to start */
 222	cmpw	0,r4,r3
 223	bne	100b
 224	/* our cpu # was at addr 0 - go */
 225	mr	r24,r3			/* cpu # */
 226	b	__secondary_start
 227#else
 228	b	.
 229#endif /* CONFIG_SMP */
 230
 231	.globl	__secondary_hold_spinloop
 232__secondary_hold_spinloop:
 233	.long	0
 234	.globl	__secondary_hold_acknowledge
 235__secondary_hold_acknowledge:
 236	.long	-1
 237
 238/* System reset */
 239/* core99 pmac starts the seconary here by changing the vector, and
 240   putting it back to what it was (unknown_async_exception) when done.  */
 241	EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, unknown_async_exception)
 242
 243/* Machine check */
 244/*
 245 * On CHRP, this is complicated by the fact that we could get a
 246 * machine check inside RTAS, and we have no guarantee that certain
 247 * critical registers will have the values we expect.  The set of
 248 * registers that might have bad values includes all the GPRs
 249 * and all the BATs.  We indicate that we are in RTAS by putting
 250 * a non-zero value, the address of the exception frame to use,
 251 * in thread.rtas_sp.  The machine check handler checks thread.rtas_sp
 252 * and uses its value if it is non-zero.
 253 * (Other exception handlers assume that r1 is a valid kernel stack
 254 * pointer when we take an exception from supervisor mode.)
 255 *	-- paulus.
 256 */
 257	START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck)
 258	EXCEPTION_PROLOG_0
 259#ifdef CONFIG_PPC_CHRP
 260	mtspr	SPRN_SPRG_SCRATCH2,r1
 261	mfspr	r1, SPRN_SPRG_THREAD
 262	lwz	r1, RTAS_SP(r1)
 263	cmpwi	cr1, r1, 0
 264	bne	cr1, 7f
 265	mfspr	r1, SPRN_SPRG_SCRATCH2
 266#endif /* CONFIG_PPC_CHRP */
 267	EXCEPTION_PROLOG_1
 2687:	EXCEPTION_PROLOG_2 0x200 MachineCheck
 269#ifdef CONFIG_PPC_CHRP
 270	beq	cr1, 1f
 271	twi	31, 0, 0
 272#endif
 2731:	prepare_transfer_to_handler
 274	bl	machine_check_exception
 275	b	interrupt_return
 276
 277/* Data access exception. */
 278	START_EXCEPTION(INTERRUPT_DATA_STORAGE, DataAccess)
 279#ifdef CONFIG_PPC_BOOK3S_604
 280BEGIN_MMU_FTR_SECTION
 281	mtspr	SPRN_SPRG_SCRATCH2,r10
 282	mfspr	r10, SPRN_SPRG_THREAD
 283	stw	r11, THR11(r10)
 284	mfspr	r10, SPRN_DSISR
 285	mfcr	r11
 286	andis.	r10, r10, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h
 287	mfspr	r10, SPRN_SPRG_THREAD
 288	beq	hash_page_dsi
 289.Lhash_page_dsi_cont:
 290	mtcr	r11
 291	lwz	r11, THR11(r10)
 292	mfspr	r10, SPRN_SPRG_SCRATCH2
 293MMU_FTR_SECTION_ELSE
 294	b	1f
 295ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
 296#endif
 2971:	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
 298	EXCEPTION_PROLOG_1
 299	EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataAccess handle_dar_dsisr=1
 300	prepare_transfer_to_handler
 301	lwz	r5, _DSISR(r1)
 302	andis.	r0, r5, DSISR_DABRMATCH@h
 303	bne-	1f
 304	bl	do_page_fault
 305	b	interrupt_return
 3061:	bl	do_break
 307	REST_NVGPRS(r1)
 308	b	interrupt_return
 309
 310
 311/* Instruction access exception. */
 312	START_EXCEPTION(INTERRUPT_INST_STORAGE, InstructionAccess)
 313	mtspr	SPRN_SPRG_SCRATCH0,r10
 314	mtspr	SPRN_SPRG_SCRATCH1,r11
 315	mfspr	r10, SPRN_SPRG_THREAD
 316	mfspr	r11, SPRN_SRR0
 317	stw	r11, SRR0(r10)
 318	mfspr	r11, SPRN_SRR1		/* check whether user or kernel */
 319	stw	r11, SRR1(r10)
 320	mfcr	r10
 321#ifdef CONFIG_PPC_BOOK3S_604
 322BEGIN_MMU_FTR_SECTION
 323	andis.	r11, r11, SRR1_ISI_NOPT@h	/* no pte found? */
 324	bne	hash_page_isi
 325.Lhash_page_isi_cont:
 326	mfspr	r11, SPRN_SRR1		/* check whether user or kernel */
 327END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
 328#endif
 329	andi.	r11, r11, MSR_PR
 330
 331	EXCEPTION_PROLOG_1
 332	EXCEPTION_PROLOG_2 INTERRUPT_INST_STORAGE InstructionAccess
 333	andis.	r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
 334	stw	r5, _DSISR(r11)
 335	stw	r12, _DAR(r11)
 336	prepare_transfer_to_handler
 337	bl	do_page_fault
 338	b	interrupt_return
 339
 340/* External interrupt */
 341	EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ)
 342
 343/* Alignment exception */
 344	START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment)
 345	EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1
 346	prepare_transfer_to_handler
 347	bl	alignment_exception
 348	REST_NVGPRS(r1)
 349	b	interrupt_return
 350
 351/* Program check exception */
 352	START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck)
 353	EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck
 354	prepare_transfer_to_handler
 355	bl	program_check_exception
 356	REST_NVGPRS(r1)
 357	b	interrupt_return
 358
 359/* Floating-point unavailable */
 360	START_EXCEPTION(0x800, FPUnavailable)
 361#ifdef CONFIG_PPC_FPU
 362BEGIN_FTR_SECTION
 363/*
 364 * Certain Freescale cores don't have a FPU and treat fp instructions
 365 * as a FP Unavailable exception.  Redirect to illegal/emulation handling.
 366 */
 367	b 	ProgramCheck
 368END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
 369	EXCEPTION_PROLOG INTERRUPT_FP_UNAVAIL FPUnavailable
 370	beq	1f
 371	bl	load_up_fpu		/* if from user, just load it up */
 372	b	fast_exception_return
 3731:	prepare_transfer_to_handler
 374	bl	kernel_fp_unavailable_exception
 375	b	interrupt_return
 376#else
 377	b 	ProgramCheck
 378#endif
 379
 380/* Decrementer */
 381	EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt)
 382
 383	EXCEPTION(0xa00, Trap_0a, unknown_exception)
 384	EXCEPTION(0xb00, Trap_0b, unknown_exception)
 385
 386/* System call */
 387	START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall)
 388	SYSCALL_ENTRY	INTERRUPT_SYSCALL
 389
 390	EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception)
 391	EXCEPTION(0xe00, Trap_0e, unknown_exception)
 392
 393/*
 394 * The Altivec unavailable trap is at 0x0f20.  Foo.
 395 * We effectively remap it to 0x3000.
 396 * We include an altivec unavailable exception vector even if
 397 * not configured for Altivec, so that you can't panic a
 398 * non-altivec kernel running on a machine with altivec just
 399 * by executing an altivec instruction.
 400 */
 401	START_EXCEPTION(INTERRUPT_PERFMON, PerformanceMonitorTrap)
 402	b	PerformanceMonitor
 403
 404	START_EXCEPTION(INTERRUPT_ALTIVEC_UNAVAIL, AltiVecUnavailableTrap)
 405	b	AltiVecUnavailable
 406
 407	__HEAD
 408/*
 409 * Handle TLB miss for instruction on 603/603e.
 410 * Note: we get an alternate set of r0 - r3 to use automatically.
 411 */
 412	. = INTERRUPT_INST_TLB_MISS_603
 413InstructionTLBMiss:
 414/*
 415 * r0:	userspace flag (later scratch)
 416 * r1:	linux style pte ( later becomes ppc hardware pte )
 417 * r2:	ptr to linux-style pte
 418 * r3:	fault address
 419 */
 420	/* Get PTE (linux-style) and check access */
 421	mfspr	r3,SPRN_IMISS
 422#ifdef CONFIG_MODULES
 423	lis	r1, TASK_SIZE@h		/* check if kernel address */
 424	cmplw	0,r1,r3
 425#endif
 426	mfspr	r2, SPRN_SDR1
 427	li	r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC
 428	rlwinm	r2, r2, 28, 0xfffff000
 429#ifdef CONFIG_MODULES
 430	li	r0, 3
 431	bgt-	112f
 432	lis	r2, (swapper_pg_dir - PAGE_OFFSET)@ha	/* if kernel address, use */
 433	li	r0, 0
 434	addi	r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l	/* kernel page table */
 435#endif
 436112:	rlwimi	r2,r3,12,20,29		/* insert top 10 bits of address */
 437	lwz	r2,0(r2)		/* get pmd entry */
 
 
 
 
 438	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
 439	beq-	InstructionAddressInvalid	/* return if no mapping */
 440	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */
 441	lwz	r2,0(r2)		/* get linux-style pte */
 442	andc.	r1,r1,r2		/* check access & ~permission */
 443	bne-	InstructionAddressInvalid /* return if access not permitted */
 444	/* Convert linux-style PTE to low word of PPC-style PTE */
 445#ifdef CONFIG_MODULES
 446	rlwimi	r2, r0, 0, 31, 31	/* userspace ? -> PP lsb */
 447#endif
 448	ori	r1, r1, 0xe06		/* clear out reserved bits */
 449	andc	r1, r2, r1		/* PP = user? 1 : 0 */
 450BEGIN_FTR_SECTION
 451	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */
 452END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
 453	mtspr	SPRN_RPA,r1
 454	tlbli	r3
 455	mfspr	r3,SPRN_SRR1		/* Need to restore CR0 */
 456	mtcrf	0x80,r3
 457	rfi
 458InstructionAddressInvalid:
 459	mfspr	r3,SPRN_SRR1
 460	rlwinm	r1,r3,9,6,6	/* Get load/store bit */
 461
 462	addis	r1,r1,0x2000
 463	mtspr	SPRN_DSISR,r1	/* (shouldn't be needed) */
 464	andi.	r2,r3,0xFFFF	/* Clear upper bits of SRR1 */
 465	or	r2,r2,r1
 466	mtspr	SPRN_SRR1,r2
 467	mfspr	r1,SPRN_IMISS	/* Get failing address */
 468	rlwinm.	r2,r2,0,31,31	/* Check for little endian access */
 469	rlwimi	r2,r2,1,30,30	/* change 1 -> 3 */
 470	xor	r1,r1,r2
 471	mtspr	SPRN_DAR,r1	/* Set fault address */
 472	mfmsr	r0		/* Restore "normal" registers */
 473	xoris	r0,r0,MSR_TGPR>>16
 474	mtcrf	0x80,r3		/* Restore CR0 */
 475	mtmsr	r0
 476	b	InstructionAccess
 477
 478/*
 479 * Handle TLB miss for DATA Load operation on 603/603e
 480 */
 481	. = INTERRUPT_DATA_LOAD_TLB_MISS_603
 482DataLoadTLBMiss:
 483/*
 484 * r0:	userspace flag (later scratch)
 485 * r1:	linux style pte ( later becomes ppc hardware pte )
 486 * r2:	ptr to linux-style pte
 487 * r3:	fault address
 488 */
 489	/* Get PTE (linux-style) and check access */
 490	mfspr	r3,SPRN_DMISS
 491	lis	r1, TASK_SIZE@h		/* check if kernel address */
 492	cmplw	0,r1,r3
 493	mfspr	r2, SPRN_SDR1
 494	li	r1, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_READ
 495	rlwinm	r2, r2, 28, 0xfffff000
 496	li	r0, 3
 497	bgt-	112f
 498	lis	r2, (swapper_pg_dir - PAGE_OFFSET)@ha	/* if kernel address, use */
 499	li	r0, 0
 500	addi	r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l	/* kernel page table */
 501112:	rlwimi	r2,r3,12,20,29		/* insert top 10 bits of address */
 502	lwz	r2,0(r2)		/* get pmd entry */
 503	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
 504	beq-	DataAddressInvalid	/* return if no mapping */
 505	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */
 
 506	lwz	r2,0(r2)		/* get linux-style pte */
 
 507	andc.	r1,r1,r2		/* check access & ~permission */
 508	bne-	DataAddressInvalid	/* return if access not permitted */
 509	/* Convert linux-style PTE to low word of PPC-style PTE */
 510	rlwinm	r1,r2,32-9,30,30	/* _PAGE_WRITE -> PP msb */
 511	rlwimi	r2,r0,0,30,31		/* userspace ? -> PP */
 512	rlwimi	r1,r2,32-3,24,24	/* _PAGE_WRITE -> _PAGE_DIRTY */
 513	xori	r1,r1,_PAGE_DIRTY	/* clear dirty when not rw */
 514	ori	r1,r1,0xe04		/* clear out reserved bits */
 515	andc	r1,r2,r1		/* PP = user? rw? 1: 3: 0 */
 516BEGIN_FTR_SECTION
 517	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */
 518END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
 519	mtspr	SPRN_RPA,r1
 520BEGIN_MMU_FTR_SECTION
 521	li	r0,1
 522	mfspr	r1,SPRN_SPRG_603_LRU
 523	rlwinm	r2,r3,20,27,31		/* Get Address bits 15:19 */
 524	slw	r0,r0,r2
 525	xor	r1,r0,r1
 526	srw	r0,r1,r2
 527	mtspr   SPRN_SPRG_603_LRU,r1
 528	mfspr	r2,SPRN_SRR1
 529	rlwimi	r2,r0,31-14,14,14
 530	mtspr   SPRN_SRR1,r2
 531	mtcrf	0x80,r2
 532	tlbld	r3
 533	rfi
 534MMU_FTR_SECTION_ELSE
 535	mfspr	r2,SPRN_SRR1		/* Need to restore CR0 */
 536	mtcrf	0x80,r2
 537	tlbld	r3
 538	rfi
 539ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
 
 
 
 
 
 
 
 
 
 
 540DataAddressInvalid:
 541	mfspr	r3,SPRN_SRR1
 542	rlwinm	r1,r3,9,6,6	/* Get load/store bit */
 543	addis	r1,r1,0x2000
 544	mtspr	SPRN_DSISR,r1
 545	andi.	r2,r3,0xFFFF	/* Clear upper bits of SRR1 */
 546	mtspr	SPRN_SRR1,r2
 547	mfspr	r1,SPRN_DMISS	/* Get failing address */
 548	rlwinm.	r2,r2,0,31,31	/* Check for little endian access */
 549	beq	20f		/* Jump if big endian */
 550	xori	r1,r1,3
 55120:	mtspr	SPRN_DAR,r1	/* Set fault address */
 552	mfmsr	r0		/* Restore "normal" registers */
 553	xoris	r0,r0,MSR_TGPR>>16
 554	mtcrf	0x80,r3		/* Restore CR0 */
 555	mtmsr	r0
 556	b	DataAccess
 557
 558/*
 559 * Handle TLB miss for DATA Store on 603/603e
 560 */
 561	. = INTERRUPT_DATA_STORE_TLB_MISS_603
 562DataStoreTLBMiss:
 563/*
 564 * r0:	userspace flag (later scratch)
 565 * r1:	linux style pte ( later becomes ppc hardware pte )
 566 * r2:	ptr to linux-style pte
 567 * r3:	fault address
 568 */
 569	/* Get PTE (linux-style) and check access */
 570	mfspr	r3,SPRN_DMISS
 571	lis	r1, TASK_SIZE@h		/* check if kernel address */
 572	cmplw	0,r1,r3
 573	mfspr	r2, SPRN_SDR1
 574	li	r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED
 575	rlwinm	r2, r2, 28, 0xfffff000
 576	li	r0, 3
 577	bgt-	112f
 578	lis	r2, (swapper_pg_dir - PAGE_OFFSET)@ha	/* if kernel address, use */
 579	li	r0, 0
 580	addi	r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l	/* kernel page table */
 581112:	rlwimi	r2,r3,12,20,29		/* insert top 10 bits of address */
 582	lwz	r2,0(r2)		/* get pmd entry */
 583	rlwinm.	r2,r2,0,0,19		/* extract address of pte page */
 584	beq-	DataAddressInvalid	/* return if no mapping */
 585	rlwimi	r2,r3,22,20,29		/* insert next 10 bits of address */
 
 
 586	lwz	r2,0(r2)		/* get linux-style pte */
 
 587	andc.	r1,r1,r2		/* check access & ~permission */
 588	bne-	DataAddressInvalid	/* return if access not permitted */
 589	/* Convert linux-style PTE to low word of PPC-style PTE */
 590	rlwimi	r2,r0,0,31,31		/* userspace ? -> PP lsb */
 591	li	r1,0xe06		/* clear out reserved bits & PP msb */
 592	andc	r1,r2,r1		/* PP = user? 1: 0 */
 593BEGIN_FTR_SECTION
 594	rlwinm	r1,r1,0,~_PAGE_COHERENT	/* clear M (coherence not required) */
 595END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
 596	mtspr	SPRN_RPA,r1
 597	mfspr	r2,SPRN_SRR1		/* Need to restore CR0 */
 598	mtcrf	0x80,r2
 599BEGIN_MMU_FTR_SECTION
 600	li	r0,1
 601	mfspr	r1,SPRN_SPRG_603_LRU
 602	rlwinm	r2,r3,20,27,31		/* Get Address bits 15:19 */
 603	slw	r0,r0,r2
 604	xor	r1,r0,r1
 605	srw	r0,r1,r2
 606	mtspr   SPRN_SPRG_603_LRU,r1
 607	mfspr	r2,SPRN_SRR1
 608	rlwimi	r2,r0,31-14,14,14
 609	mtspr   SPRN_SRR1,r2
 610	mtcrf	0x80,r2
 611	tlbld	r3
 612	rfi
 613MMU_FTR_SECTION_ELSE
 614	mfspr	r2,SPRN_SRR1		/* Need to restore CR0 */
 615	mtcrf	0x80,r2
 616	tlbld	r3
 617	rfi
 618ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
 
 
 
 
 
 
 
 
 
 
 619
 620#ifndef CONFIG_ALTIVEC
 621#define altivec_assist_exception	unknown_exception
 622#endif
 623
 624#ifndef CONFIG_TAU_INT
 625#define TAUException	unknown_async_exception
 626#endif
 627
 628	EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception)
 629	EXCEPTION(0x1400, SMI, SMIException)
 630	EXCEPTION(0x1500, Trap_15, unknown_exception)
 631	EXCEPTION(0x1600, Trap_16, altivec_assist_exception)
 632	EXCEPTION(0x1700, Trap_17, TAUException)
 633	EXCEPTION(0x1800, Trap_18, unknown_exception)
 634	EXCEPTION(0x1900, Trap_19, unknown_exception)
 635	EXCEPTION(0x1a00, Trap_1a, unknown_exception)
 636	EXCEPTION(0x1b00, Trap_1b, unknown_exception)
 637	EXCEPTION(0x1c00, Trap_1c, unknown_exception)
 638	EXCEPTION(0x1d00, Trap_1d, unknown_exception)
 639	EXCEPTION(0x1e00, Trap_1e, unknown_exception)
 640	EXCEPTION(0x1f00, Trap_1f, unknown_exception)
 641	EXCEPTION(0x2000, RunMode, RunModeException)
 642	EXCEPTION(0x2100, Trap_21, unknown_exception)
 643	EXCEPTION(0x2200, Trap_22, unknown_exception)
 644	EXCEPTION(0x2300, Trap_23, unknown_exception)
 645	EXCEPTION(0x2400, Trap_24, unknown_exception)
 646	EXCEPTION(0x2500, Trap_25, unknown_exception)
 647	EXCEPTION(0x2600, Trap_26, unknown_exception)
 648	EXCEPTION(0x2700, Trap_27, unknown_exception)
 649	EXCEPTION(0x2800, Trap_28, unknown_exception)
 650	EXCEPTION(0x2900, Trap_29, unknown_exception)
 651	EXCEPTION(0x2a00, Trap_2a, unknown_exception)
 652	EXCEPTION(0x2b00, Trap_2b, unknown_exception)
 653	EXCEPTION(0x2c00, Trap_2c, unknown_exception)
 654	EXCEPTION(0x2d00, Trap_2d, unknown_exception)
 655	EXCEPTION(0x2e00, Trap_2e, unknown_exception)
 656	EXCEPTION(0x2f00, Trap_2f, unknown_exception)
 657
 658	__HEAD
 659	. = 0x3000
 660
 661#ifdef CONFIG_PPC_BOOK3S_604
 662.macro save_regs_thread		thread
 663	stw	r0, THR0(\thread)
 664	stw	r3, THR3(\thread)
 665	stw	r4, THR4(\thread)
 666	stw	r5, THR5(\thread)
 667	stw	r6, THR6(\thread)
 668	stw	r8, THR8(\thread)
 669	stw	r9, THR9(\thread)
 670	mflr	r0
 671	stw	r0, THLR(\thread)
 672	mfctr	r0
 673	stw	r0, THCTR(\thread)
 674.endm
 675
 676.macro restore_regs_thread	thread
 677	lwz	r0, THLR(\thread)
 678	mtlr	r0
 679	lwz	r0, THCTR(\thread)
 680	mtctr	r0
 681	lwz	r0, THR0(\thread)
 682	lwz	r3, THR3(\thread)
 683	lwz	r4, THR4(\thread)
 684	lwz	r5, THR5(\thread)
 685	lwz	r6, THR6(\thread)
 686	lwz	r8, THR8(\thread)
 687	lwz	r9, THR9(\thread)
 688.endm
 689
 690hash_page_dsi:
 691	save_regs_thread	r10
 692	mfdsisr	r3
 693	mfdar	r4
 694	mfsrr0	r5
 695	mfsrr1	r9
 696	rlwinm	r3, r3, 32 - 15, _PAGE_WRITE	/* DSISR_STORE -> _PAGE_WRITE */
 697	ori	r3, r3, _PAGE_PRESENT | _PAGE_READ
 698	bl	hash_page
 699	mfspr	r10, SPRN_SPRG_THREAD
 700	restore_regs_thread r10
 701	b	.Lhash_page_dsi_cont
 702
 703hash_page_isi:
 704	mr	r11, r10
 705	mfspr	r10, SPRN_SPRG_THREAD
 706	save_regs_thread	r10
 707	li	r3, _PAGE_PRESENT | _PAGE_EXEC
 708	lwz	r4, SRR0(r10)
 709	lwz	r9, SRR1(r10)
 710	bl	hash_page
 711	mfspr	r10, SPRN_SPRG_THREAD
 712	restore_regs_thread r10
 713	mr	r10, r11
 714	b	.Lhash_page_isi_cont
 715
 716	.globl fast_hash_page_return
 717fast_hash_page_return:
 718	andis.	r10, r9, SRR1_ISI_NOPT@h	/* Set on ISI, cleared on DSI */
 719	mfspr	r10, SPRN_SPRG_THREAD
 720	restore_regs_thread r10
 721	bne	1f
 722
 723	/* DSI */
 724	mtcr	r11
 725	lwz	r11, THR11(r10)
 726	mfspr	r10, SPRN_SPRG_SCRATCH2
 727	rfi
 728
 7291:	/* ISI */
 730	mtcr	r11
 731	mfspr	r11, SPRN_SPRG_SCRATCH1
 732	mfspr	r10, SPRN_SPRG_SCRATCH0
 733	rfi
 734#endif /* CONFIG_PPC_BOOK3S_604 */
 735
 736#ifdef CONFIG_VMAP_STACK
 737	vmap_stack_overflow_exception
 738#endif
 739
 740	__HEAD
 741AltiVecUnavailable:
 742	EXCEPTION_PROLOG 0xf20 AltiVecUnavailable
 743#ifdef CONFIG_ALTIVEC
 744	beq	1f
 745	bl	load_up_altivec		/* if from user, just load it up */
 746	b	fast_exception_return
 747#endif /* CONFIG_ALTIVEC */
 7481:	prepare_transfer_to_handler
 749	bl	altivec_unavailable_exception
 750	b	interrupt_return
 751
 752	__HEAD
 753PerformanceMonitor:
 754	EXCEPTION_PROLOG 0xf00 PerformanceMonitor
 755	prepare_transfer_to_handler
 756	bl	performance_monitor_exception
 757	b	interrupt_return
 758
 759
 760	__HEAD
 761/*
 762 * This code is jumped to from the startup code to copy
 763 * the kernel image to physical address PHYSICAL_START.
 764 */
 765relocate_kernel:
 766	lis	r3,PHYSICAL_START@h	/* Destination base address */
 767	li	r6,0			/* Destination offset */
 768	li	r5,0x4000		/* # bytes of memory to copy */
 769	bl	copy_and_flush		/* copy the first 0x4000 bytes */
 770	addi	r0,r3,4f@l		/* jump to the address of 4f */
 771	mtctr	r0			/* in copy and do the rest. */
 772	bctr				/* jump to the copy */
 7734:	lis	r5,_end-KERNELBASE@h
 774	ori	r5,r5,_end-KERNELBASE@l
 775	bl	copy_and_flush		/* copy the rest */
 776	b	turn_on_mmu
 777
 778/*
 779 * Copy routine used to copy the kernel to start at physical address 0
 780 * and flush and invalidate the caches as needed.
 781 * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
 782 * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
 783 */
 784_GLOBAL(copy_and_flush)
 785	addi	r5,r5,-4
 786	addi	r6,r6,-4
 7874:	li	r0,L1_CACHE_BYTES/4
 788	mtctr	r0
 7893:	addi	r6,r6,4			/* copy a cache line */
 790	lwzx	r0,r6,r4
 791	stwx	r0,r6,r3
 792	bdnz	3b
 793	dcbst	r6,r3			/* write it to memory */
 794	sync
 795	icbi	r6,r3			/* flush the icache line */
 796	cmplw	0,r6,r5
 797	blt	4b
 798	sync				/* additional sync needed on g4 */
 799	isync
 800	addi	r5,r5,4
 801	addi	r6,r6,4
 802	blr
 803
 804#ifdef CONFIG_SMP
 805	.globl __secondary_start_mpc86xx
 806__secondary_start_mpc86xx:
 807	mfspr	r3, SPRN_PIR
 808	stw	r3, __secondary_hold_acknowledge@l(0)
 809	mr	r24, r3			/* cpu # */
 810	b	__secondary_start
 811
 812	.globl	__secondary_start_pmac_0
 813__secondary_start_pmac_0:
 814	/* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
 815	li	r24,0
 816	b	1f
 817	li	r24,1
 818	b	1f
 819	li	r24,2
 820	b	1f
 821	li	r24,3
 8221:
 823	/* on powersurge, we come in here with IR=0 and DR=1, and DBAT 0
 824	   set to map the 0xf0000000 - 0xffffffff region */
 825	mfmsr	r0
 826	rlwinm	r0,r0,0,28,26		/* clear DR (0x10) */
 827	mtmsr	r0
 828	isync
 829
 830	.globl	__secondary_start
 831__secondary_start:
 832	/* Copy some CPU settings from CPU 0 */
 833	bl	__restore_cpu_setup
 834
 835	lis	r3,-KERNELBASE@h
 836	mr	r4,r24
 837	bl	call_setup_cpu		/* Call setup_cpu for this CPU */
 838	lis	r3,-KERNELBASE@h
 839	bl	init_idle_6xx
 840
 841	/* get current's stack and current */
 842	lis	r2,secondary_current@ha
 843	tophys(r2,r2)
 844	lwz	r2,secondary_current@l(r2)
 845	tophys(r1,r2)
 846	lwz	r1,TASK_STACK(r1)
 847
 848	/* stack */
 849	addi	r1,r1,THREAD_SIZE-STACK_FRAME_MIN_SIZE
 850	li	r0,0
 851	tophys(r3,r1)
 852	stw	r0,0(r3)
 853
 854	/* load up the MMU */
 855	bl	load_segment_registers
 856	bl	load_up_mmu
 857
 858	/* ptr to phys current thread */
 859	tophys(r4,r2)
 860	addi	r4,r4,THREAD	/* phys address of our thread_struct */
 861	mtspr	SPRN_SPRG_THREAD,r4
 862BEGIN_MMU_FTR_SECTION
 863	lis	r4, (swapper_pg_dir - PAGE_OFFSET)@h
 864	ori	r4, r4, (swapper_pg_dir - PAGE_OFFSET)@l
 865	rlwinm	r4, r4, 4, 0xffff01ff
 866	mtspr	SPRN_SDR1, r4
 867END_MMU_FTR_SECTION_IFCLR(MMU_FTR_HPTE_TABLE)
 868
 869	/* enable MMU and jump to start_secondary */
 870	li	r4,MSR_KERNEL
 871	lis	r3,start_secondary@h
 872	ori	r3,r3,start_secondary@l
 873	mtspr	SPRN_SRR0,r3
 874	mtspr	SPRN_SRR1,r4
 875	rfi
 876#endif /* CONFIG_SMP */
 877
 878#ifdef CONFIG_KVM_BOOK3S_HANDLER
 879#include "../kvm/book3s_rmhandlers.S"
 880#endif
 881
 882/*
 883 * Load stuff into the MMU.  Intended to be called with
 884 * IR=0 and DR=0.
 885 */
 886SYM_FUNC_START_LOCAL(early_hash_table)
 887	sync			/* Force all PTE updates to finish */
 888	isync
 889	tlbia			/* Clear all TLB entries */
 890	sync			/* wait for tlbia/tlbie to finish */
 891	TLBSYNC			/* ... on all CPUs */
 892	/* Load the SDR1 register (hash table base & size) */
 893	lis	r6, early_hash - PAGE_OFFSET@h
 894	ori	r6, r6, 3	/* 256kB table */
 895	mtspr	SPRN_SDR1, r6
 896	blr
 897SYM_FUNC_END(early_hash_table)
 898
 899SYM_FUNC_START_LOCAL(load_up_mmu)
 900	sync			/* Force all PTE updates to finish */
 901	isync
 902	tlbia			/* Clear all TLB entries */
 903	sync			/* wait for tlbia/tlbie to finish */
 904	TLBSYNC			/* ... on all CPUs */
 905BEGIN_MMU_FTR_SECTION
 906	/* Load the SDR1 register (hash table base & size) */
 907	lis	r6,_SDR1@ha
 908	tophys(r6,r6)
 909	lwz	r6,_SDR1@l(r6)
 910	mtspr	SPRN_SDR1,r6
 911END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
 912
 913/* Load the BAT registers with the values set up by MMU_init. */
 914	lis	r3,BATS@ha
 915	addi	r3,r3,BATS@l
 916	tophys(r3,r3)
 917	LOAD_BAT(0,r3,r4,r5)
 918	LOAD_BAT(1,r3,r4,r5)
 919	LOAD_BAT(2,r3,r4,r5)
 920	LOAD_BAT(3,r3,r4,r5)
 921BEGIN_MMU_FTR_SECTION
 922	LOAD_BAT(4,r3,r4,r5)
 923	LOAD_BAT(5,r3,r4,r5)
 924	LOAD_BAT(6,r3,r4,r5)
 925	LOAD_BAT(7,r3,r4,r5)
 926END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
 927	blr
 928SYM_FUNC_END(load_up_mmu)
 929
 930_GLOBAL(load_segment_registers)
 931	li	r0, NUM_USER_SEGMENTS /* load up user segment register values */
 932	mtctr	r0		/* for context 0 */
 933#ifdef CONFIG_PPC_KUEP
 934	lis	r3, SR_NX@h	/* Kp = 0, Ks = 0, VSID = 0 */
 935#else
 936	li	r3, 0		/* Kp = 0, Ks = 0, VSID = 0 */
 937#endif
 938	li	r4, 0
 9393:	mtsrin	r3, r4
 940	addi	r3, r3, 0x111	/* increment VSID */
 941	addis	r4, r4, 0x1000	/* address of next segment */
 942	bdnz	3b
 943	li	r0, 16 - NUM_USER_SEGMENTS /* load up kernel segment registers */
 944	mtctr	r0			/* for context 0 */
 945	rlwinm	r3, r3, 0, ~SR_NX	/* Nx = 0 */
 946	rlwinm	r3, r3, 0, ~SR_KS	/* Ks = 0 */
 947	oris	r3, r3, SR_KP@h		/* Kp = 1 */
 9483:	mtsrin	r3, r4
 949	addi	r3, r3, 0x111	/* increment VSID */
 950	addis	r4, r4, 0x1000	/* address of next segment */
 951	bdnz	3b
 952	blr
 953
 954/*
 955 * This is where the main kernel code starts.
 956 */
 957start_here:
 958	/* ptr to current */
 959	lis	r2,init_task@h
 960	ori	r2,r2,init_task@l
 961	/* Set up for using our exception vectors */
 962	/* ptr to phys current thread */
 963	tophys(r4,r2)
 964	addi	r4,r4,THREAD	/* init task's THREAD */
 965	mtspr	SPRN_SPRG_THREAD,r4
 966BEGIN_MMU_FTR_SECTION
 967	lis	r4, (swapper_pg_dir - PAGE_OFFSET)@h
 968	ori	r4, r4, (swapper_pg_dir - PAGE_OFFSET)@l
 969	rlwinm	r4, r4, 4, 0xffff01ff
 970	mtspr	SPRN_SDR1, r4
 971END_MMU_FTR_SECTION_IFCLR(MMU_FTR_HPTE_TABLE)
 972
 973	/* stack */
 974	lis	r1,init_thread_union@ha
 975	addi	r1,r1,init_thread_union@l
 976	li	r0,0
 977	stwu	r0,THREAD_SIZE-STACK_FRAME_MIN_SIZE(r1)
 978/*
 979 * Do early platform-specific initialization,
 980 * and set up the MMU.
 981 */
 982#ifdef CONFIG_KASAN
 983	bl	kasan_early_init
 984#endif
 985	li	r3,0
 986	mr	r4,r31
 987	bl	machine_init
 988	bl	__save_cpu_setup
 989	bl	MMU_init
 990	bl	MMU_init_hw_patch
 991
 992/*
 993 * Go back to running unmapped so we can load up new values
 994 * for SDR1 (hash table pointer) and the segment registers
 995 * and change to using our exception vectors.
 996 */
 997	lis	r4,2f@h
 998	ori	r4,r4,2f@l
 999	tophys(r4,r4)
1000	li	r3,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1001
1002	.align	4
1003	mtspr	SPRN_SRR0,r4
1004	mtspr	SPRN_SRR1,r3
1005	rfi
1006/* Load up the kernel context */
10072:	bl	load_up_mmu
1008
1009#ifdef CONFIG_BDI_SWITCH
1010	/* Add helper information for the Abatron bdiGDB debugger.
1011	 * We do this here because we know the mmu is disabled, and
1012	 * will be enabled for real in just a few instructions.
1013	 */
1014	lis	r5, abatron_pteptrs@h
1015	ori	r5, r5, abatron_pteptrs@l
1016	stw	r5, 0xf0(0)	/* This much match your Abatron config */
1017	lis	r6, swapper_pg_dir@h
1018	ori	r6, r6, swapper_pg_dir@l
1019	tophys(r5, r5)
1020	stw	r6, 0(r5)
1021#endif /* CONFIG_BDI_SWITCH */
1022
1023/* Now turn on the MMU for real! */
1024	li	r4,MSR_KERNEL
1025	lis	r3,start_kernel@h
1026	ori	r3,r3,start_kernel@l
1027	mtspr	SPRN_SRR0,r3
1028	mtspr	SPRN_SRR1,r4
1029	rfi
1030
1031/*
1032 * An undocumented "feature" of 604e requires that the v bit
1033 * be cleared before changing BAT values.
1034 *
1035 * Also, newer IBM firmware does not clear bat3 and 4 so
1036 * this makes sure it's done.
1037 *  -- Cort
1038 */
1039SYM_FUNC_START_LOCAL(clear_bats)
1040	li	r10,0
1041
1042	mtspr	SPRN_DBAT0U,r10
1043	mtspr	SPRN_DBAT0L,r10
1044	mtspr	SPRN_DBAT1U,r10
1045	mtspr	SPRN_DBAT1L,r10
1046	mtspr	SPRN_DBAT2U,r10
1047	mtspr	SPRN_DBAT2L,r10
1048	mtspr	SPRN_DBAT3U,r10
1049	mtspr	SPRN_DBAT3L,r10
1050	mtspr	SPRN_IBAT0U,r10
1051	mtspr	SPRN_IBAT0L,r10
1052	mtspr	SPRN_IBAT1U,r10
1053	mtspr	SPRN_IBAT1L,r10
1054	mtspr	SPRN_IBAT2U,r10
1055	mtspr	SPRN_IBAT2L,r10
1056	mtspr	SPRN_IBAT3U,r10
1057	mtspr	SPRN_IBAT3L,r10
1058BEGIN_MMU_FTR_SECTION
1059	/* Here's a tweak: at this point, CPU setup have
1060	 * not been called yet, so HIGH_BAT_EN may not be
1061	 * set in HID0 for the 745x processors. However, it
1062	 * seems that doesn't affect our ability to actually
1063	 * write to these SPRs.
1064	 */
1065	mtspr	SPRN_DBAT4U,r10
1066	mtspr	SPRN_DBAT4L,r10
1067	mtspr	SPRN_DBAT5U,r10
1068	mtspr	SPRN_DBAT5L,r10
1069	mtspr	SPRN_DBAT6U,r10
1070	mtspr	SPRN_DBAT6L,r10
1071	mtspr	SPRN_DBAT7U,r10
1072	mtspr	SPRN_DBAT7L,r10
1073	mtspr	SPRN_IBAT4U,r10
1074	mtspr	SPRN_IBAT4L,r10
1075	mtspr	SPRN_IBAT5U,r10
1076	mtspr	SPRN_IBAT5L,r10
1077	mtspr	SPRN_IBAT6U,r10
1078	mtspr	SPRN_IBAT6L,r10
1079	mtspr	SPRN_IBAT7U,r10
1080	mtspr	SPRN_IBAT7L,r10
1081END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
1082	blr
1083SYM_FUNC_END(clear_bats)
1084
1085_GLOBAL(update_bats)
1086	lis	r4, 1f@h
1087	ori	r4, r4, 1f@l
1088	tophys(r4, r4)
1089	mfmsr	r6
1090	mflr	r7
1091	li	r3, MSR_KERNEL & ~(MSR_IR | MSR_DR)
1092	rlwinm	r0, r6, 0, ~MSR_RI
1093	rlwinm	r0, r0, 0, ~MSR_EE
1094	mtmsr	r0
1095
1096	.align	4
1097	mtspr	SPRN_SRR0, r4
1098	mtspr	SPRN_SRR1, r3
1099	rfi
11001:	bl	clear_bats
1101	lis	r3, BATS@ha
1102	addi	r3, r3, BATS@l
1103	tophys(r3, r3)
1104	LOAD_BAT(0, r3, r4, r5)
1105	LOAD_BAT(1, r3, r4, r5)
1106	LOAD_BAT(2, r3, r4, r5)
1107	LOAD_BAT(3, r3, r4, r5)
1108BEGIN_MMU_FTR_SECTION
1109	LOAD_BAT(4, r3, r4, r5)
1110	LOAD_BAT(5, r3, r4, r5)
1111	LOAD_BAT(6, r3, r4, r5)
1112	LOAD_BAT(7, r3, r4, r5)
1113END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS)
1114	li	r3, MSR_KERNEL & ~(MSR_IR | MSR_DR | MSR_RI)
1115	mtmsr	r3
1116	mtspr	SPRN_SRR0, r7
1117	mtspr	SPRN_SRR1, r6
1118	rfi
1119
1120SYM_FUNC_START_LOCAL(flush_tlbs)
1121	lis	r10, 0x40
11221:	addic.	r10, r10, -0x1000
1123	tlbie	r10
1124	bgt	1b
1125	sync
1126	blr
1127SYM_FUNC_END(flush_tlbs)
1128
1129SYM_FUNC_START_LOCAL(mmu_off)
1130 	addi	r4, r3, __after_mmu_off - _start
1131	mfmsr	r3
1132	andi.	r0,r3,MSR_DR|MSR_IR		/* MMU enabled? */
1133	beqlr
1134	andc	r3,r3,r0
1135
1136	.align	4
1137	mtspr	SPRN_SRR0,r4
1138	mtspr	SPRN_SRR1,r3
1139	sync
1140	rfi
1141SYM_FUNC_END(mmu_off)
1142
1143/* We use one BAT to map up to 256M of RAM at _PAGE_OFFSET */
1144SYM_FUNC_START_LOCAL(initial_bats)
1145	lis	r11,PAGE_OFFSET@h
1146	tophys(r8,r11)
1147#ifdef CONFIG_SMP
1148	ori	r8,r8,0x12		/* R/W access, M=1 */
1149#else
1150	ori	r8,r8,2			/* R/W access */
1151#endif /* CONFIG_SMP */
1152	ori	r11,r11,BL_256M<<2|0x2	/* set up BAT registers for 604 */
1153
1154	mtspr	SPRN_DBAT0L,r8		/* N.B. 6xx have valid */
1155	mtspr	SPRN_DBAT0U,r11		/* bit in upper BAT register */
1156	mtspr	SPRN_IBAT0L,r8
1157	mtspr	SPRN_IBAT0U,r11
1158	isync
1159	blr
1160SYM_FUNC_END(initial_bats)
1161
1162#ifdef CONFIG_BOOTX_TEXT
1163SYM_FUNC_START_LOCAL(setup_disp_bat)
1164	/*
1165	 * setup the display bat prepared for us in prom.c
1166	 */
1167	mflr	r8
1168	bl	reloc_offset
1169	mtlr	r8
1170	addis	r8,r3,disp_BAT@ha
1171	addi	r8,r8,disp_BAT@l
1172	cmpwi	cr0,r8,0
1173	beqlr
1174	lwz	r11,0(r8)
1175	lwz	r8,4(r8)
1176	mtspr	SPRN_DBAT3L,r8
1177	mtspr	SPRN_DBAT3U,r11
1178	blr
1179SYM_FUNC_END(setup_disp_bat)
1180#endif /* CONFIG_BOOTX_TEXT */
1181
1182#ifdef CONFIG_PPC_EARLY_DEBUG_CPM
1183SYM_FUNC_START_LOCAL(setup_cpm_bat)
1184	lis	r8, 0xf000
1185	ori	r8, r8,	0x002a
1186	mtspr	SPRN_DBAT1L, r8
1187
1188	lis	r11, 0xf000
1189	ori	r11, r11, (BL_1M << 2) | 2
1190	mtspr	SPRN_DBAT1U, r11
1191
1192	blr
1193SYM_FUNC_END(setup_cpm_bat)
1194#endif
1195
1196#ifdef CONFIG_PPC_EARLY_DEBUG_USBGECKO
1197SYM_FUNC_START_LOCAL(setup_usbgecko_bat)
1198	/* prepare a BAT for early io */
1199#if defined(CONFIG_GAMECUBE)
1200	lis	r8, 0x0c00
1201#elif defined(CONFIG_WII)
1202	lis	r8, 0x0d00
1203#else
1204#error Invalid platform for USB Gecko based early debugging.
1205#endif
1206	/*
1207	 * The virtual address used must match the virtual address
1208	 * associated to the fixmap entry FIX_EARLY_DEBUG_BASE.
1209	 */
1210	lis	r11, 0xfffe	/* top 128K */
1211	ori	r8, r8, 0x002a	/* uncached, guarded ,rw */
1212	ori	r11, r11, 0x2	/* 128K, Vs=1, Vp=0 */
1213	mtspr	SPRN_DBAT1L, r8
1214	mtspr	SPRN_DBAT1U, r11
1215	blr
1216SYM_FUNC_END(setup_usbgecko_bat)
1217#endif
1218
1219	.data