Linux Audio

Check our new training course

Loading...
v4.6
 
   1/*
   2 *  linux/arch/arm/kernel/entry-armv.S
   3 *
   4 *  Copyright (C) 1996,1997,1998 Russell King.
   5 *  ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
   6 *  nommu support by Hyok S. Choi (hyok.choi@samsung.com)
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License version 2 as
  10 * published by the Free Software Foundation.
  11 *
  12 *  Low-level vector interface routines
  13 *
  14 *  Note:  there is a StrongARM bug in the STMIA rn, {regs}^ instruction
  15 *  that causes it to save wrong values...  Be aware!
  16 */
  17
  18#include <linux/init.h>
  19
  20#include <asm/assembler.h>
  21#include <asm/memory.h>
  22#include <asm/glue-df.h>
  23#include <asm/glue-pf.h>
  24#include <asm/vfpmacros.h>
  25#ifndef CONFIG_MULTI_IRQ_HANDLER
  26#include <mach/entry-macro.S>
  27#endif
  28#include <asm/thread_notify.h>
  29#include <asm/unwind.h>
  30#include <asm/unistd.h>
  31#include <asm/tls.h>
  32#include <asm/system_info.h>
 
  33
  34#include "entry-header.S"
  35#include <asm/entry-macro-multi.S>
  36#include <asm/probes.h>
  37
  38/*
  39 * Interrupt handling.
  40 */
  41	.macro	irq_handler
  42#ifdef CONFIG_MULTI_IRQ_HANDLER
  43	ldr	r1, =handle_arch_irq
  44	mov	r0, sp
  45	badr	lr, 9997f
  46	ldr	pc, [r1]
  47#else
  48	arch_irq_handler_default
  49#endif
  509997:
  51	.endm
  52
  53	.macro	pabt_helper
  54	@ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5
  55#ifdef MULTI_PABORT
  56	ldr	ip, .LCprocfns
  57	mov	lr, pc
  58	ldr	pc, [ip, #PROCESSOR_PABT_FUNC]
  59#else
  60	bl	CPU_PABORT_HANDLER
  61#endif
  62	.endm
  63
  64	.macro	dabt_helper
  65
  66	@
  67	@ Call the processor-specific abort handler:
  68	@
  69	@  r2 - pt_regs
  70	@  r4 - aborted context pc
  71	@  r5 - aborted context psr
  72	@
  73	@ The abort handler must return the aborted address in r0, and
  74	@ the fault status register in r1.  r9 must be preserved.
  75	@
  76#ifdef MULTI_DABORT
  77	ldr	ip, .LCprocfns
  78	mov	lr, pc
  79	ldr	pc, [ip, #PROCESSOR_DABT_FUNC]
  80#else
  81	bl	CPU_DABORT_HANDLER
  82#endif
  83	.endm
  84
  85#ifdef CONFIG_KPROBES
  86	.section	.kprobes.text,"ax",%progbits
  87#else
  88	.text
  89#endif
  90
  91/*
  92 * Invalid mode handlers
  93 */
  94	.macro	inv_entry, reason
  95	sub	sp, sp, #S_FRAME_SIZE
  96 ARM(	stmib	sp, {r1 - lr}		)
  97 THUMB(	stmia	sp, {r0 - r12}		)
  98 THUMB(	str	sp, [sp, #S_SP]		)
  99 THUMB(	str	lr, [sp, #S_LR]		)
 100	mov	r1, #\reason
 101	.endm
 102
 103__pabt_invalid:
 104	inv_entry BAD_PREFETCH
 105	b	common_invalid
 106ENDPROC(__pabt_invalid)
 107
 108__dabt_invalid:
 109	inv_entry BAD_DATA
 110	b	common_invalid
 111ENDPROC(__dabt_invalid)
 112
 113__irq_invalid:
 114	inv_entry BAD_IRQ
 115	b	common_invalid
 116ENDPROC(__irq_invalid)
 117
 118__und_invalid:
 119	inv_entry BAD_UNDEFINSTR
 120
 121	@
 122	@ XXX fall through to common_invalid
 123	@
 124
 125@
 126@ common_invalid - generic code for failed exception (re-entrant version of handlers)
 127@
 128common_invalid:
 129	zero_fp
 130
 131	ldmia	r0, {r4 - r6}
 132	add	r0, sp, #S_PC		@ here for interlock avoidance
 133	mov	r7, #-1			@  ""   ""    ""        ""
 134	str	r4, [sp]		@ save preserved r0
 135	stmia	r0, {r5 - r7}		@ lr_<exception>,
 136					@ cpsr_<exception>, "old_r0"
 137
 138	mov	r0, sp
 139	b	bad_mode
 140ENDPROC(__und_invalid)
 141
 142/*
 143 * SVC mode handlers
 144 */
 145
 146#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
 147#define SPFIX(code...) code
 148#else
 149#define SPFIX(code...)
 150#endif
 151
 152	.macro	svc_entry, stack_hole=0, trace=1, uaccess=1
 153 UNWIND(.fnstart		)
 154 UNWIND(.save {r0 - pc}		)
 155	sub	sp, sp, #(S_FRAME_SIZE + 8 + \stack_hole - 4)
 156#ifdef CONFIG_THUMB2_KERNEL
 157 SPFIX(	str	r0, [sp]	)	@ temporarily saved
 158 SPFIX(	mov	r0, sp		)
 159 SPFIX(	tst	r0, #4		)	@ test original stack alignment
 160 SPFIX(	ldr	r0, [sp]	)	@ restored
 161#else
 162 SPFIX(	tst	sp, #4		)
 163#endif
 164 SPFIX(	subeq	sp, sp, #4	)
 165	stmia	sp, {r1 - r12}
 166
 167	ldmia	r0, {r3 - r5}
 168	add	r7, sp, #S_SP - 4	@ here for interlock avoidance
 169	mov	r6, #-1			@  ""  ""      ""       ""
 170	add	r2, sp, #(S_FRAME_SIZE + 8 + \stack_hole - 4)
 171 SPFIX(	addeq	r2, r2, #4	)
 172	str	r3, [sp, #-4]!		@ save the "real" r0 copied
 173					@ from the exception stack
 174
 175	mov	r3, lr
 176
 177	@
 178	@ We are now ready to fill in the remaining blanks on the stack:
 179	@
 180	@  r2 - sp_svc
 181	@  r3 - lr_svc
 182	@  r4 - lr_<exception>, already fixed up for correct return/restart
 183	@  r5 - spsr_<exception>
 184	@  r6 - orig_r0 (see pt_regs definition in ptrace.h)
 185	@
 186	stmia	r7, {r2 - r6}
 187
 188	uaccess_save r0
 189	.if \uaccess
 190	uaccess_disable r0
 191	.endif
 192
 193	.if \trace
 194#ifdef CONFIG_TRACE_IRQFLAGS
 195	bl	trace_hardirqs_off
 196#endif
 197	.endif
 198	.endm
 199
 200	.align	5
 201__dabt_svc:
 202	svc_entry uaccess=0
 203	mov	r2, sp
 204	dabt_helper
 205 THUMB(	ldr	r5, [sp, #S_PSR]	)	@ potentially updated CPSR
 206	svc_exit r5				@ return from exception
 207 UNWIND(.fnend		)
 208ENDPROC(__dabt_svc)
 209
 210	.align	5
 211__irq_svc:
 212	svc_entry
 213	irq_handler
 214
 215#ifdef CONFIG_PREEMPT
 216	get_thread_info tsk
 217	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
 218	ldr	r0, [tsk, #TI_FLAGS]		@ get flags
 219	teq	r8, #0				@ if preempt count != 0
 220	movne	r0, #0				@ force flags to 0
 221	tst	r0, #_TIF_NEED_RESCHED
 222	blne	svc_preempt
 223#endif
 224
 225	svc_exit r5, irq = 1			@ return from exception
 226 UNWIND(.fnend		)
 227ENDPROC(__irq_svc)
 228
 229	.ltorg
 230
 231#ifdef CONFIG_PREEMPT
 232svc_preempt:
 233	mov	r8, lr
 2341:	bl	preempt_schedule_irq		@ irq en/disable is done inside
 235	ldr	r0, [tsk, #TI_FLAGS]		@ get new tasks TI_FLAGS
 236	tst	r0, #_TIF_NEED_RESCHED
 237	reteq	r8				@ go again
 238	b	1b
 239#endif
 240
 241__und_fault:
 242	@ Correct the PC such that it is pointing at the instruction
 243	@ which caused the fault.  If the faulting instruction was ARM
 244	@ the PC will be pointing at the next instruction, and have to
 245	@ subtract 4.  Otherwise, it is Thumb, and the PC will be
 246	@ pointing at the second half of the Thumb instruction.  We
 247	@ have to subtract 2.
 248	ldr	r2, [r0, #S_PC]
 249	sub	r2, r2, r1
 250	str	r2, [r0, #S_PC]
 251	b	do_undefinstr
 252ENDPROC(__und_fault)
 253
 254	.align	5
 255__und_svc:
 256#ifdef CONFIG_KPROBES
 257	@ If a kprobe is about to simulate a "stmdb sp..." instruction,
 258	@ it obviously needs free stack space which then will belong to
 259	@ the saved context.
 260	svc_entry MAX_STACK_SIZE
 261#else
 262	svc_entry
 263#endif
 264	@
 265	@ call emulation code, which returns using r9 if it has emulated
 266	@ the instruction, or the more conventional lr if we are to treat
 267	@ this as a real undefined instruction
 268	@
 269	@  r0 - instruction
 270	@
 271#ifndef CONFIG_THUMB2_KERNEL
 272	ldr	r0, [r4, #-4]
 273#else
 274	mov	r1, #2
 275	ldrh	r0, [r4, #-2]			@ Thumb instruction at LR - 2
 276	cmp	r0, #0xe800			@ 32-bit instruction if xx >= 0
 277	blo	__und_svc_fault
 278	ldrh	r9, [r4]			@ bottom 16 bits
 279	add	r4, r4, #2
 280	str	r4, [sp, #S_PC]
 281	orr	r0, r9, r0, lsl #16
 282#endif
 283	badr	r9, __und_svc_finish
 284	mov	r2, r4
 285	bl	call_fpe
 286
 287	mov	r1, #4				@ PC correction to apply
 288__und_svc_fault:
 
 289	mov	r0, sp				@ struct pt_regs *regs
 290	bl	__und_fault
 291
 292__und_svc_finish:
 
 293	ldr	r5, [sp, #S_PSR]		@ Get SVC cpsr
 294	svc_exit r5				@ return from exception
 295 UNWIND(.fnend		)
 296ENDPROC(__und_svc)
 297
 298	.align	5
 299__pabt_svc:
 300	svc_entry
 301	mov	r2, sp				@ regs
 302	pabt_helper
 303	svc_exit r5				@ return from exception
 304 UNWIND(.fnend		)
 305ENDPROC(__pabt_svc)
 306
 307	.align	5
 308__fiq_svc:
 309	svc_entry trace=0
 310	mov	r0, sp				@ struct pt_regs *regs
 311	bl	handle_fiq_as_nmi
 312	svc_exit_via_fiq
 313 UNWIND(.fnend		)
 314ENDPROC(__fiq_svc)
 315
 316	.align	5
 317.LCcralign:
 318	.word	cr_alignment
 319#ifdef MULTI_DABORT
 320.LCprocfns:
 321	.word	processor
 322#endif
 323.LCfp:
 324	.word	fp_enter
 325
 326/*
 327 * Abort mode handlers
 328 */
 329
 330@
 331@ Taking a FIQ in abort mode is similar to taking a FIQ in SVC mode
 332@ and reuses the same macros. However in abort mode we must also
 333@ save/restore lr_abt and spsr_abt to make nested aborts safe.
 334@
 335	.align 5
 336__fiq_abt:
 337	svc_entry trace=0
 338
 339 ARM(	msr	cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 340 THUMB( mov	r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 341 THUMB( msr	cpsr_c, r0 )
 342	mov	r1, lr		@ Save lr_abt
 343	mrs	r2, spsr	@ Save spsr_abt, abort is now safe
 344 ARM(	msr	cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 345 THUMB( mov	r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 346 THUMB( msr	cpsr_c, r0 )
 347	stmfd	sp!, {r1 - r2}
 348
 349	add	r0, sp, #8			@ struct pt_regs *regs
 350	bl	handle_fiq_as_nmi
 351
 352	ldmfd	sp!, {r1 - r2}
 353 ARM(	msr	cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 354 THUMB( mov	r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 355 THUMB( msr	cpsr_c, r0 )
 356	mov	lr, r1		@ Restore lr_abt, abort is unsafe
 357	msr	spsr_cxsf, r2	@ Restore spsr_abt
 358 ARM(	msr	cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 359 THUMB( mov	r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 360 THUMB( msr	cpsr_c, r0 )
 361
 362	svc_exit_via_fiq
 363 UNWIND(.fnend		)
 364ENDPROC(__fiq_abt)
 365
 366/*
 367 * User mode handlers
 368 *
 369 * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE
 370 */
 371
 372#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7)
 373#error "sizeof(struct pt_regs) must be a multiple of 8"
 374#endif
 375
 376	.macro	usr_entry, trace=1, uaccess=1
 377 UNWIND(.fnstart	)
 378 UNWIND(.cantunwind	)	@ don't unwind the user space
 379	sub	sp, sp, #S_FRAME_SIZE
 380 ARM(	stmib	sp, {r1 - r12}	)
 381 THUMB(	stmia	sp, {r0 - r12}	)
 382
 383 ATRAP(	mrc	p15, 0, r7, c1, c0, 0)
 384 ATRAP(	ldr	r8, .LCcralign)
 385
 386	ldmia	r0, {r3 - r5}
 387	add	r0, sp, #S_PC		@ here for interlock avoidance
 388	mov	r6, #-1			@  ""  ""     ""        ""
 389
 390	str	r3, [sp]		@ save the "real" r0 copied
 391					@ from the exception stack
 392
 393 ATRAP(	ldr	r8, [r8, #0])
 394
 395	@
 396	@ We are now ready to fill in the remaining blanks on the stack:
 397	@
 398	@  r4 - lr_<exception>, already fixed up for correct return/restart
 399	@  r5 - spsr_<exception>
 400	@  r6 - orig_r0 (see pt_regs definition in ptrace.h)
 401	@
 402	@ Also, separately save sp_usr and lr_usr
 403	@
 404	stmia	r0, {r4 - r6}
 405 ARM(	stmdb	r0, {sp, lr}^			)
 406 THUMB(	store_user_sp_lr r0, r1, S_SP - S_PC	)
 407
 408	.if \uaccess
 409	uaccess_disable ip
 410	.endif
 411
 412	@ Enable the alignment trap while in kernel mode
 413 ATRAP(	teq	r8, r7)
 414 ATRAP( mcrne	p15, 0, r8, c1, c0, 0)
 415
 416	@
 417	@ Clear FP to mark the first stack frame
 418	@
 419	zero_fp
 420
 421	.if	\trace
 422#ifdef CONFIG_TRACE_IRQFLAGS
 423	bl	trace_hardirqs_off
 424#endif
 425	ct_user_exit save = 0
 426	.endif
 427	.endm
 428
 429	.macro	kuser_cmpxchg_check
 430#if !defined(CONFIG_CPU_32v6K) && defined(CONFIG_KUSER_HELPERS)
 431#ifndef CONFIG_MMU
 432#warning "NPTL on non MMU needs fixing"
 433#else
 434	@ Make sure our user space atomic helper is restarted
 435	@ if it was interrupted in a critical region.  Here we
 436	@ perform a quick test inline since it should be false
 437	@ 99.9999% of the time.  The rest is done out of line.
 438	cmp	r4, #TASK_SIZE
 
 439	blhs	kuser_cmpxchg64_fixup
 440#endif
 441#endif
 442	.endm
 443
 444	.align	5
 445__dabt_usr:
 446	usr_entry uaccess=0
 447	kuser_cmpxchg_check
 448	mov	r2, sp
 449	dabt_helper
 450	b	ret_from_exception
 451 UNWIND(.fnend		)
 452ENDPROC(__dabt_usr)
 453
 454	.align	5
 455__irq_usr:
 456	usr_entry
 457	kuser_cmpxchg_check
 458	irq_handler
 459	get_thread_info tsk
 460	mov	why, #0
 461	b	ret_to_user_from_irq
 462 UNWIND(.fnend		)
 463ENDPROC(__irq_usr)
 464
 465	.ltorg
 466
 467	.align	5
 468__und_usr:
 469	usr_entry uaccess=0
 470
 471	mov	r2, r4
 472	mov	r3, r5
 473
 474	@ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the
 475	@      faulting instruction depending on Thumb mode.
 476	@ r3 = regs->ARM_cpsr
 477	@
 478	@ The emulation code returns using r9 if it has emulated the
 479	@ instruction, or the more conventional lr if we are to treat
 480	@ this as a real undefined instruction
 481	@
 482	badr	r9, ret_from_exception
 483
 484	@ IRQs must be enabled before attempting to read the instruction from
 485	@ user space since that could cause a page/translation fault if the
 486	@ page table was modified by another CPU.
 487	enable_irq
 488
 489	tst	r3, #PSR_T_BIT			@ Thumb mode?
 490	bne	__und_usr_thumb
 491	sub	r4, r2, #4			@ ARM instr at LR - 4
 4921:	ldrt	r0, [r4]
 493 ARM_BE8(rev	r0, r0)				@ little endian instruction
 494
 495	uaccess_disable ip
 496
 497	@ r0 = 32-bit ARM instruction which caused the exception
 498	@ r2 = PC value for the following instruction (:= regs->ARM_pc)
 499	@ r4 = PC value for the faulting instruction
 500	@ lr = 32-bit undefined instruction function
 501	badr	lr, __und_usr_fault_32
 502	b	call_fpe
 503
 504__und_usr_thumb:
 505	@ Thumb instruction
 506	sub	r4, r2, #2			@ First half of thumb instr at LR - 2
 507#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
 508/*
 509 * Thumb-2 instruction handling.  Note that because pre-v6 and >= v6 platforms
 510 * can never be supported in a single kernel, this code is not applicable at
 511 * all when __LINUX_ARM_ARCH__ < 6.  This allows simplifying assumptions to be
 512 * made about .arch directives.
 513 */
 514#if __LINUX_ARM_ARCH__ < 7
 515/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */
 516#define NEED_CPU_ARCHITECTURE
 517	ldr	r5, .LCcpu_architecture
 518	ldr	r5, [r5]
 519	cmp	r5, #CPU_ARCH_ARMv7
 520	blo	__und_usr_fault_16		@ 16bit undefined instruction
 521/*
 522 * The following code won't get run unless the running CPU really is v7, so
 523 * coding round the lack of ldrht on older arches is pointless.  Temporarily
 524 * override the assembler target arch with the minimum required instead:
 525 */
 526	.arch	armv6t2
 527#endif
 5282:	ldrht	r5, [r4]
 529ARM_BE8(rev16	r5, r5)				@ little endian instruction
 530	cmp	r5, #0xe800			@ 32bit instruction if xx != 0
 531	blo	__und_usr_fault_16_pan		@ 16bit undefined instruction
 5323:	ldrht	r0, [r2]
 533ARM_BE8(rev16	r0, r0)				@ little endian instruction
 534	uaccess_disable ip
 535	add	r2, r2, #2			@ r2 is PC + 2, make it PC + 4
 536	str	r2, [sp, #S_PC]			@ it's a 2x16bit instr, update
 537	orr	r0, r0, r5, lsl #16
 538	badr	lr, __und_usr_fault_32
 539	@ r0 = the two 16-bit Thumb instructions which caused the exception
 540	@ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc)
 541	@ r4 = PC value for the first 16-bit Thumb instruction
 542	@ lr = 32bit undefined instruction function
 543
 544#if __LINUX_ARM_ARCH__ < 7
 545/* If the target arch was overridden, change it back: */
 546#ifdef CONFIG_CPU_32v6K
 547	.arch	armv6k
 548#else
 549	.arch	armv6
 550#endif
 551#endif /* __LINUX_ARM_ARCH__ < 7 */
 552#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */
 553	b	__und_usr_fault_16
 554#endif
 555 UNWIND(.fnend)
 556ENDPROC(__und_usr)
 557
 558/*
 559 * The out of line fixup for the ldrt instructions above.
 560 */
 561	.pushsection .text.fixup, "ax"
 562	.align	2
 5634:	str     r4, [sp, #S_PC]			@ retry current instruction
 564	ret	r9
 565	.popsection
 566	.pushsection __ex_table,"a"
 567	.long	1b, 4b
 568#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
 569	.long	2b, 4b
 570	.long	3b, 4b
 571#endif
 572	.popsection
 573
 574/*
 575 * Check whether the instruction is a co-processor instruction.
 576 * If yes, we need to call the relevant co-processor handler.
 577 *
 578 * Note that we don't do a full check here for the co-processor
 579 * instructions; all instructions with bit 27 set are well
 580 * defined.  The only instructions that should fault are the
 581 * co-processor instructions.  However, we have to watch out
 582 * for the ARM6/ARM7 SWI bug.
 583 *
 584 * NEON is a special case that has to be handled here. Not all
 585 * NEON instructions are co-processor instructions, so we have
 586 * to make a special case of checking for them. Plus, there's
 587 * five groups of them, so we have a table of mask/opcode pairs
 588 * to check against, and if any match then we branch off into the
 589 * NEON handler code.
 590 *
 591 * Emulators may wish to make use of the following registers:
 592 *  r0  = instruction opcode (32-bit ARM or two 16-bit Thumb)
 593 *  r2  = PC value to resume execution after successful emulation
 594 *  r9  = normal "successful" return address
 595 *  r10 = this threads thread_info structure
 596 *  lr  = unrecognised instruction return address
 597 * IRQs enabled, FIQs enabled.
 598 */
 599	@
 600	@ Fall-through from Thumb-2 __und_usr
 601	@
 602#ifdef CONFIG_NEON
 603	get_thread_info r10			@ get current thread
 604	adr	r6, .LCneon_thumb_opcodes
 605	b	2f
 606#endif
 607call_fpe:
 608	get_thread_info r10			@ get current thread
 609#ifdef CONFIG_NEON
 610	adr	r6, .LCneon_arm_opcodes
 6112:	ldr	r5, [r6], #4			@ mask value
 612	ldr	r7, [r6], #4			@ opcode bits matching in mask
 613	cmp	r5, #0				@ end mask?
 614	beq	1f
 615	and	r8, r0, r5
 616	cmp	r8, r7				@ NEON instruction?
 617	bne	2b
 618	mov	r7, #1
 619	strb	r7, [r10, #TI_USED_CP + 10]	@ mark CP#10 as used
 620	strb	r7, [r10, #TI_USED_CP + 11]	@ mark CP#11 as used
 621	b	do_vfp				@ let VFP handler handle this
 6221:
 623#endif
 624	tst	r0, #0x08000000			@ only CDP/CPRT/LDC/STC have bit 27
 625	tstne	r0, #0x04000000			@ bit 26 set on both ARM and Thumb-2
 626	reteq	lr
 627	and	r8, r0, #0x00000f00		@ mask out CP number
 628 THUMB(	lsr	r8, r8, #8		)
 629	mov	r7, #1
 630	add	r6, r10, #TI_USED_CP
 631 ARM(	strb	r7, [r6, r8, lsr #8]	)	@ set appropriate used_cp[]
 632 THUMB(	strb	r7, [r6, r8]		)	@ set appropriate used_cp[]
 633#ifdef CONFIG_IWMMXT
 634	@ Test if we need to give access to iWMMXt coprocessors
 635	ldr	r5, [r10, #TI_FLAGS]
 636	rsbs	r7, r8, #(1 << 8)		@ CP 0 or 1 only
 637	movcss	r7, r5, lsr #(TIF_USING_IWMMXT + 1)
 638	bcs	iwmmxt_task_enable
 639#endif
 640 ARM(	add	pc, pc, r8, lsr #6	)
 641 THUMB(	lsl	r8, r8, #2		)
 642 THUMB(	add	pc, r8			)
 643	nop
 644
 645	ret.w	lr				@ CP#0
 646	W(b)	do_fpe				@ CP#1 (FPE)
 647	W(b)	do_fpe				@ CP#2 (FPE)
 648	ret.w	lr				@ CP#3
 649#ifdef CONFIG_CRUNCH
 650	b	crunch_task_enable		@ CP#4 (MaverickCrunch)
 651	b	crunch_task_enable		@ CP#5 (MaverickCrunch)
 652	b	crunch_task_enable		@ CP#6 (MaverickCrunch)
 653#else
 654	ret.w	lr				@ CP#4
 655	ret.w	lr				@ CP#5
 656	ret.w	lr				@ CP#6
 657#endif
 658	ret.w	lr				@ CP#7
 659	ret.w	lr				@ CP#8
 660	ret.w	lr				@ CP#9
 661#ifdef CONFIG_VFP
 662	W(b)	do_vfp				@ CP#10 (VFP)
 663	W(b)	do_vfp				@ CP#11 (VFP)
 664#else
 665	ret.w	lr				@ CP#10 (VFP)
 666	ret.w	lr				@ CP#11 (VFP)
 667#endif
 668	ret.w	lr				@ CP#12
 669	ret.w	lr				@ CP#13
 670	ret.w	lr				@ CP#14 (Debug)
 671	ret.w	lr				@ CP#15 (Control)
 672
 673#ifdef NEED_CPU_ARCHITECTURE
 674	.align	2
 675.LCcpu_architecture:
 676	.word	__cpu_architecture
 677#endif
 678
 679#ifdef CONFIG_NEON
 680	.align	6
 681
 682.LCneon_arm_opcodes:
 683	.word	0xfe000000			@ mask
 684	.word	0xf2000000			@ opcode
 685
 686	.word	0xff100000			@ mask
 687	.word	0xf4000000			@ opcode
 688
 689	.word	0x00000000			@ mask
 690	.word	0x00000000			@ opcode
 691
 692.LCneon_thumb_opcodes:
 693	.word	0xef000000			@ mask
 694	.word	0xef000000			@ opcode
 695
 696	.word	0xff100000			@ mask
 697	.word	0xf9000000			@ opcode
 698
 699	.word	0x00000000			@ mask
 700	.word	0x00000000			@ opcode
 701#endif
 702
 703do_fpe:
 704	ldr	r4, .LCfp
 705	add	r10, r10, #TI_FPSTATE		@ r10 = workspace
 706	ldr	pc, [r4]			@ Call FP module USR entry point
 707
 708/*
 709 * The FP module is called with these registers set:
 710 *  r0  = instruction
 711 *  r2  = PC+4
 712 *  r9  = normal "successful" return address
 713 *  r10 = FP workspace
 714 *  lr  = unrecognised FP instruction return address
 715 */
 716
 717	.pushsection .data
 
 718ENTRY(fp_enter)
 719	.word	no_fp
 720	.popsection
 721
 722ENTRY(no_fp)
 723	ret	lr
 724ENDPROC(no_fp)
 725
 726__und_usr_fault_32:
 727	mov	r1, #4
 728	b	1f
 729__und_usr_fault_16_pan:
 730	uaccess_disable ip
 731__und_usr_fault_16:
 732	mov	r1, #2
 7331:	mov	r0, sp
 734	badr	lr, ret_from_exception
 735	b	__und_fault
 736ENDPROC(__und_usr_fault_32)
 737ENDPROC(__und_usr_fault_16)
 738
 739	.align	5
 740__pabt_usr:
 741	usr_entry
 742	mov	r2, sp				@ regs
 743	pabt_helper
 744 UNWIND(.fnend		)
 745	/* fall through */
 746/*
 747 * This is the return code to user mode for abort handlers
 748 */
 749ENTRY(ret_from_exception)
 750 UNWIND(.fnstart	)
 751 UNWIND(.cantunwind	)
 752	get_thread_info tsk
 753	mov	why, #0
 754	b	ret_to_user
 755 UNWIND(.fnend		)
 756ENDPROC(__pabt_usr)
 757ENDPROC(ret_from_exception)
 758
 759	.align	5
 760__fiq_usr:
 761	usr_entry trace=0
 762	kuser_cmpxchg_check
 763	mov	r0, sp				@ struct pt_regs *regs
 764	bl	handle_fiq_as_nmi
 765	get_thread_info tsk
 766	restore_user_regs fast = 0, offset = 0
 767 UNWIND(.fnend		)
 768ENDPROC(__fiq_usr)
 769
 770/*
 771 * Register switch for ARMv3 and ARMv4 processors
 772 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info
 773 * previous and next are guaranteed not to be the same.
 774 */
 775ENTRY(__switch_to)
 776 UNWIND(.fnstart	)
 777 UNWIND(.cantunwind	)
 778	add	ip, r1, #TI_CPU_SAVE
 779 ARM(	stmia	ip!, {r4 - sl, fp, sp, lr} )	@ Store most regs on stack
 780 THUMB(	stmia	ip!, {r4 - sl, fp}	   )	@ Store most regs on stack
 781 THUMB(	str	sp, [ip], #4		   )
 782 THUMB(	str	lr, [ip], #4		   )
 783	ldr	r4, [r2, #TI_TP_VALUE]
 784	ldr	r5, [r2, #TI_TP_VALUE + 4]
 785#ifdef CONFIG_CPU_USE_DOMAINS
 786	mrc	p15, 0, r6, c3, c0, 0		@ Get domain register
 787	str	r6, [r1, #TI_CPU_DOMAIN]	@ Save old domain register
 788	ldr	r6, [r2, #TI_CPU_DOMAIN]
 789#endif
 790	switch_tls r1, r4, r5, r3, r7
 791#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
 792	ldr	r7, [r2, #TI_TASK]
 793	ldr	r8, =__stack_chk_guard
 794	ldr	r7, [r7, #TSK_STACK_CANARY]
 
 
 
 795#endif
 796#ifdef CONFIG_CPU_USE_DOMAINS
 797	mcr	p15, 0, r6, c3, c0, 0		@ Set domain register
 798#endif
 799	mov	r5, r0
 800	add	r4, r2, #TI_CPU_SAVE
 801	ldr	r0, =thread_notify_head
 802	mov	r1, #THREAD_NOTIFY_SWITCH
 803	bl	atomic_notifier_call_chain
 804#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
 805	str	r7, [r8]
 806#endif
 807 THUMB(	mov	ip, r4			   )
 808	mov	r0, r5
 809 ARM(	ldmia	r4, {r4 - sl, fp, sp, pc}  )	@ Load all regs saved previously
 810 THUMB(	ldmia	ip!, {r4 - sl, fp}	   )	@ Load all regs saved previously
 811 THUMB(	ldr	sp, [ip], #4		   )
 812 THUMB(	ldr	pc, [ip]		   )
 813 UNWIND(.fnend		)
 814ENDPROC(__switch_to)
 815
 816	__INIT
 817
 818/*
 819 * User helpers.
 820 *
 821 * Each segment is 32-byte aligned and will be moved to the top of the high
 822 * vector page.  New segments (if ever needed) must be added in front of
 823 * existing ones.  This mechanism should be used only for things that are
 824 * really small and justified, and not be abused freely.
 825 *
 826 * See Documentation/arm/kernel_user_helpers.txt for formal definitions.
 827 */
 828 THUMB(	.arm	)
 829
 830	.macro	usr_ret, reg
 831#ifdef CONFIG_ARM_THUMB
 832	bx	\reg
 833#else
 834	ret	\reg
 835#endif
 836	.endm
 837
 838	.macro	kuser_pad, sym, size
 839	.if	(. - \sym) & 3
 840	.rept	4 - (. - \sym) & 3
 841	.byte	0
 842	.endr
 843	.endif
 844	.rept	(\size - (. - \sym)) / 4
 845	.word	0xe7fddef1
 846	.endr
 847	.endm
 848
 849#ifdef CONFIG_KUSER_HELPERS
 850	.align	5
 851	.globl	__kuser_helper_start
 852__kuser_helper_start:
 853
 854/*
 855 * Due to the length of some sequences, __kuser_cmpxchg64 spans 2 regular
 856 * kuser "slots", therefore 0xffff0f80 is not used as a valid entry point.
 857 */
 858
 859__kuser_cmpxchg64:				@ 0xffff0f60
 860
 861#if defined(CONFIG_CPU_32v6K)
 862
 863	stmfd	sp!, {r4, r5, r6, r7}
 864	ldrd	r4, r5, [r0]			@ load old val
 865	ldrd	r6, r7, [r1]			@ load new val
 866	smp_dmb	arm
 8671:	ldrexd	r0, r1, [r2]			@ load current val
 868	eors	r3, r0, r4			@ compare with oldval (1)
 869	eoreqs	r3, r1, r5			@ compare with oldval (2)
 870	strexdeq r3, r6, r7, [r2]		@ store newval if eq
 871	teqeq	r3, #1				@ success?
 872	beq	1b				@ if no then retry
 873	smp_dmb	arm
 874	rsbs	r0, r3, #0			@ set returned val and C flag
 875	ldmfd	sp!, {r4, r5, r6, r7}
 876	usr_ret	lr
 877
 878#elif !defined(CONFIG_SMP)
 879
 880#ifdef CONFIG_MMU
 881
 882	/*
 883	 * The only thing that can break atomicity in this cmpxchg64
 884	 * implementation is either an IRQ or a data abort exception
 885	 * causing another process/thread to be scheduled in the middle of
 886	 * the critical sequence.  The same strategy as for cmpxchg is used.
 887	 */
 888	stmfd	sp!, {r4, r5, r6, lr}
 889	ldmia	r0, {r4, r5}			@ load old val
 890	ldmia	r1, {r6, lr}			@ load new val
 8911:	ldmia	r2, {r0, r1}			@ load current val
 892	eors	r3, r0, r4			@ compare with oldval (1)
 893	eoreqs	r3, r1, r5			@ compare with oldval (2)
 8942:	stmeqia	r2, {r6, lr}			@ store newval if eq
 895	rsbs	r0, r3, #0			@ set return val and C flag
 896	ldmfd	sp!, {r4, r5, r6, pc}
 897
 898	.text
 899kuser_cmpxchg64_fixup:
 900	@ Called from kuser_cmpxchg_fixup.
 901	@ r4 = address of interrupted insn (must be preserved).
 902	@ sp = saved regs. r7 and r8 are clobbered.
 903	@ 1b = first critical insn, 2b = last critical insn.
 904	@ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
 905	mov	r7, #0xffff0fff
 906	sub	r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64)))
 907	subs	r8, r4, r7
 908	rsbcss	r8, r8, #(2b - 1b)
 909	strcs	r7, [sp, #S_PC]
 910#if __LINUX_ARM_ARCH__ < 6
 911	bcc	kuser_cmpxchg32_fixup
 912#endif
 913	ret	lr
 914	.previous
 915
 916#else
 917#warning "NPTL on non MMU needs fixing"
 918	mov	r0, #-1
 919	adds	r0, r0, #0
 920	usr_ret	lr
 921#endif
 922
 923#else
 924#error "incoherent kernel configuration"
 925#endif
 926
 927	kuser_pad __kuser_cmpxchg64, 64
 928
 929__kuser_memory_barrier:				@ 0xffff0fa0
 930	smp_dmb	arm
 931	usr_ret	lr
 932
 933	kuser_pad __kuser_memory_barrier, 32
 934
 935__kuser_cmpxchg:				@ 0xffff0fc0
 936
 937#if __LINUX_ARM_ARCH__ < 6
 938
 939#ifdef CONFIG_MMU
 940
 941	/*
 942	 * The only thing that can break atomicity in this cmpxchg
 943	 * implementation is either an IRQ or a data abort exception
 944	 * causing another process/thread to be scheduled in the middle
 945	 * of the critical sequence.  To prevent this, code is added to
 946	 * the IRQ and data abort exception handlers to set the pc back
 947	 * to the beginning of the critical section if it is found to be
 948	 * within that critical section (see kuser_cmpxchg_fixup).
 949	 */
 9501:	ldr	r3, [r2]			@ load current val
 951	subs	r3, r3, r0			@ compare with oldval
 9522:	streq	r1, [r2]			@ store newval if eq
 953	rsbs	r0, r3, #0			@ set return val and C flag
 954	usr_ret	lr
 955
 956	.text
 957kuser_cmpxchg32_fixup:
 958	@ Called from kuser_cmpxchg_check macro.
 959	@ r4 = address of interrupted insn (must be preserved).
 960	@ sp = saved regs. r7 and r8 are clobbered.
 961	@ 1b = first critical insn, 2b = last critical insn.
 962	@ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
 963	mov	r7, #0xffff0fff
 964	sub	r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
 965	subs	r8, r4, r7
 966	rsbcss	r8, r8, #(2b - 1b)
 967	strcs	r7, [sp, #S_PC]
 968	ret	lr
 969	.previous
 970
 971#else
 972#warning "NPTL on non MMU needs fixing"
 973	mov	r0, #-1
 974	adds	r0, r0, #0
 975	usr_ret	lr
 976#endif
 977
 978#else
 979
 980	smp_dmb	arm
 9811:	ldrex	r3, [r2]
 982	subs	r3, r3, r0
 983	strexeq	r3, r1, [r2]
 984	teqeq	r3, #1
 985	beq	1b
 986	rsbs	r0, r3, #0
 987	/* beware -- each __kuser slot must be 8 instructions max */
 988	ALT_SMP(b	__kuser_memory_barrier)
 989	ALT_UP(usr_ret	lr)
 990
 991#endif
 992
 993	kuser_pad __kuser_cmpxchg, 32
 994
 995__kuser_get_tls:				@ 0xffff0fe0
 996	ldr	r0, [pc, #(16 - 8)]	@ read TLS, set in kuser_get_tls_init
 997	usr_ret	lr
 998	mrc	p15, 0, r0, c13, c0, 3	@ 0xffff0fe8 hardware TLS code
 999	kuser_pad __kuser_get_tls, 16
1000	.rep	3
1001	.word	0			@ 0xffff0ff0 software TLS value, then
1002	.endr				@ pad up to __kuser_helper_version
1003
1004__kuser_helper_version:				@ 0xffff0ffc
1005	.word	((__kuser_helper_end - __kuser_helper_start) >> 5)
1006
1007	.globl	__kuser_helper_end
1008__kuser_helper_end:
1009
1010#endif
1011
1012 THUMB(	.thumb	)
1013
1014/*
1015 * Vector stubs.
1016 *
1017 * This code is copied to 0xffff1000 so we can use branches in the
1018 * vectors, rather than ldr's.  Note that this code must not exceed
1019 * a page size.
1020 *
1021 * Common stub entry macro:
1022 *   Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
1023 *
1024 * SP points to a minimal amount of processor-private memory, the address
1025 * of which is copied into r0 for the mode specific abort handler.
1026 */
1027	.macro	vector_stub, name, mode, correction=0
1028	.align	5
1029
1030vector_\name:
1031	.if \correction
1032	sub	lr, lr, #\correction
1033	.endif
1034
1035	@
1036	@ Save r0, lr_<exception> (parent PC) and spsr_<exception>
1037	@ (parent CPSR)
1038	@
1039	stmia	sp, {r0, lr}		@ save r0, lr
1040	mrs	lr, spsr
1041	str	lr, [sp, #8]		@ save spsr
1042
1043	@
1044	@ Prepare for SVC32 mode.  IRQs remain disabled.
1045	@
1046	mrs	r0, cpsr
1047	eor	r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE)
1048	msr	spsr_cxsf, r0
1049
1050	@
1051	@ the branch table must immediately follow this code
1052	@
1053	and	lr, lr, #0x0f
1054 THUMB(	adr	r0, 1f			)
1055 THUMB(	ldr	lr, [r0, lr, lsl #2]	)
1056	mov	r0, sp
1057 ARM(	ldr	lr, [pc, lr, lsl #2]	)
1058	movs	pc, lr			@ branch to handler in SVC mode
1059ENDPROC(vector_\name)
1060
1061	.align	2
1062	@ handler addresses follow this label
10631:
1064	.endm
1065
1066	.section .stubs, "ax", %progbits
1067	@ This must be the first word
1068	.word	vector_swi
1069
1070vector_rst:
1071 ARM(	swi	SYS_ERROR0	)
1072 THUMB(	svc	#0		)
1073 THUMB(	nop			)
1074	b	vector_und
1075
1076/*
1077 * Interrupt dispatcher
1078 */
1079	vector_stub	irq, IRQ_MODE, 4
1080
1081	.long	__irq_usr			@  0  (USR_26 / USR_32)
1082	.long	__irq_invalid			@  1  (FIQ_26 / FIQ_32)
1083	.long	__irq_invalid			@  2  (IRQ_26 / IRQ_32)
1084	.long	__irq_svc			@  3  (SVC_26 / SVC_32)
1085	.long	__irq_invalid			@  4
1086	.long	__irq_invalid			@  5
1087	.long	__irq_invalid			@  6
1088	.long	__irq_invalid			@  7
1089	.long	__irq_invalid			@  8
1090	.long	__irq_invalid			@  9
1091	.long	__irq_invalid			@  a
1092	.long	__irq_invalid			@  b
1093	.long	__irq_invalid			@  c
1094	.long	__irq_invalid			@  d
1095	.long	__irq_invalid			@  e
1096	.long	__irq_invalid			@  f
1097
1098/*
1099 * Data abort dispatcher
1100 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1101 */
1102	vector_stub	dabt, ABT_MODE, 8
1103
1104	.long	__dabt_usr			@  0  (USR_26 / USR_32)
1105	.long	__dabt_invalid			@  1  (FIQ_26 / FIQ_32)
1106	.long	__dabt_invalid			@  2  (IRQ_26 / IRQ_32)
1107	.long	__dabt_svc			@  3  (SVC_26 / SVC_32)
1108	.long	__dabt_invalid			@  4
1109	.long	__dabt_invalid			@  5
1110	.long	__dabt_invalid			@  6
1111	.long	__dabt_invalid			@  7
1112	.long	__dabt_invalid			@  8
1113	.long	__dabt_invalid			@  9
1114	.long	__dabt_invalid			@  a
1115	.long	__dabt_invalid			@  b
1116	.long	__dabt_invalid			@  c
1117	.long	__dabt_invalid			@  d
1118	.long	__dabt_invalid			@  e
1119	.long	__dabt_invalid			@  f
1120
1121/*
1122 * Prefetch abort dispatcher
1123 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1124 */
1125	vector_stub	pabt, ABT_MODE, 4
1126
1127	.long	__pabt_usr			@  0 (USR_26 / USR_32)
1128	.long	__pabt_invalid			@  1 (FIQ_26 / FIQ_32)
1129	.long	__pabt_invalid			@  2 (IRQ_26 / IRQ_32)
1130	.long	__pabt_svc			@  3 (SVC_26 / SVC_32)
1131	.long	__pabt_invalid			@  4
1132	.long	__pabt_invalid			@  5
1133	.long	__pabt_invalid			@  6
1134	.long	__pabt_invalid			@  7
1135	.long	__pabt_invalid			@  8
1136	.long	__pabt_invalid			@  9
1137	.long	__pabt_invalid			@  a
1138	.long	__pabt_invalid			@  b
1139	.long	__pabt_invalid			@  c
1140	.long	__pabt_invalid			@  d
1141	.long	__pabt_invalid			@  e
1142	.long	__pabt_invalid			@  f
1143
1144/*
1145 * Undef instr entry dispatcher
1146 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
1147 */
1148	vector_stub	und, UND_MODE
1149
1150	.long	__und_usr			@  0 (USR_26 / USR_32)
1151	.long	__und_invalid			@  1 (FIQ_26 / FIQ_32)
1152	.long	__und_invalid			@  2 (IRQ_26 / IRQ_32)
1153	.long	__und_svc			@  3 (SVC_26 / SVC_32)
1154	.long	__und_invalid			@  4
1155	.long	__und_invalid			@  5
1156	.long	__und_invalid			@  6
1157	.long	__und_invalid			@  7
1158	.long	__und_invalid			@  8
1159	.long	__und_invalid			@  9
1160	.long	__und_invalid			@  a
1161	.long	__und_invalid			@  b
1162	.long	__und_invalid			@  c
1163	.long	__und_invalid			@  d
1164	.long	__und_invalid			@  e
1165	.long	__und_invalid			@  f
1166
1167	.align	5
1168
1169/*=============================================================================
1170 * Address exception handler
1171 *-----------------------------------------------------------------------------
1172 * These aren't too critical.
1173 * (they're not supposed to happen, and won't happen in 32-bit data mode).
1174 */
1175
1176vector_addrexcptn:
1177	b	vector_addrexcptn
1178
1179/*=============================================================================
1180 * FIQ "NMI" handler
1181 *-----------------------------------------------------------------------------
1182 * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86
1183 * systems.
1184 */
1185	vector_stub	fiq, FIQ_MODE, 4
1186
1187	.long	__fiq_usr			@  0  (USR_26 / USR_32)
1188	.long	__fiq_svc			@  1  (FIQ_26 / FIQ_32)
1189	.long	__fiq_svc			@  2  (IRQ_26 / IRQ_32)
1190	.long	__fiq_svc			@  3  (SVC_26 / SVC_32)
1191	.long	__fiq_svc			@  4
1192	.long	__fiq_svc			@  5
1193	.long	__fiq_svc			@  6
1194	.long	__fiq_abt			@  7
1195	.long	__fiq_svc			@  8
1196	.long	__fiq_svc			@  9
1197	.long	__fiq_svc			@  a
1198	.long	__fiq_svc			@  b
1199	.long	__fiq_svc			@  c
1200	.long	__fiq_svc			@  d
1201	.long	__fiq_svc			@  e
1202	.long	__fiq_svc			@  f
1203
1204	.globl	vector_fiq
1205
1206	.section .vectors, "ax", %progbits
1207.L__vectors_start:
1208	W(b)	vector_rst
1209	W(b)	vector_und
1210	W(ldr)	pc, .L__vectors_start + 0x1000
1211	W(b)	vector_pabt
1212	W(b)	vector_dabt
1213	W(b)	vector_addrexcptn
1214	W(b)	vector_irq
1215	W(b)	vector_fiq
1216
1217	.data
 
1218
1219	.globl	cr_alignment
1220cr_alignment:
1221	.space	4
1222
1223#ifdef CONFIG_MULTI_IRQ_HANDLER
1224	.globl	handle_arch_irq
1225handle_arch_irq:
1226	.space	4
1227#endif
v5.14.15
   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 *  linux/arch/arm/kernel/entry-armv.S
   4 *
   5 *  Copyright (C) 1996,1997,1998 Russell King.
   6 *  ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
   7 *  nommu support by Hyok S. Choi (hyok.choi@samsung.com)
   8 *
 
 
 
 
   9 *  Low-level vector interface routines
  10 *
  11 *  Note:  there is a StrongARM bug in the STMIA rn, {regs}^ instruction
  12 *  that causes it to save wrong values...  Be aware!
  13 */
  14
  15#include <linux/init.h>
  16
  17#include <asm/assembler.h>
  18#include <asm/memory.h>
  19#include <asm/glue-df.h>
  20#include <asm/glue-pf.h>
  21#include <asm/vfpmacros.h>
  22#ifndef CONFIG_GENERIC_IRQ_MULTI_HANDLER
  23#include <mach/entry-macro.S>
  24#endif
  25#include <asm/thread_notify.h>
  26#include <asm/unwind.h>
  27#include <asm/unistd.h>
  28#include <asm/tls.h>
  29#include <asm/system_info.h>
  30#include <asm/uaccess-asm.h>
  31
  32#include "entry-header.S"
  33#include <asm/entry-macro-multi.S>
  34#include <asm/probes.h>
  35
  36/*
  37 * Interrupt handling.
  38 */
  39	.macro	irq_handler
  40#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
  41	ldr	r1, =handle_arch_irq
  42	mov	r0, sp
  43	badr	lr, 9997f
  44	ldr	pc, [r1]
  45#else
  46	arch_irq_handler_default
  47#endif
  489997:
  49	.endm
  50
  51	.macro	pabt_helper
  52	@ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5
  53#ifdef MULTI_PABORT
  54	ldr	ip, .LCprocfns
  55	mov	lr, pc
  56	ldr	pc, [ip, #PROCESSOR_PABT_FUNC]
  57#else
  58	bl	CPU_PABORT_HANDLER
  59#endif
  60	.endm
  61
  62	.macro	dabt_helper
  63
  64	@
  65	@ Call the processor-specific abort handler:
  66	@
  67	@  r2 - pt_regs
  68	@  r4 - aborted context pc
  69	@  r5 - aborted context psr
  70	@
  71	@ The abort handler must return the aborted address in r0, and
  72	@ the fault status register in r1.  r9 must be preserved.
  73	@
  74#ifdef MULTI_DABORT
  75	ldr	ip, .LCprocfns
  76	mov	lr, pc
  77	ldr	pc, [ip, #PROCESSOR_DABT_FUNC]
  78#else
  79	bl	CPU_DABORT_HANDLER
  80#endif
  81	.endm
  82
  83	.section	.entry.text,"ax",%progbits
 
 
 
 
  84
  85/*
  86 * Invalid mode handlers
  87 */
  88	.macro	inv_entry, reason
  89	sub	sp, sp, #PT_REGS_SIZE
  90 ARM(	stmib	sp, {r1 - lr}		)
  91 THUMB(	stmia	sp, {r0 - r12}		)
  92 THUMB(	str	sp, [sp, #S_SP]		)
  93 THUMB(	str	lr, [sp, #S_LR]		)
  94	mov	r1, #\reason
  95	.endm
  96
  97__pabt_invalid:
  98	inv_entry BAD_PREFETCH
  99	b	common_invalid
 100ENDPROC(__pabt_invalid)
 101
 102__dabt_invalid:
 103	inv_entry BAD_DATA
 104	b	common_invalid
 105ENDPROC(__dabt_invalid)
 106
 107__irq_invalid:
 108	inv_entry BAD_IRQ
 109	b	common_invalid
 110ENDPROC(__irq_invalid)
 111
 112__und_invalid:
 113	inv_entry BAD_UNDEFINSTR
 114
 115	@
 116	@ XXX fall through to common_invalid
 117	@
 118
 119@
 120@ common_invalid - generic code for failed exception (re-entrant version of handlers)
 121@
 122common_invalid:
 123	zero_fp
 124
 125	ldmia	r0, {r4 - r6}
 126	add	r0, sp, #S_PC		@ here for interlock avoidance
 127	mov	r7, #-1			@  ""   ""    ""        ""
 128	str	r4, [sp]		@ save preserved r0
 129	stmia	r0, {r5 - r7}		@ lr_<exception>,
 130					@ cpsr_<exception>, "old_r0"
 131
 132	mov	r0, sp
 133	b	bad_mode
 134ENDPROC(__und_invalid)
 135
 136/*
 137 * SVC mode handlers
 138 */
 139
 140#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5)
 141#define SPFIX(code...) code
 142#else
 143#define SPFIX(code...)
 144#endif
 145
 146	.macro	svc_entry, stack_hole=0, trace=1, uaccess=1
 147 UNWIND(.fnstart		)
 148 UNWIND(.save {r0 - pc}		)
 149	sub	sp, sp, #(SVC_REGS_SIZE + \stack_hole - 4)
 150#ifdef CONFIG_THUMB2_KERNEL
 151 SPFIX(	str	r0, [sp]	)	@ temporarily saved
 152 SPFIX(	mov	r0, sp		)
 153 SPFIX(	tst	r0, #4		)	@ test original stack alignment
 154 SPFIX(	ldr	r0, [sp]	)	@ restored
 155#else
 156 SPFIX(	tst	sp, #4		)
 157#endif
 158 SPFIX(	subeq	sp, sp, #4	)
 159	stmia	sp, {r1 - r12}
 160
 161	ldmia	r0, {r3 - r5}
 162	add	r7, sp, #S_SP - 4	@ here for interlock avoidance
 163	mov	r6, #-1			@  ""  ""      ""       ""
 164	add	r2, sp, #(SVC_REGS_SIZE + \stack_hole - 4)
 165 SPFIX(	addeq	r2, r2, #4	)
 166	str	r3, [sp, #-4]!		@ save the "real" r0 copied
 167					@ from the exception stack
 168
 169	mov	r3, lr
 170
 171	@
 172	@ We are now ready to fill in the remaining blanks on the stack:
 173	@
 174	@  r2 - sp_svc
 175	@  r3 - lr_svc
 176	@  r4 - lr_<exception>, already fixed up for correct return/restart
 177	@  r5 - spsr_<exception>
 178	@  r6 - orig_r0 (see pt_regs definition in ptrace.h)
 179	@
 180	stmia	r7, {r2 - r6}
 181
 182	get_thread_info tsk
 183	uaccess_entry tsk, r0, r1, r2, \uaccess
 
 
 184
 185	.if \trace
 186#ifdef CONFIG_TRACE_IRQFLAGS
 187	bl	trace_hardirqs_off
 188#endif
 189	.endif
 190	.endm
 191
 192	.align	5
 193__dabt_svc:
 194	svc_entry uaccess=0
 195	mov	r2, sp
 196	dabt_helper
 197 THUMB(	ldr	r5, [sp, #S_PSR]	)	@ potentially updated CPSR
 198	svc_exit r5				@ return from exception
 199 UNWIND(.fnend		)
 200ENDPROC(__dabt_svc)
 201
 202	.align	5
 203__irq_svc:
 204	svc_entry
 205	irq_handler
 206
 207#ifdef CONFIG_PREEMPTION
 
 208	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
 209	ldr	r0, [tsk, #TI_FLAGS]		@ get flags
 210	teq	r8, #0				@ if preempt count != 0
 211	movne	r0, #0				@ force flags to 0
 212	tst	r0, #_TIF_NEED_RESCHED
 213	blne	svc_preempt
 214#endif
 215
 216	svc_exit r5, irq = 1			@ return from exception
 217 UNWIND(.fnend		)
 218ENDPROC(__irq_svc)
 219
 220	.ltorg
 221
 222#ifdef CONFIG_PREEMPTION
 223svc_preempt:
 224	mov	r8, lr
 2251:	bl	preempt_schedule_irq		@ irq en/disable is done inside
 226	ldr	r0, [tsk, #TI_FLAGS]		@ get new tasks TI_FLAGS
 227	tst	r0, #_TIF_NEED_RESCHED
 228	reteq	r8				@ go again
 229	b	1b
 230#endif
 231
 232__und_fault:
 233	@ Correct the PC such that it is pointing at the instruction
 234	@ which caused the fault.  If the faulting instruction was ARM
 235	@ the PC will be pointing at the next instruction, and have to
 236	@ subtract 4.  Otherwise, it is Thumb, and the PC will be
 237	@ pointing at the second half of the Thumb instruction.  We
 238	@ have to subtract 2.
 239	ldr	r2, [r0, #S_PC]
 240	sub	r2, r2, r1
 241	str	r2, [r0, #S_PC]
 242	b	do_undefinstr
 243ENDPROC(__und_fault)
 244
 245	.align	5
 246__und_svc:
 247#ifdef CONFIG_KPROBES
 248	@ If a kprobe is about to simulate a "stmdb sp..." instruction,
 249	@ it obviously needs free stack space which then will belong to
 250	@ the saved context.
 251	svc_entry MAX_STACK_SIZE
 252#else
 253	svc_entry
 254#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 255
 256	mov	r1, #4				@ PC correction to apply
 257 THUMB(	tst	r5, #PSR_T_BIT		)	@ exception taken in Thumb mode?
 258 THUMB(	movne	r1, #2			)	@ if so, fix up PC correction
 259	mov	r0, sp				@ struct pt_regs *regs
 260	bl	__und_fault
 261
 262__und_svc_finish:
 263	get_thread_info tsk
 264	ldr	r5, [sp, #S_PSR]		@ Get SVC cpsr
 265	svc_exit r5				@ return from exception
 266 UNWIND(.fnend		)
 267ENDPROC(__und_svc)
 268
 269	.align	5
 270__pabt_svc:
 271	svc_entry
 272	mov	r2, sp				@ regs
 273	pabt_helper
 274	svc_exit r5				@ return from exception
 275 UNWIND(.fnend		)
 276ENDPROC(__pabt_svc)
 277
 278	.align	5
 279__fiq_svc:
 280	svc_entry trace=0
 281	mov	r0, sp				@ struct pt_regs *regs
 282	bl	handle_fiq_as_nmi
 283	svc_exit_via_fiq
 284 UNWIND(.fnend		)
 285ENDPROC(__fiq_svc)
 286
 287	.align	5
 288.LCcralign:
 289	.word	cr_alignment
 290#ifdef MULTI_DABORT
 291.LCprocfns:
 292	.word	processor
 293#endif
 294.LCfp:
 295	.word	fp_enter
 296
 297/*
 298 * Abort mode handlers
 299 */
 300
 301@
 302@ Taking a FIQ in abort mode is similar to taking a FIQ in SVC mode
 303@ and reuses the same macros. However in abort mode we must also
 304@ save/restore lr_abt and spsr_abt to make nested aborts safe.
 305@
 306	.align 5
 307__fiq_abt:
 308	svc_entry trace=0
 309
 310 ARM(	msr	cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 311 THUMB( mov	r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 312 THUMB( msr	cpsr_c, r0 )
 313	mov	r1, lr		@ Save lr_abt
 314	mrs	r2, spsr	@ Save spsr_abt, abort is now safe
 315 ARM(	msr	cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 316 THUMB( mov	r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 317 THUMB( msr	cpsr_c, r0 )
 318	stmfd	sp!, {r1 - r2}
 319
 320	add	r0, sp, #8			@ struct pt_regs *regs
 321	bl	handle_fiq_as_nmi
 322
 323	ldmfd	sp!, {r1 - r2}
 324 ARM(	msr	cpsr_c, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 325 THUMB( mov	r0, #ABT_MODE | PSR_I_BIT | PSR_F_BIT )
 326 THUMB( msr	cpsr_c, r0 )
 327	mov	lr, r1		@ Restore lr_abt, abort is unsafe
 328	msr	spsr_cxsf, r2	@ Restore spsr_abt
 329 ARM(	msr	cpsr_c, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 330 THUMB( mov	r0, #SVC_MODE | PSR_I_BIT | PSR_F_BIT )
 331 THUMB( msr	cpsr_c, r0 )
 332
 333	svc_exit_via_fiq
 334 UNWIND(.fnend		)
 335ENDPROC(__fiq_abt)
 336
 337/*
 338 * User mode handlers
 339 *
 340 * EABI note: sp_svc is always 64-bit aligned here, so should PT_REGS_SIZE
 341 */
 342
 343#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (PT_REGS_SIZE & 7)
 344#error "sizeof(struct pt_regs) must be a multiple of 8"
 345#endif
 346
 347	.macro	usr_entry, trace=1, uaccess=1
 348 UNWIND(.fnstart	)
 349 UNWIND(.cantunwind	)	@ don't unwind the user space
 350	sub	sp, sp, #PT_REGS_SIZE
 351 ARM(	stmib	sp, {r1 - r12}	)
 352 THUMB(	stmia	sp, {r0 - r12}	)
 353
 354 ATRAP(	mrc	p15, 0, r7, c1, c0, 0)
 355 ATRAP(	ldr	r8, .LCcralign)
 356
 357	ldmia	r0, {r3 - r5}
 358	add	r0, sp, #S_PC		@ here for interlock avoidance
 359	mov	r6, #-1			@  ""  ""     ""        ""
 360
 361	str	r3, [sp]		@ save the "real" r0 copied
 362					@ from the exception stack
 363
 364 ATRAP(	ldr	r8, [r8, #0])
 365
 366	@
 367	@ We are now ready to fill in the remaining blanks on the stack:
 368	@
 369	@  r4 - lr_<exception>, already fixed up for correct return/restart
 370	@  r5 - spsr_<exception>
 371	@  r6 - orig_r0 (see pt_regs definition in ptrace.h)
 372	@
 373	@ Also, separately save sp_usr and lr_usr
 374	@
 375	stmia	r0, {r4 - r6}
 376 ARM(	stmdb	r0, {sp, lr}^			)
 377 THUMB(	store_user_sp_lr r0, r1, S_SP - S_PC	)
 378
 379	.if \uaccess
 380	uaccess_disable ip
 381	.endif
 382
 383	@ Enable the alignment trap while in kernel mode
 384 ATRAP(	teq	r8, r7)
 385 ATRAP( mcrne	p15, 0, r8, c1, c0, 0)
 386
 387	@
 388	@ Clear FP to mark the first stack frame
 389	@
 390	zero_fp
 391
 392	.if	\trace
 393#ifdef CONFIG_TRACE_IRQFLAGS
 394	bl	trace_hardirqs_off
 395#endif
 396	ct_user_exit save = 0
 397	.endif
 398	.endm
 399
 400	.macro	kuser_cmpxchg_check
 401#if !defined(CONFIG_CPU_32v6K) && defined(CONFIG_KUSER_HELPERS)
 402#ifndef CONFIG_MMU
 403#warning "NPTL on non MMU needs fixing"
 404#else
 405	@ Make sure our user space atomic helper is restarted
 406	@ if it was interrupted in a critical region.  Here we
 407	@ perform a quick test inline since it should be false
 408	@ 99.9999% of the time.  The rest is done out of line.
 409	ldr	r0, =TASK_SIZE
 410	cmp	r4, r0
 411	blhs	kuser_cmpxchg64_fixup
 412#endif
 413#endif
 414	.endm
 415
 416	.align	5
 417__dabt_usr:
 418	usr_entry uaccess=0
 419	kuser_cmpxchg_check
 420	mov	r2, sp
 421	dabt_helper
 422	b	ret_from_exception
 423 UNWIND(.fnend		)
 424ENDPROC(__dabt_usr)
 425
 426	.align	5
 427__irq_usr:
 428	usr_entry
 429	kuser_cmpxchg_check
 430	irq_handler
 431	get_thread_info tsk
 432	mov	why, #0
 433	b	ret_to_user_from_irq
 434 UNWIND(.fnend		)
 435ENDPROC(__irq_usr)
 436
 437	.ltorg
 438
 439	.align	5
 440__und_usr:
 441	usr_entry uaccess=0
 442
 443	mov	r2, r4
 444	mov	r3, r5
 445
 446	@ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the
 447	@      faulting instruction depending on Thumb mode.
 448	@ r3 = regs->ARM_cpsr
 449	@
 450	@ The emulation code returns using r9 if it has emulated the
 451	@ instruction, or the more conventional lr if we are to treat
 452	@ this as a real undefined instruction
 453	@
 454	badr	r9, ret_from_exception
 455
 456	@ IRQs must be enabled before attempting to read the instruction from
 457	@ user space since that could cause a page/translation fault if the
 458	@ page table was modified by another CPU.
 459	enable_irq
 460
 461	tst	r3, #PSR_T_BIT			@ Thumb mode?
 462	bne	__und_usr_thumb
 463	sub	r4, r2, #4			@ ARM instr at LR - 4
 4641:	ldrt	r0, [r4]
 465 ARM_BE8(rev	r0, r0)				@ little endian instruction
 466
 467	uaccess_disable ip
 468
 469	@ r0 = 32-bit ARM instruction which caused the exception
 470	@ r2 = PC value for the following instruction (:= regs->ARM_pc)
 471	@ r4 = PC value for the faulting instruction
 472	@ lr = 32-bit undefined instruction function
 473	badr	lr, __und_usr_fault_32
 474	b	call_fpe
 475
 476__und_usr_thumb:
 477	@ Thumb instruction
 478	sub	r4, r2, #2			@ First half of thumb instr at LR - 2
 479#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
 480/*
 481 * Thumb-2 instruction handling.  Note that because pre-v6 and >= v6 platforms
 482 * can never be supported in a single kernel, this code is not applicable at
 483 * all when __LINUX_ARM_ARCH__ < 6.  This allows simplifying assumptions to be
 484 * made about .arch directives.
 485 */
 486#if __LINUX_ARM_ARCH__ < 7
 487/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */
 488#define NEED_CPU_ARCHITECTURE
 489	ldr	r5, .LCcpu_architecture
 490	ldr	r5, [r5]
 491	cmp	r5, #CPU_ARCH_ARMv7
 492	blo	__und_usr_fault_16		@ 16bit undefined instruction
 493/*
 494 * The following code won't get run unless the running CPU really is v7, so
 495 * coding round the lack of ldrht on older arches is pointless.  Temporarily
 496 * override the assembler target arch with the minimum required instead:
 497 */
 498	.arch	armv6t2
 499#endif
 5002:	ldrht	r5, [r4]
 501ARM_BE8(rev16	r5, r5)				@ little endian instruction
 502	cmp	r5, #0xe800			@ 32bit instruction if xx != 0
 503	blo	__und_usr_fault_16_pan		@ 16bit undefined instruction
 5043:	ldrht	r0, [r2]
 505ARM_BE8(rev16	r0, r0)				@ little endian instruction
 506	uaccess_disable ip
 507	add	r2, r2, #2			@ r2 is PC + 2, make it PC + 4
 508	str	r2, [sp, #S_PC]			@ it's a 2x16bit instr, update
 509	orr	r0, r0, r5, lsl #16
 510	badr	lr, __und_usr_fault_32
 511	@ r0 = the two 16-bit Thumb instructions which caused the exception
 512	@ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc)
 513	@ r4 = PC value for the first 16-bit Thumb instruction
 514	@ lr = 32bit undefined instruction function
 515
 516#if __LINUX_ARM_ARCH__ < 7
 517/* If the target arch was overridden, change it back: */
 518#ifdef CONFIG_CPU_32v6K
 519	.arch	armv6k
 520#else
 521	.arch	armv6
 522#endif
 523#endif /* __LINUX_ARM_ARCH__ < 7 */
 524#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */
 525	b	__und_usr_fault_16
 526#endif
 527 UNWIND(.fnend)
 528ENDPROC(__und_usr)
 529
 530/*
 531 * The out of line fixup for the ldrt instructions above.
 532 */
 533	.pushsection .text.fixup, "ax"
 534	.align	2
 5354:	str     r4, [sp, #S_PC]			@ retry current instruction
 536	ret	r9
 537	.popsection
 538	.pushsection __ex_table,"a"
 539	.long	1b, 4b
 540#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7
 541	.long	2b, 4b
 542	.long	3b, 4b
 543#endif
 544	.popsection
 545
 546/*
 547 * Check whether the instruction is a co-processor instruction.
 548 * If yes, we need to call the relevant co-processor handler.
 549 *
 550 * Note that we don't do a full check here for the co-processor
 551 * instructions; all instructions with bit 27 set are well
 552 * defined.  The only instructions that should fault are the
 553 * co-processor instructions.  However, we have to watch out
 554 * for the ARM6/ARM7 SWI bug.
 555 *
 556 * NEON is a special case that has to be handled here. Not all
 557 * NEON instructions are co-processor instructions, so we have
 558 * to make a special case of checking for them. Plus, there's
 559 * five groups of them, so we have a table of mask/opcode pairs
 560 * to check against, and if any match then we branch off into the
 561 * NEON handler code.
 562 *
 563 * Emulators may wish to make use of the following registers:
 564 *  r0  = instruction opcode (32-bit ARM or two 16-bit Thumb)
 565 *  r2  = PC value to resume execution after successful emulation
 566 *  r9  = normal "successful" return address
 567 *  r10 = this threads thread_info structure
 568 *  lr  = unrecognised instruction return address
 569 * IRQs enabled, FIQs enabled.
 570 */
 571	@
 572	@ Fall-through from Thumb-2 __und_usr
 573	@
 574#ifdef CONFIG_NEON
 575	get_thread_info r10			@ get current thread
 576	adr	r6, .LCneon_thumb_opcodes
 577	b	2f
 578#endif
 579call_fpe:
 580	get_thread_info r10			@ get current thread
 581#ifdef CONFIG_NEON
 582	adr	r6, .LCneon_arm_opcodes
 5832:	ldr	r5, [r6], #4			@ mask value
 584	ldr	r7, [r6], #4			@ opcode bits matching in mask
 585	cmp	r5, #0				@ end mask?
 586	beq	1f
 587	and	r8, r0, r5
 588	cmp	r8, r7				@ NEON instruction?
 589	bne	2b
 590	mov	r7, #1
 591	strb	r7, [r10, #TI_USED_CP + 10]	@ mark CP#10 as used
 592	strb	r7, [r10, #TI_USED_CP + 11]	@ mark CP#11 as used
 593	b	do_vfp				@ let VFP handler handle this
 5941:
 595#endif
 596	tst	r0, #0x08000000			@ only CDP/CPRT/LDC/STC have bit 27
 597	tstne	r0, #0x04000000			@ bit 26 set on both ARM and Thumb-2
 598	reteq	lr
 599	and	r8, r0, #0x00000f00		@ mask out CP number
 600 THUMB(	lsr	r8, r8, #8		)
 601	mov	r7, #1
 602	add	r6, r10, #TI_USED_CP
 603 ARM(	strb	r7, [r6, r8, lsr #8]	)	@ set appropriate used_cp[]
 604 THUMB(	strb	r7, [r6, r8]		)	@ set appropriate used_cp[]
 605#ifdef CONFIG_IWMMXT
 606	@ Test if we need to give access to iWMMXt coprocessors
 607	ldr	r5, [r10, #TI_FLAGS]
 608	rsbs	r7, r8, #(1 << 8)		@ CP 0 or 1 only
 609	movscs	r7, r5, lsr #(TIF_USING_IWMMXT + 1)
 610	bcs	iwmmxt_task_enable
 611#endif
 612 ARM(	add	pc, pc, r8, lsr #6	)
 613 THUMB(	lsl	r8, r8, #2		)
 614 THUMB(	add	pc, r8			)
 615	nop
 616
 617	ret.w	lr				@ CP#0
 618	W(b)	do_fpe				@ CP#1 (FPE)
 619	W(b)	do_fpe				@ CP#2 (FPE)
 620	ret.w	lr				@ CP#3
 621#ifdef CONFIG_CRUNCH
 622	b	crunch_task_enable		@ CP#4 (MaverickCrunch)
 623	b	crunch_task_enable		@ CP#5 (MaverickCrunch)
 624	b	crunch_task_enable		@ CP#6 (MaverickCrunch)
 625#else
 626	ret.w	lr				@ CP#4
 627	ret.w	lr				@ CP#5
 628	ret.w	lr				@ CP#6
 629#endif
 630	ret.w	lr				@ CP#7
 631	ret.w	lr				@ CP#8
 632	ret.w	lr				@ CP#9
 633#ifdef CONFIG_VFP
 634	W(b)	do_vfp				@ CP#10 (VFP)
 635	W(b)	do_vfp				@ CP#11 (VFP)
 636#else
 637	ret.w	lr				@ CP#10 (VFP)
 638	ret.w	lr				@ CP#11 (VFP)
 639#endif
 640	ret.w	lr				@ CP#12
 641	ret.w	lr				@ CP#13
 642	ret.w	lr				@ CP#14 (Debug)
 643	ret.w	lr				@ CP#15 (Control)
 644
 645#ifdef NEED_CPU_ARCHITECTURE
 646	.align	2
 647.LCcpu_architecture:
 648	.word	__cpu_architecture
 649#endif
 650
 651#ifdef CONFIG_NEON
 652	.align	6
 653
 654.LCneon_arm_opcodes:
 655	.word	0xfe000000			@ mask
 656	.word	0xf2000000			@ opcode
 657
 658	.word	0xff100000			@ mask
 659	.word	0xf4000000			@ opcode
 660
 661	.word	0x00000000			@ mask
 662	.word	0x00000000			@ opcode
 663
 664.LCneon_thumb_opcodes:
 665	.word	0xef000000			@ mask
 666	.word	0xef000000			@ opcode
 667
 668	.word	0xff100000			@ mask
 669	.word	0xf9000000			@ opcode
 670
 671	.word	0x00000000			@ mask
 672	.word	0x00000000			@ opcode
 673#endif
 674
 675do_fpe:
 676	ldr	r4, .LCfp
 677	add	r10, r10, #TI_FPSTATE		@ r10 = workspace
 678	ldr	pc, [r4]			@ Call FP module USR entry point
 679
 680/*
 681 * The FP module is called with these registers set:
 682 *  r0  = instruction
 683 *  r2  = PC+4
 684 *  r9  = normal "successful" return address
 685 *  r10 = FP workspace
 686 *  lr  = unrecognised FP instruction return address
 687 */
 688
 689	.pushsection .data
 690	.align	2
 691ENTRY(fp_enter)
 692	.word	no_fp
 693	.popsection
 694
 695ENTRY(no_fp)
 696	ret	lr
 697ENDPROC(no_fp)
 698
 699__und_usr_fault_32:
 700	mov	r1, #4
 701	b	1f
 702__und_usr_fault_16_pan:
 703	uaccess_disable ip
 704__und_usr_fault_16:
 705	mov	r1, #2
 7061:	mov	r0, sp
 707	badr	lr, ret_from_exception
 708	b	__und_fault
 709ENDPROC(__und_usr_fault_32)
 710ENDPROC(__und_usr_fault_16)
 711
 712	.align	5
 713__pabt_usr:
 714	usr_entry
 715	mov	r2, sp				@ regs
 716	pabt_helper
 717 UNWIND(.fnend		)
 718	/* fall through */
 719/*
 720 * This is the return code to user mode for abort handlers
 721 */
 722ENTRY(ret_from_exception)
 723 UNWIND(.fnstart	)
 724 UNWIND(.cantunwind	)
 725	get_thread_info tsk
 726	mov	why, #0
 727	b	ret_to_user
 728 UNWIND(.fnend		)
 729ENDPROC(__pabt_usr)
 730ENDPROC(ret_from_exception)
 731
 732	.align	5
 733__fiq_usr:
 734	usr_entry trace=0
 735	kuser_cmpxchg_check
 736	mov	r0, sp				@ struct pt_regs *regs
 737	bl	handle_fiq_as_nmi
 738	get_thread_info tsk
 739	restore_user_regs fast = 0, offset = 0
 740 UNWIND(.fnend		)
 741ENDPROC(__fiq_usr)
 742
 743/*
 744 * Register switch for ARMv3 and ARMv4 processors
 745 * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info
 746 * previous and next are guaranteed not to be the same.
 747 */
 748ENTRY(__switch_to)
 749 UNWIND(.fnstart	)
 750 UNWIND(.cantunwind	)
 751	add	ip, r1, #TI_CPU_SAVE
 752 ARM(	stmia	ip!, {r4 - sl, fp, sp, lr} )	@ Store most regs on stack
 753 THUMB(	stmia	ip!, {r4 - sl, fp}	   )	@ Store most regs on stack
 754 THUMB(	str	sp, [ip], #4		   )
 755 THUMB(	str	lr, [ip], #4		   )
 756	ldr	r4, [r2, #TI_TP_VALUE]
 757	ldr	r5, [r2, #TI_TP_VALUE + 4]
 758#ifdef CONFIG_CPU_USE_DOMAINS
 759	mrc	p15, 0, r6, c3, c0, 0		@ Get domain register
 760	str	r6, [r1, #TI_CPU_DOMAIN]	@ Save old domain register
 761	ldr	r6, [r2, #TI_CPU_DOMAIN]
 762#endif
 763	switch_tls r1, r4, r5, r3, r7
 764#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP)
 765	ldr	r7, [r2, #TI_TASK]
 766	ldr	r8, =__stack_chk_guard
 767	.if (TSK_STACK_CANARY > IMM12_MASK)
 768	add	r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK
 769	.endif
 770	ldr	r7, [r7, #TSK_STACK_CANARY & IMM12_MASK]
 771#endif
 772#ifdef CONFIG_CPU_USE_DOMAINS
 773	mcr	p15, 0, r6, c3, c0, 0		@ Set domain register
 774#endif
 775	mov	r5, r0
 776	add	r4, r2, #TI_CPU_SAVE
 777	ldr	r0, =thread_notify_head
 778	mov	r1, #THREAD_NOTIFY_SWITCH
 779	bl	atomic_notifier_call_chain
 780#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP)
 781	str	r7, [r8]
 782#endif
 783 THUMB(	mov	ip, r4			   )
 784	mov	r0, r5
 785 ARM(	ldmia	r4, {r4 - sl, fp, sp, pc}  )	@ Load all regs saved previously
 786 THUMB(	ldmia	ip!, {r4 - sl, fp}	   )	@ Load all regs saved previously
 787 THUMB(	ldr	sp, [ip], #4		   )
 788 THUMB(	ldr	pc, [ip]		   )
 789 UNWIND(.fnend		)
 790ENDPROC(__switch_to)
 791
 792	__INIT
 793
 794/*
 795 * User helpers.
 796 *
 797 * Each segment is 32-byte aligned and will be moved to the top of the high
 798 * vector page.  New segments (if ever needed) must be added in front of
 799 * existing ones.  This mechanism should be used only for things that are
 800 * really small and justified, and not be abused freely.
 801 *
 802 * See Documentation/arm/kernel_user_helpers.rst for formal definitions.
 803 */
 804 THUMB(	.arm	)
 805
 806	.macro	usr_ret, reg
 807#ifdef CONFIG_ARM_THUMB
 808	bx	\reg
 809#else
 810	ret	\reg
 811#endif
 812	.endm
 813
 814	.macro	kuser_pad, sym, size
 815	.if	(. - \sym) & 3
 816	.rept	4 - (. - \sym) & 3
 817	.byte	0
 818	.endr
 819	.endif
 820	.rept	(\size - (. - \sym)) / 4
 821	.word	0xe7fddef1
 822	.endr
 823	.endm
 824
 825#ifdef CONFIG_KUSER_HELPERS
 826	.align	5
 827	.globl	__kuser_helper_start
 828__kuser_helper_start:
 829
 830/*
 831 * Due to the length of some sequences, __kuser_cmpxchg64 spans 2 regular
 832 * kuser "slots", therefore 0xffff0f80 is not used as a valid entry point.
 833 */
 834
 835__kuser_cmpxchg64:				@ 0xffff0f60
 836
 837#if defined(CONFIG_CPU_32v6K)
 838
 839	stmfd	sp!, {r4, r5, r6, r7}
 840	ldrd	r4, r5, [r0]			@ load old val
 841	ldrd	r6, r7, [r1]			@ load new val
 842	smp_dmb	arm
 8431:	ldrexd	r0, r1, [r2]			@ load current val
 844	eors	r3, r0, r4			@ compare with oldval (1)
 845	eorseq	r3, r1, r5			@ compare with oldval (2)
 846	strexdeq r3, r6, r7, [r2]		@ store newval if eq
 847	teqeq	r3, #1				@ success?
 848	beq	1b				@ if no then retry
 849	smp_dmb	arm
 850	rsbs	r0, r3, #0			@ set returned val and C flag
 851	ldmfd	sp!, {r4, r5, r6, r7}
 852	usr_ret	lr
 853
 854#elif !defined(CONFIG_SMP)
 855
 856#ifdef CONFIG_MMU
 857
 858	/*
 859	 * The only thing that can break atomicity in this cmpxchg64
 860	 * implementation is either an IRQ or a data abort exception
 861	 * causing another process/thread to be scheduled in the middle of
 862	 * the critical sequence.  The same strategy as for cmpxchg is used.
 863	 */
 864	stmfd	sp!, {r4, r5, r6, lr}
 865	ldmia	r0, {r4, r5}			@ load old val
 866	ldmia	r1, {r6, lr}			@ load new val
 8671:	ldmia	r2, {r0, r1}			@ load current val
 868	eors	r3, r0, r4			@ compare with oldval (1)
 869	eorseq	r3, r1, r5			@ compare with oldval (2)
 8702:	stmiaeq	r2, {r6, lr}			@ store newval if eq
 871	rsbs	r0, r3, #0			@ set return val and C flag
 872	ldmfd	sp!, {r4, r5, r6, pc}
 873
 874	.text
 875kuser_cmpxchg64_fixup:
 876	@ Called from kuser_cmpxchg_fixup.
 877	@ r4 = address of interrupted insn (must be preserved).
 878	@ sp = saved regs. r7 and r8 are clobbered.
 879	@ 1b = first critical insn, 2b = last critical insn.
 880	@ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
 881	mov	r7, #0xffff0fff
 882	sub	r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64)))
 883	subs	r8, r4, r7
 884	rsbscs	r8, r8, #(2b - 1b)
 885	strcs	r7, [sp, #S_PC]
 886#if __LINUX_ARM_ARCH__ < 6
 887	bcc	kuser_cmpxchg32_fixup
 888#endif
 889	ret	lr
 890	.previous
 891
 892#else
 893#warning "NPTL on non MMU needs fixing"
 894	mov	r0, #-1
 895	adds	r0, r0, #0
 896	usr_ret	lr
 897#endif
 898
 899#else
 900#error "incoherent kernel configuration"
 901#endif
 902
 903	kuser_pad __kuser_cmpxchg64, 64
 904
 905__kuser_memory_barrier:				@ 0xffff0fa0
 906	smp_dmb	arm
 907	usr_ret	lr
 908
 909	kuser_pad __kuser_memory_barrier, 32
 910
 911__kuser_cmpxchg:				@ 0xffff0fc0
 912
 913#if __LINUX_ARM_ARCH__ < 6
 914
 915#ifdef CONFIG_MMU
 916
 917	/*
 918	 * The only thing that can break atomicity in this cmpxchg
 919	 * implementation is either an IRQ or a data abort exception
 920	 * causing another process/thread to be scheduled in the middle
 921	 * of the critical sequence.  To prevent this, code is added to
 922	 * the IRQ and data abort exception handlers to set the pc back
 923	 * to the beginning of the critical section if it is found to be
 924	 * within that critical section (see kuser_cmpxchg_fixup).
 925	 */
 9261:	ldr	r3, [r2]			@ load current val
 927	subs	r3, r3, r0			@ compare with oldval
 9282:	streq	r1, [r2]			@ store newval if eq
 929	rsbs	r0, r3, #0			@ set return val and C flag
 930	usr_ret	lr
 931
 932	.text
 933kuser_cmpxchg32_fixup:
 934	@ Called from kuser_cmpxchg_check macro.
 935	@ r4 = address of interrupted insn (must be preserved).
 936	@ sp = saved regs. r7 and r8 are clobbered.
 937	@ 1b = first critical insn, 2b = last critical insn.
 938	@ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b.
 939	mov	r7, #0xffff0fff
 940	sub	r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg)))
 941	subs	r8, r4, r7
 942	rsbscs	r8, r8, #(2b - 1b)
 943	strcs	r7, [sp, #S_PC]
 944	ret	lr
 945	.previous
 946
 947#else
 948#warning "NPTL on non MMU needs fixing"
 949	mov	r0, #-1
 950	adds	r0, r0, #0
 951	usr_ret	lr
 952#endif
 953
 954#else
 955
 956	smp_dmb	arm
 9571:	ldrex	r3, [r2]
 958	subs	r3, r3, r0
 959	strexeq	r3, r1, [r2]
 960	teqeq	r3, #1
 961	beq	1b
 962	rsbs	r0, r3, #0
 963	/* beware -- each __kuser slot must be 8 instructions max */
 964	ALT_SMP(b	__kuser_memory_barrier)
 965	ALT_UP(usr_ret	lr)
 966
 967#endif
 968
 969	kuser_pad __kuser_cmpxchg, 32
 970
 971__kuser_get_tls:				@ 0xffff0fe0
 972	ldr	r0, [pc, #(16 - 8)]	@ read TLS, set in kuser_get_tls_init
 973	usr_ret	lr
 974	mrc	p15, 0, r0, c13, c0, 3	@ 0xffff0fe8 hardware TLS code
 975	kuser_pad __kuser_get_tls, 16
 976	.rep	3
 977	.word	0			@ 0xffff0ff0 software TLS value, then
 978	.endr				@ pad up to __kuser_helper_version
 979
 980__kuser_helper_version:				@ 0xffff0ffc
 981	.word	((__kuser_helper_end - __kuser_helper_start) >> 5)
 982
 983	.globl	__kuser_helper_end
 984__kuser_helper_end:
 985
 986#endif
 987
 988 THUMB(	.thumb	)
 989
 990/*
 991 * Vector stubs.
 992 *
 993 * This code is copied to 0xffff1000 so we can use branches in the
 994 * vectors, rather than ldr's.  Note that this code must not exceed
 995 * a page size.
 996 *
 997 * Common stub entry macro:
 998 *   Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
 999 *
1000 * SP points to a minimal amount of processor-private memory, the address
1001 * of which is copied into r0 for the mode specific abort handler.
1002 */
1003	.macro	vector_stub, name, mode, correction=0
1004	.align	5
1005
1006vector_\name:
1007	.if \correction
1008	sub	lr, lr, #\correction
1009	.endif
1010
1011	@
1012	@ Save r0, lr_<exception> (parent PC) and spsr_<exception>
1013	@ (parent CPSR)
1014	@
1015	stmia	sp, {r0, lr}		@ save r0, lr
1016	mrs	lr, spsr
1017	str	lr, [sp, #8]		@ save spsr
1018
1019	@
1020	@ Prepare for SVC32 mode.  IRQs remain disabled.
1021	@
1022	mrs	r0, cpsr
1023	eor	r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE)
1024	msr	spsr_cxsf, r0
1025
1026	@
1027	@ the branch table must immediately follow this code
1028	@
1029	and	lr, lr, #0x0f
1030 THUMB(	adr	r0, 1f			)
1031 THUMB(	ldr	lr, [r0, lr, lsl #2]	)
1032	mov	r0, sp
1033 ARM(	ldr	lr, [pc, lr, lsl #2]	)
1034	movs	pc, lr			@ branch to handler in SVC mode
1035ENDPROC(vector_\name)
1036
1037	.align	2
1038	@ handler addresses follow this label
10391:
1040	.endm
1041
1042	.section .stubs, "ax", %progbits
1043	@ This must be the first word
1044	.word	vector_swi
1045
1046vector_rst:
1047 ARM(	swi	SYS_ERROR0	)
1048 THUMB(	svc	#0		)
1049 THUMB(	nop			)
1050	b	vector_und
1051
1052/*
1053 * Interrupt dispatcher
1054 */
1055	vector_stub	irq, IRQ_MODE, 4
1056
1057	.long	__irq_usr			@  0  (USR_26 / USR_32)
1058	.long	__irq_invalid			@  1  (FIQ_26 / FIQ_32)
1059	.long	__irq_invalid			@  2  (IRQ_26 / IRQ_32)
1060	.long	__irq_svc			@  3  (SVC_26 / SVC_32)
1061	.long	__irq_invalid			@  4
1062	.long	__irq_invalid			@  5
1063	.long	__irq_invalid			@  6
1064	.long	__irq_invalid			@  7
1065	.long	__irq_invalid			@  8
1066	.long	__irq_invalid			@  9
1067	.long	__irq_invalid			@  a
1068	.long	__irq_invalid			@  b
1069	.long	__irq_invalid			@  c
1070	.long	__irq_invalid			@  d
1071	.long	__irq_invalid			@  e
1072	.long	__irq_invalid			@  f
1073
1074/*
1075 * Data abort dispatcher
1076 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1077 */
1078	vector_stub	dabt, ABT_MODE, 8
1079
1080	.long	__dabt_usr			@  0  (USR_26 / USR_32)
1081	.long	__dabt_invalid			@  1  (FIQ_26 / FIQ_32)
1082	.long	__dabt_invalid			@  2  (IRQ_26 / IRQ_32)
1083	.long	__dabt_svc			@  3  (SVC_26 / SVC_32)
1084	.long	__dabt_invalid			@  4
1085	.long	__dabt_invalid			@  5
1086	.long	__dabt_invalid			@  6
1087	.long	__dabt_invalid			@  7
1088	.long	__dabt_invalid			@  8
1089	.long	__dabt_invalid			@  9
1090	.long	__dabt_invalid			@  a
1091	.long	__dabt_invalid			@  b
1092	.long	__dabt_invalid			@  c
1093	.long	__dabt_invalid			@  d
1094	.long	__dabt_invalid			@  e
1095	.long	__dabt_invalid			@  f
1096
1097/*
1098 * Prefetch abort dispatcher
1099 * Enter in ABT mode, spsr = USR CPSR, lr = USR PC
1100 */
1101	vector_stub	pabt, ABT_MODE, 4
1102
1103	.long	__pabt_usr			@  0 (USR_26 / USR_32)
1104	.long	__pabt_invalid			@  1 (FIQ_26 / FIQ_32)
1105	.long	__pabt_invalid			@  2 (IRQ_26 / IRQ_32)
1106	.long	__pabt_svc			@  3 (SVC_26 / SVC_32)
1107	.long	__pabt_invalid			@  4
1108	.long	__pabt_invalid			@  5
1109	.long	__pabt_invalid			@  6
1110	.long	__pabt_invalid			@  7
1111	.long	__pabt_invalid			@  8
1112	.long	__pabt_invalid			@  9
1113	.long	__pabt_invalid			@  a
1114	.long	__pabt_invalid			@  b
1115	.long	__pabt_invalid			@  c
1116	.long	__pabt_invalid			@  d
1117	.long	__pabt_invalid			@  e
1118	.long	__pabt_invalid			@  f
1119
1120/*
1121 * Undef instr entry dispatcher
1122 * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC
1123 */
1124	vector_stub	und, UND_MODE
1125
1126	.long	__und_usr			@  0 (USR_26 / USR_32)
1127	.long	__und_invalid			@  1 (FIQ_26 / FIQ_32)
1128	.long	__und_invalid			@  2 (IRQ_26 / IRQ_32)
1129	.long	__und_svc			@  3 (SVC_26 / SVC_32)
1130	.long	__und_invalid			@  4
1131	.long	__und_invalid			@  5
1132	.long	__und_invalid			@  6
1133	.long	__und_invalid			@  7
1134	.long	__und_invalid			@  8
1135	.long	__und_invalid			@  9
1136	.long	__und_invalid			@  a
1137	.long	__und_invalid			@  b
1138	.long	__und_invalid			@  c
1139	.long	__und_invalid			@  d
1140	.long	__und_invalid			@  e
1141	.long	__und_invalid			@  f
1142
1143	.align	5
1144
1145/*=============================================================================
1146 * Address exception handler
1147 *-----------------------------------------------------------------------------
1148 * These aren't too critical.
1149 * (they're not supposed to happen, and won't happen in 32-bit data mode).
1150 */
1151
1152vector_addrexcptn:
1153	b	vector_addrexcptn
1154
1155/*=============================================================================
1156 * FIQ "NMI" handler
1157 *-----------------------------------------------------------------------------
1158 * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86
1159 * systems.
1160 */
1161	vector_stub	fiq, FIQ_MODE, 4
1162
1163	.long	__fiq_usr			@  0  (USR_26 / USR_32)
1164	.long	__fiq_svc			@  1  (FIQ_26 / FIQ_32)
1165	.long	__fiq_svc			@  2  (IRQ_26 / IRQ_32)
1166	.long	__fiq_svc			@  3  (SVC_26 / SVC_32)
1167	.long	__fiq_svc			@  4
1168	.long	__fiq_svc			@  5
1169	.long	__fiq_svc			@  6
1170	.long	__fiq_abt			@  7
1171	.long	__fiq_svc			@  8
1172	.long	__fiq_svc			@  9
1173	.long	__fiq_svc			@  a
1174	.long	__fiq_svc			@  b
1175	.long	__fiq_svc			@  c
1176	.long	__fiq_svc			@  d
1177	.long	__fiq_svc			@  e
1178	.long	__fiq_svc			@  f
1179
1180	.globl	vector_fiq
1181
1182	.section .vectors, "ax", %progbits
1183.L__vectors_start:
1184	W(b)	vector_rst
1185	W(b)	vector_und
1186	W(ldr)	pc, .L__vectors_start + 0x1000
1187	W(b)	vector_pabt
1188	W(b)	vector_dabt
1189	W(b)	vector_addrexcptn
1190	W(b)	vector_irq
1191	W(b)	vector_fiq
1192
1193	.data
1194	.align	2
1195
1196	.globl	cr_alignment
1197cr_alignment:
1198	.space	4