1/* 
  2 * Linux/PA-RISC Project (http://www.parisc-linux.org/)
  3 * 
  4 * System call entry code / Linux gateway page
  5 * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org>
  6 * Licensed under the GNU GPL.
  7 * thanks to Philipp Rumpf, Mike Shaver and various others
  8 * sorry about the wall, puffin..
  9 */
 10
 11/*
 12How does the Linux gateway page on PA-RISC work?
 13------------------------------------------------
 14The Linux gateway page on PA-RISC is "special".
 15It actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc
 16terminology it's Execute, promote to PL0) in the page map.  So anything
 17executing on this page executes with kernel level privilege (there's more to it
 18than that: to have this happen, you also have to use a branch with a ,gate
 19completer to activate the privilege promotion).  The upshot is that everything
 20that runs on the gateway page runs at kernel privilege but with the current
 21user process address space (although you have access to kernel space via %sr2).
 22For the 0x100 syscall entry, we redo the space registers to point to the kernel
 23address space (preserving the user address space in %sr3), move to wide mode if
 24required, save the user registers and branch into the kernel syscall entry
 25point.  For all the other functions, we execute at kernel privilege but don't
 26flip address spaces. The basic upshot of this is that these code snippets are
 27executed atomically (because the kernel can't be pre-empted) and they may
 28perform architecturally forbidden (to PL3) operations (like setting control
 29registers).
 30*/
 31
 32
 33#include <asm/asm-offsets.h>
 34#include <asm/unistd.h>
 35#include <asm/errno.h>
 36#include <asm/page.h>
 37#include <asm/psw.h>
 38#include <asm/thread_info.h>
 39#include <asm/assembly.h>
 40#include <asm/processor.h>
 41#include <asm/cache.h>
 42
 43#include <linux/linkage.h>
 44
 45	/* We fill the empty parts of the gateway page with
 46 	 * something that will kill the kernel or a
 47 	 * userspace application.
 48	 */
 49#define KILL_INSN	break	0,0
 50
 51	.level          PA_ASM_LEVEL
 52
 53	.text
 54
 55	.import syscall_exit,code
 56	.import syscall_exit_rfi,code
 57
 58	/* Linux gateway page is aliased to virtual page 0 in the kernel
 59	 * address space. Since it is a gateway page it cannot be
 60	 * dereferenced, so null pointers will still fault. We start
 61	 * the actual entry point at 0x100. We put break instructions
 62	 * at the beginning of the page to trap null indirect function
 63	 * pointers.
 64	 */
 65
 66	.align PAGE_SIZE
 67ENTRY(linux_gateway_page)
 68
 69        /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */
 70	.rept 44
 71	KILL_INSN
 72	.endr
 73
 74	/* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */
 75	/* Light-weight-syscall entry must always be located at 0xb0 */
 76	/* WARNING: Keep this number updated with table size changes */
 77#define __NR_lws_entries (3)
 78
 79lws_entry:
 80	gate	lws_start, %r0		/* increase privilege */
 81	depi	3, 31, 2, %r31		/* Ensure we return into user mode. */
 82
 83	/* Fill from 0xb8 to 0xe0 */
 84	.rept 10
 85	KILL_INSN
 86	.endr
 87
 88	/* This function MUST be located at 0xe0 for glibc's threading 
 89	mechanism to work. DO NOT MOVE THIS CODE EVER! */
 90set_thread_pointer:
 91	gate	.+8, %r0		/* increase privilege */
 92	depi	3, 31, 2, %r31		/* Ensure we return into user mode. */
 93	be	0(%sr7,%r31)		/* return to user space */
 94	mtctl	%r26, %cr27		/* move arg0 to the control register */
 95
 96	/* Increase the chance of trapping if random jumps occur to this
 97	address, fill from 0xf0 to 0x100 */
 98	.rept 4
 99	KILL_INSN
100	.endr
101
102/* This address must remain fixed at 0x100 for glibc's syscalls to work */
103	.align LINUX_GATEWAY_ADDR
104linux_gateway_entry:
105	gate	.+8, %r0			/* become privileged */
106	mtsp	%r0,%sr4			/* get kernel space into sr4 */
107	mtsp	%r0,%sr5			/* get kernel space into sr5 */
108	mtsp	%r0,%sr6			/* get kernel space into sr6 */
109
110#ifdef CONFIG_64BIT
111	/* Store W bit on entry to the syscall in case it's a wide userland
112	 * process. */
113	ssm	PSW_SM_W, %r1
114	extrd,u	%r1,PSW_W_BIT,1,%r1
115	/* sp must be aligned on 4, so deposit the W bit setting into
116	 * the bottom of sp temporarily */
117	or,ev	%r1,%r30,%r30
118	b,n	1f
119	/* The top halves of argument registers must be cleared on syscall
120	 * entry from narrow executable.
121	 */
122	depdi	0, 31, 32, %r26
123	depdi	0, 31, 32, %r25
124	depdi	0, 31, 32, %r24
125	depdi	0, 31, 32, %r23
126	depdi	0, 31, 32, %r22
127	depdi	0, 31, 32, %r21
1281:	
129#endif
130
131	/* We use a rsm/ssm pair to prevent sr3 from being clobbered
132	 * by external interrupts.
133	 */
134	mfsp    %sr7,%r1                        /* save user sr7 */
135	rsm	PSW_SM_I, %r0			/* disable interrupts */
136	mtsp    %r1,%sr3                        /* and store it in sr3 */
137
138	mfctl   %cr30,%r1
139	xor     %r1,%r30,%r30                   /* ye olde xor trick */
140	xor     %r1,%r30,%r1
141	xor     %r1,%r30,%r30
142	
143	ldo     THREAD_SZ_ALGN+FRAME_SIZE(%r30),%r30  /* set up kernel stack */
144
145	/* N.B.: It is critical that we don't set sr7 to 0 until r30
146	 *       contains a valid kernel stack pointer. It is also
147	 *       critical that we don't start using the kernel stack
148	 *       until after sr7 has been set to 0.
149	 */
150
151	mtsp	%r0,%sr7			/* get kernel space into sr7 */
152	ssm	PSW_SM_I, %r0			/* enable interrupts */
153	STREGM	%r1,FRAME_SIZE(%r30)		/* save r1 (usp) here for now */
154	mfctl	%cr30,%r1			/* get task ptr in %r1 */
155	LDREG	TI_TASK(%r1),%r1
156
157	/* Save some registers for sigcontext and potential task
158	   switch (see entry.S for the details of which ones are
159	   saved/restored).  TASK_PT_PSW is zeroed so we can see whether
160	   a process is on a syscall or not.  For an interrupt the real
161	   PSW value is stored.  This is needed for gdb and sys_ptrace. */
162	STREG	%r0,  TASK_PT_PSW(%r1)
163	STREG	%r2,  TASK_PT_GR2(%r1)		/* preserve rp */
164	STREG	%r19, TASK_PT_GR19(%r1)
165
166	LDREGM	-FRAME_SIZE(%r30), %r2		/* get users sp back */
167#ifdef CONFIG_64BIT
168	extrd,u	%r2,63,1,%r19			/* W hidden in bottom bit */
169#if 0
170	xor	%r19,%r2,%r2			/* clear bottom bit */
171	depd,z	%r19,1,1,%r19
172	std	%r19,TASK_PT_PSW(%r1)
173#endif
174#endif
175	STREG	%r2,  TASK_PT_GR30(%r1)		/* ... and save it */
176	
177	STREG	%r20, TASK_PT_GR20(%r1)		/* Syscall number */
178	STREG	%r21, TASK_PT_GR21(%r1)
179	STREG	%r22, TASK_PT_GR22(%r1)
180	STREG	%r23, TASK_PT_GR23(%r1)		/* 4th argument */
181	STREG	%r24, TASK_PT_GR24(%r1)		/* 3rd argument */
182	STREG	%r25, TASK_PT_GR25(%r1)		/* 2nd argument */
183	STREG	%r26, TASK_PT_GR26(%r1)	 	/* 1st argument */
184	STREG	%r27, TASK_PT_GR27(%r1)		/* user dp */
185	STREG   %r28, TASK_PT_GR28(%r1)         /* return value 0 */
186	STREG   %r0, TASK_PT_ORIG_R28(%r1)      /* don't prohibit restarts */
187	STREG	%r29, TASK_PT_GR29(%r1)		/* return value 1 */
188	STREG	%r31, TASK_PT_GR31(%r1)		/* preserve syscall return ptr */
189	
190	ldo	TASK_PT_FR0(%r1), %r27		/* save fpregs from the kernel */
191	save_fp	%r27				/* or potential task switch  */
192
193	mfctl	%cr11, %r27			/* i.e. SAR */
194	STREG	%r27, TASK_PT_SAR(%r1)
195
196	loadgp
197
198#ifdef CONFIG_64BIT
199	ldo	-16(%r30),%r29			/* Reference param save area */
200	copy	%r19,%r2			/* W bit back to r2 */
201#else
202	/* no need to save these on stack in wide mode because the first 8
203	 * args are passed in registers */
204	stw     %r22, -52(%r30)                 /* 5th argument */
205	stw     %r21, -56(%r30)                 /* 6th argument */
206#endif
207
208	/* Are we being ptraced? */
209	mfctl	%cr30, %r1
210	LDREG	TI_FLAGS(%r1),%r1
211	ldi	_TIF_SYSCALL_TRACE_MASK, %r19
212	and,COND(=) %r1, %r19, %r0
213	b,n	.Ltracesys
214	
215	/* Note!  We cannot use the syscall table that is mapped
216	nearby since the gateway page is mapped execute-only. */
217
218#ifdef CONFIG_64BIT
219	ldil	L%sys_call_table, %r1
220	or,=	%r2,%r2,%r2
221	addil	L%(sys_call_table64-sys_call_table), %r1
222	ldo	R%sys_call_table(%r1), %r19
223	or,=	%r2,%r2,%r2
224	ldo	R%sys_call_table64(%r1), %r19
225#else
226	load32	sys_call_table, %r19
227#endif	
228	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
229	b,n	.Lsyscall_nosys
230	
231	LDREGX  %r20(%r19), %r19
232
233	/* If this is a sys_rt_sigreturn call, and the signal was received
234	 * when not in_syscall, then we want to return via syscall_exit_rfi,
235	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
236	 * trampoline code in signal.c).
237	 */
238	ldi	__NR_rt_sigreturn,%r2
239	comb,=	%r2,%r20,.Lrt_sigreturn
240.Lin_syscall:
241	ldil	L%syscall_exit,%r2
242	be      0(%sr7,%r19)
243	ldo	R%syscall_exit(%r2),%r2
244.Lrt_sigreturn:
245	comib,<> 0,%r25,.Lin_syscall
246	ldil	L%syscall_exit_rfi,%r2
247	be      0(%sr7,%r19)
248	ldo	R%syscall_exit_rfi(%r2),%r2
249
250	/* Note!  Because we are not running where we were linked, any
251	calls to functions external to this file must be indirect.  To
252	be safe, we apply the opposite rule to functions within this
253	file, with local labels given to them to ensure correctness. */
254	
255.Lsyscall_nosys:
256syscall_nosys:
257	ldil	L%syscall_exit,%r1
258	be	R%syscall_exit(%sr7,%r1)
259	ldo	-ENOSYS(%r0),%r28		   /* set errno */
260
261
262/* Warning! This trace code is a virtual duplicate of the code above so be
263 * sure to maintain both! */
264.Ltracesys:
265tracesys:
266	/* Need to save more registers so the debugger can see where we
267	 * are.  This saves only the lower 8 bits of PSW, so that the C
268	 * bit is still clear on syscalls, and the D bit is set if this
269	 * full register save path has been executed.  We check the D
270	 * bit on syscall_return_rfi to determine which registers to
271	 * restore.  An interrupt results in a full PSW saved with the
272	 * C bit set, a non-straced syscall entry results in C and D clear
273	 * in the saved PSW.
274	 */
275	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
276	LDREG	TI_TASK(%r1), %r1
277	ssm	0,%r2
278	STREG	%r2,TASK_PT_PSW(%r1)		/* Lower 8 bits only!! */
279	mfsp	%sr0,%r2
280	STREG	%r2,TASK_PT_SR0(%r1)
281	mfsp	%sr1,%r2
282	STREG	%r2,TASK_PT_SR1(%r1)
283	mfsp	%sr2,%r2
284	STREG	%r2,TASK_PT_SR2(%r1)
285	mfsp	%sr3,%r2
286	STREG	%r2,TASK_PT_SR3(%r1)
287	STREG	%r2,TASK_PT_SR4(%r1)
288	STREG	%r2,TASK_PT_SR5(%r1)
289	STREG	%r2,TASK_PT_SR6(%r1)
290	STREG	%r2,TASK_PT_SR7(%r1)
291	STREG	%r2,TASK_PT_IASQ0(%r1)
292	STREG	%r2,TASK_PT_IASQ1(%r1)
293	LDREG	TASK_PT_GR31(%r1),%r2
294	STREG	%r2,TASK_PT_IAOQ0(%r1)
295	ldo	4(%r2),%r2
296	STREG	%r2,TASK_PT_IAOQ1(%r1)
297	ldo	TASK_REGS(%r1),%r2
298	/* reg_save %r2 */
299	STREG	%r3,PT_GR3(%r2)
300	STREG	%r4,PT_GR4(%r2)
301	STREG	%r5,PT_GR5(%r2)
302	STREG	%r6,PT_GR6(%r2)
303	STREG	%r7,PT_GR7(%r2)
304	STREG	%r8,PT_GR8(%r2)
305	STREG	%r9,PT_GR9(%r2)
306	STREG	%r10,PT_GR10(%r2)
307	STREG	%r11,PT_GR11(%r2)
308	STREG	%r12,PT_GR12(%r2)
309	STREG	%r13,PT_GR13(%r2)
310	STREG	%r14,PT_GR14(%r2)
311	STREG	%r15,PT_GR15(%r2)
312	STREG	%r16,PT_GR16(%r2)
313	STREG	%r17,PT_GR17(%r2)
314	STREG	%r18,PT_GR18(%r2)
315	/* Finished saving things for the debugger */
316
317	copy	%r2,%r26
318	ldil	L%do_syscall_trace_enter,%r1
319	ldil	L%tracesys_next,%r2
320	be	R%do_syscall_trace_enter(%sr7,%r1)
321	ldo	R%tracesys_next(%r2),%r2
322	
323tracesys_next:
324	/* do_syscall_trace_enter either returned the syscallno, or -1L,
325	 *  so we skip restoring the PT_GR20 below, since we pulled it from
326	 *  task->thread.regs.gr[20] above.
327	 */
328	copy	%ret0,%r20
329
330	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
331	LDREG	TI_TASK(%r1), %r1
332	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return value */
333	LDREG   TASK_PT_GR26(%r1), %r26		/* Restore the users args */
334	LDREG   TASK_PT_GR25(%r1), %r25
335	LDREG   TASK_PT_GR24(%r1), %r24
336	LDREG   TASK_PT_GR23(%r1), %r23
337	LDREG   TASK_PT_GR22(%r1), %r22
338	LDREG   TASK_PT_GR21(%r1), %r21
339#ifdef CONFIG_64BIT
340	ldo	-16(%r30),%r29			/* Reference param save area */
341#else
342	stw     %r22, -52(%r30)                 /* 5th argument */
343	stw     %r21, -56(%r30)                 /* 6th argument */
344#endif
345
346	cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */
347	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
348	b,n	.Ltracesys_nosys
349
350	/* Note!  We cannot use the syscall table that is mapped
351	nearby since the gateway page is mapped execute-only. */
352
353#ifdef CONFIG_64BIT
354	LDREG	TASK_PT_GR30(%r1), %r19		/* get users sp back */
355	extrd,u	%r19,63,1,%r2			/* W hidden in bottom bit */
356
357	ldil	L%sys_call_table, %r1
358	or,=	%r2,%r2,%r2
359	addil	L%(sys_call_table64-sys_call_table), %r1
360	ldo	R%sys_call_table(%r1), %r19
361	or,=	%r2,%r2,%r2
362	ldo	R%sys_call_table64(%r1), %r19
363#else
364	load32	sys_call_table, %r19
365#endif
366
367	LDREGX  %r20(%r19), %r19
368
369	/* If this is a sys_rt_sigreturn call, and the signal was received
370	 * when not in_syscall, then we want to return via syscall_exit_rfi,
371	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
372	 * trampoline code in signal.c).
373	 */
374	ldi	__NR_rt_sigreturn,%r2
375	comb,=	%r2,%r20,.Ltrace_rt_sigreturn
376.Ltrace_in_syscall:
377	ldil	L%tracesys_exit,%r2
378	be      0(%sr7,%r19)
379	ldo	R%tracesys_exit(%r2),%r2
380
381.Ltracesys_nosys:
382	ldo	-ENOSYS(%r0),%r28		/* set errno */
383
384	/* Do *not* call this function on the gateway page, because it
385	makes a direct call to syscall_trace. */
386	
387tracesys_exit:
388	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
389	LDREG	TI_TASK(%r1), %r1
390#ifdef CONFIG_64BIT
391	ldo	-16(%r30),%r29			/* Reference param save area */
392#endif
393	ldo	TASK_REGS(%r1),%r26
394	BL	do_syscall_trace_exit,%r2
395	STREG   %r28,TASK_PT_GR28(%r1)          /* save return value now */
396	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
397	LDREG	TI_TASK(%r1), %r1
398	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return val. */
399
400	ldil	L%syscall_exit,%r1
401	be,n	R%syscall_exit(%sr7,%r1)
402
403.Ltrace_rt_sigreturn:
404	comib,<> 0,%r25,.Ltrace_in_syscall
405	ldil	L%tracesys_sigexit,%r2
406	be      0(%sr7,%r19)
407	ldo	R%tracesys_sigexit(%r2),%r2
408
409tracesys_sigexit:
410	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
411	LDREG	TI_TASK(%r1), %r1
412#ifdef CONFIG_64BIT
413	ldo	-16(%r30),%r29			/* Reference param save area */
414#endif
415	BL	do_syscall_trace_exit,%r2
416	ldo	TASK_REGS(%r1),%r26
417
418	ldil	L%syscall_exit_rfi,%r1
419	be,n	R%syscall_exit_rfi(%sr7,%r1)
420
421
422	/*********************************************************
423		32/64-bit Light-Weight-Syscall ABI
424
425		* - Indicates a hint for userspace inline asm
426		implementations.
427
428		Syscall number (caller-saves)
429	        - %r20
430	        * In asm clobber.
431
432		Argument registers (caller-saves)
433	        - %r26, %r25, %r24, %r23, %r22
434	        * In asm input.
435
436		Return registers (caller-saves)
437	        - %r28 (return), %r21 (errno)
438	        * In asm output.
439
440		Caller-saves registers
441	        - %r1, %r27, %r29
442	        - %r2 (return pointer)
443	        - %r31 (ble link register)
444	        * In asm clobber.
445
446		Callee-saves registers
447	        - %r3-%r18
448	        - %r30 (stack pointer)
449	        * Not in asm clobber.
450
451		If userspace is 32-bit:
452		Callee-saves registers
453	        - %r19 (32-bit PIC register)
454
455		Differences from 32-bit calling convention:
456		- Syscall number in %r20
457		- Additional argument register %r22 (arg4)
458		- Callee-saves %r19.
459
460		If userspace is 64-bit:
461		Callee-saves registers
462		- %r27 (64-bit PIC register)
463
464		Differences from 64-bit calling convention:
465		- Syscall number in %r20
466		- Additional argument register %r22 (arg4)
467		- Callee-saves %r27.
468
469		Error codes returned by entry path:
470
471		ENOSYS - r20 was an invalid LWS number.
472
473	*********************************************************/
474lws_start:
475
476#ifdef CONFIG_64BIT
477	ssm	PSW_SM_W, %r1
478	extrd,u	%r1,PSW_W_BIT,1,%r1
479	/* sp must be aligned on 4, so deposit the W bit setting into
480	 * the bottom of sp temporarily */
481	or,ev	%r1,%r30,%r30
482
483	/* Clip LWS number to a 32-bit value for 32-bit processes */
484	depdi	0, 31, 32, %r20
485#endif	
486
487        /* Is the lws entry number valid? */
488	comiclr,>>	__NR_lws_entries, %r20, %r0
489	b,n	lws_exit_nosys
490
491	/* Load table start */
492	ldil	L%lws_table, %r1
493	ldo	R%lws_table(%r1), %r28	/* Scratch use of r28 */
494	LDREGX	%r20(%sr2,r28), %r21	/* Scratch use of r21 */
495
496	/* Jump to lws, lws table pointers already relocated */
497	be,n	0(%sr2,%r21)
498
499lws_exit_nosys:
500	ldo	-ENOSYS(%r0),%r21		   /* set errno */
501	/* Fall through: Return to userspace */
502
503lws_exit:
504#ifdef CONFIG_64BIT
505	/* decide whether to reset the wide mode bit
506	 *
507	 * For a syscall, the W bit is stored in the lowest bit
508	 * of sp.  Extract it and reset W if it is zero */
509	extrd,u,*<>	%r30,63,1,%r1
510	rsm	PSW_SM_W, %r0
511	/* now reset the lowest bit of sp if it was set */
512	xor	%r30,%r1,%r30
513#endif
514	be,n	0(%sr7, %r31)
515
516
517	
518	/***************************************************
519		Implementing 32bit CAS as an atomic operation:
520
521		%r26 - Address to examine
522		%r25 - Old value to check (old)
523		%r24 - New value to set (new)
524		%r28 - Return prev through this register.
525		%r21 - Kernel error code
526
527		If debugging is DISabled:
528
529		%r21 has the following meanings:
530
531		EAGAIN - CAS is busy, ldcw failed, try again.
532		EFAULT - Read or write failed.		
533
534		If debugging is enabled:
535
536		EDEADLOCK - CAS called recursively.
537		EAGAIN && r28 == 1 - CAS is busy. Lock contended.
538		EAGAIN && r28 == 2 - CAS is busy. ldcw failed.
539		EFAULT - Read or write failed.
540
541		Scratch: r20, r28, r1
542
543	****************************************************/
544
545	/* Do not enable LWS debugging */
546#define ENABLE_LWS_DEBUG 0 
547
548	/* ELF64 Process entry path */
549lws_compare_and_swap64:
550#ifdef CONFIG_64BIT
551	b,n	lws_compare_and_swap
552#else
553	/* If we are not a 64-bit kernel, then we don't
554	 * have 64-bit input registers, and calling
555	 * the 64-bit LWS CAS returns ENOSYS.
556	 */
557	b,n	lws_exit_nosys
558#endif
559
560	/* ELF32 Process entry path */
561lws_compare_and_swap32:
562#ifdef CONFIG_64BIT
563	/* Clip all the input registers */
564	depdi	0, 31, 32, %r26
565	depdi	0, 31, 32, %r25
566	depdi	0, 31, 32, %r24
567#endif
568
569lws_compare_and_swap:
570	/* Load start of lock table */
571	ldil	L%lws_lock_start, %r20
572	ldo	R%lws_lock_start(%r20), %r28
573
574	/* Extract four bits from r26 and hash lock (Bits 4-7) */
575	extru  %r26, 27, 4, %r20
576
577	/* Find lock to use, the hash is either one of 0 to
578	   15, multiplied by 16 (keep it 16-byte aligned)
579	   and add to the lock table offset. */
580	shlw	%r20, 4, %r20
581	add	%r20, %r28, %r20
582
583# if ENABLE_LWS_DEBUG
584	/*	
585		DEBUG, check for deadlock! 
586		If the thread register values are the same
587		then we were the one that locked it last and
588		this is a recurisve call that will deadlock.
589		We *must* giveup this call and fail.
590	*/
591	ldw	4(%sr2,%r20), %r28			/* Load thread register */
592	/* WARNING: If cr27 cycles to the same value we have problems */
593	mfctl	%cr27, %r21				/* Get current thread register */
594	cmpb,<>,n	%r21, %r28, cas_lock		/* Called recursive? */
595	b	lws_exit				/* Return error! */
596	ldo	-EDEADLOCK(%r0), %r21
597cas_lock:
598	cmpb,=,n	%r0, %r28, cas_nocontend	/* Is nobody using it? */
599	ldo	1(%r0), %r28				/* 1st case */
600	b	lws_exit				/* Contended... */
601	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
602cas_nocontend:
603# endif
604/* ENABLE_LWS_DEBUG */
605
606	rsm	PSW_SM_I, %r0				/* Disable interrupts */
607	/* COW breaks can cause contention on UP systems */
608	LDCW	0(%sr2,%r20), %r28			/* Try to acquire the lock */
609	cmpb,<>,n	%r0, %r28, cas_action		/* Did we get it? */
610cas_wouldblock:
611	ldo	2(%r0), %r28				/* 2nd case */
612	ssm	PSW_SM_I, %r0
613	b	lws_exit				/* Contended... */
614	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
615
616	/*
617		prev = *addr;
618		if ( prev == old )
619		  *addr = new;
620		return prev;
621	*/
622
623	/* NOTES:
624		This all works becuse intr_do_signal
625		and schedule both check the return iasq
626		and see that we are on the kernel page
627		so this process is never scheduled off
628		or is ever sent any signal of any sort,
629		thus it is wholly atomic from usrspaces
630		perspective
631	*/
632cas_action:
633#if defined CONFIG_SMP && ENABLE_LWS_DEBUG
634	/* DEBUG */
635	mfctl	%cr27, %r1
636	stw	%r1, 4(%sr2,%r20)
637#endif
638	/* The load and store could fail */
6391:	ldw	0(%r26), %r28
640	sub,<>	%r28, %r25, %r0
6412:	stw	%r24, 0(%r26)
642	/* Free lock */
643#ifdef CONFIG_SMP
64498:	LDCW	0(%sr2,%r20), %r1			/* Barrier */
64599:	ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
646#endif
647	stw	%r20, 0(%sr2,%r20)
648#if ENABLE_LWS_DEBUG
649	/* Clear thread register indicator */
650	stw	%r0, 4(%sr2,%r20)
651#endif
652	/* Enable interrupts */
653	ssm	PSW_SM_I, %r0
654	/* Return to userspace, set no error */
655	b	lws_exit
656	copy	%r0, %r21
657
6583:		
659	/* Error occurred on load or store */
660	/* Free lock */
661#ifdef CONFIG_SMP
66298:	LDCW	0(%sr2,%r20), %r1			/* Barrier */
66399:	ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
664#endif
665	stw	%r20, 0(%sr2,%r20)
666#if ENABLE_LWS_DEBUG
667	stw	%r0, 4(%sr2,%r20)
668#endif
669	ssm	PSW_SM_I, %r0
670	b	lws_exit
671	ldo	-EFAULT(%r0),%r21	/* set errno */
672	nop
673	nop
674	nop
675	nop
676
677	/* Two exception table entries, one for the load,
678	   the other for the store. Either return -EFAULT.
679	   Each of the entries must be relocated. */
680	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page)
681	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page)
682
683
684	/***************************************************
685		New CAS implementation which uses pointers and variable size
686		information. The value pointed by old and new MUST NOT change
687		while performing CAS. The lock only protect the value at %r26.
688
689		%r26 - Address to examine
690		%r25 - Pointer to the value to check (old)
691		%r24 - Pointer to the value to set (new)
692		%r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit)
693		%r28 - Return non-zero on failure
694		%r21 - Kernel error code
695
696		%r21 has the following meanings:
697
698		EAGAIN - CAS is busy, ldcw failed, try again.
699		EFAULT - Read or write failed.
700
701		Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only)
702
703	****************************************************/
704
705	/* ELF32 Process entry path */
706lws_compare_and_swap_2:
707#ifdef CONFIG_64BIT
708	/* Clip the input registers. We don't need to clip %r23 as we
709	   only use it for word operations */
710	depdi	0, 31, 32, %r26
711	depdi	0, 31, 32, %r25
712	depdi	0, 31, 32, %r24
713#endif
714
715	/* Check the validity of the size pointer */
716	subi,>>= 3, %r23, %r0
717	b,n	lws_exit_nosys
718
719	/* Jump to the functions which will load the old and new values into
720	   registers depending on the their size */
721	shlw	%r23, 2, %r29
722	blr	%r29, %r0
723	nop
724
725	/* 8bit load */
7264:	ldb	0(%r25), %r25
727	b	cas2_lock_start
7285:	ldb	0(%r24), %r24
729	nop
730	nop
731	nop
732	nop
733	nop
734
735	/* 16bit load */
7366:	ldh	0(%r25), %r25
737	b	cas2_lock_start
7387:	ldh	0(%r24), %r24
739	nop
740	nop
741	nop
742	nop
743	nop
744
745	/* 32bit load */
7468:	ldw	0(%r25), %r25
747	b	cas2_lock_start
7489:	ldw	0(%r24), %r24
749	nop
750	nop
751	nop
752	nop
753	nop
754
755	/* 64bit load */
756#ifdef CONFIG_64BIT
75710:	ldd	0(%r25), %r25
75811:	ldd	0(%r24), %r24
759#else
760	/* Load old value into r22/r23 - high/low */
76110:	ldw	0(%r25), %r22
76211:	ldw	4(%r25), %r23
763	/* Load new value into fr4 for atomic store later */
76412:	flddx	0(%r24), %fr4
765#endif
766
767cas2_lock_start:
768	/* Load start of lock table */
769	ldil	L%lws_lock_start, %r20
770	ldo	R%lws_lock_start(%r20), %r28
771
772	/* Extract four bits from r26 and hash lock (Bits 4-7) */
773	extru  %r26, 27, 4, %r20
774
775	/* Find lock to use, the hash is either one of 0 to
776	   15, multiplied by 16 (keep it 16-byte aligned)
777	   and add to the lock table offset. */
778	shlw	%r20, 4, %r20
779	add	%r20, %r28, %r20
780
781	rsm	PSW_SM_I, %r0			/* Disable interrupts */
782	/* COW breaks can cause contention on UP systems */
783	LDCW	0(%sr2,%r20), %r28		/* Try to acquire the lock */
784	cmpb,<>,n	%r0, %r28, cas2_action	/* Did we get it? */
785cas2_wouldblock:
786	ldo	2(%r0), %r28			/* 2nd case */
787	ssm	PSW_SM_I, %r0
788	b	lws_exit			/* Contended... */
789	ldo	-EAGAIN(%r0), %r21		/* Spin in userspace */
790
791	/*
792		prev = *addr;
793		if ( prev == old )
794		  *addr = new;
795		return prev;
796	*/
797
798	/* NOTES:
799		This all works becuse intr_do_signal
800		and schedule both check the return iasq
801		and see that we are on the kernel page
802		so this process is never scheduled off
803		or is ever sent any signal of any sort,
804		thus it is wholly atomic from usrspaces
805		perspective
806	*/
807cas2_action:
808	/* Jump to the correct function */
809	blr	%r29, %r0
810	/* Set %r28 as non-zero for now */
811	ldo	1(%r0),%r28
812
813	/* 8bit CAS */
81413:	ldb	0(%r26), %r29
815	sub,=	%r29, %r25, %r0
816	b,n	cas2_end
81714:	stb	%r24, 0(%r26)
818	b	cas2_end
819	copy	%r0, %r28
820	nop
821	nop
822
823	/* 16bit CAS */
82415:	ldh	0(%r26), %r29
825	sub,=	%r29, %r25, %r0
826	b,n	cas2_end
82716:	sth	%r24, 0(%r26)
828	b	cas2_end
829	copy	%r0, %r28
830	nop
831	nop
832
833	/* 32bit CAS */
83417:	ldw	0(%r26), %r29
835	sub,=	%r29, %r25, %r0
836	b,n	cas2_end
83718:	stw	%r24, 0(%r26)
838	b	cas2_end
839	copy	%r0, %r28
840	nop
841	nop
842
843	/* 64bit CAS */
844#ifdef CONFIG_64BIT
84519:	ldd	0(%r26), %r29
846	sub,*=	%r29, %r25, %r0
847	b,n	cas2_end
84820:	std	%r24, 0(%r26)
849	copy	%r0, %r28
850#else
851	/* Compare first word */
85219:	ldw	0(%r26), %r29
853	sub,=	%r29, %r22, %r0
854	b,n	cas2_end
855	/* Compare second word */
85620:	ldw	4(%r26), %r29
857	sub,=	%r29, %r23, %r0
858	b,n	cas2_end
859	/* Perform the store */
86021:	fstdx	%fr4, 0(%r26)
861	copy	%r0, %r28
862#endif
863
864cas2_end:
865	/* Free lock */
866#ifdef CONFIG_SMP
86798:	LDCW	0(%sr2,%r20), %r1			/* Barrier */
86899:	ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
869#endif
870	stw	%r20, 0(%sr2,%r20)
871	/* Enable interrupts */
872	ssm	PSW_SM_I, %r0
873	/* Return to userspace, set no error */
874	b	lws_exit
875	copy	%r0, %r21
876
87722:
878	/* Error occurred on load or store */
879	/* Free lock */
880#ifdef CONFIG_SMP
88198:	LDCW	0(%sr2,%r20), %r1			/* Barrier */
88299:	ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)
883#endif
884	stw	%r20, 0(%sr2,%r20)
885	ssm	PSW_SM_I, %r0
886	ldo	1(%r0),%r28
887	b	lws_exit
888	ldo	-EFAULT(%r0),%r21	/* set errno */
889	nop
890	nop
891	nop
892
893	/* Exception table entries, for the load and store, return EFAULT.
894	   Each of the entries must be relocated. */
895	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page)
896	ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page)
897	ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page)
898	ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page)
899	ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page)
900	ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page)
901	ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page)
902	ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page)
903	ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page)
904	ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page)
905	ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page)
906	ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page)
907	ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page)
908	ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page)
909	ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page)
910	ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page)
911#ifndef CONFIG_64BIT
912	ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page)
913	ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
914#endif
915
916	/* Make sure nothing else is placed on this page */
917	.align PAGE_SIZE
918END(linux_gateway_page)
919ENTRY(end_linux_gateway_page)
920
921	/* Relocate symbols assuming linux_gateway_page is mapped
922	   to virtual address 0x0 */
923
924#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page)
925
926	.section .rodata,"a"
927
928	.align 8
929	/* Light-weight-syscall table */
930	/* Start of lws table. */
931ENTRY(lws_table)
932	LWS_ENTRY(compare_and_swap32)		/* 0 - ELF32 Atomic 32bit CAS */
933	LWS_ENTRY(compare_and_swap64)		/* 1 - ELF64 Atomic 32bit CAS */
934	LWS_ENTRY(compare_and_swap_2)		/* 2 - ELF32 Atomic 64bit CAS */
935END(lws_table)
936	/* End of lws table */
937
938#define __SYSCALL(nr, entry, nargs) ASM_ULONG_INSN entry
939	.align 8
940ENTRY(sys_call_table)
941	.export sys_call_table,data
942#ifdef CONFIG_64BIT
943#include <asm/syscall_table_c32.h>   /* Compat syscalls */
944#else
945#include <asm/syscall_table_32.h>    /* 32-bit native syscalls */
946#endif
947END(sys_call_table)
948
949#ifdef CONFIG_64BIT
950	.align 8
951ENTRY(sys_call_table64)
952#include <asm/syscall_table_64.h>    /* 64-bit native syscalls */
953END(sys_call_table64)
954#endif
955#undef __SYSCALL
956
957	/*
958		All light-weight-syscall atomic operations 
959		will use this set of locks 
960
961		NOTE: The lws_lock_start symbol must be
962		at least 16-byte aligned for safe use
963		with ldcw.
964	*/
965	.section .data
966	.align	L1_CACHE_BYTES
967ENTRY(lws_lock_start)
968	/* lws locks */
969	.rept 16
970	/* Keep locks aligned at 16-bytes */
971	.word 1
972	.word 0 
973	.word 0
974	.word 0
975	.endr
976END(lws_lock_start)
977	.previous
978
979.end
980
981