1/* 
  2 * Linux/PA-RISC Project (http://www.parisc-linux.org/)
  3 * 
  4 * System call entry code / Linux gateway page
  5 * Copyright (c) Matthew Wilcox 1999 <willy@bofh.ai>
  6 * Licensed under the GNU GPL.
  7 * thanks to Philipp Rumpf, Mike Shaver and various others
  8 * sorry about the wall, puffin..
  9 */
 10
 11/*
 12How does the Linux gateway page on PA-RISC work?
 13------------------------------------------------
 14The Linux gateway page on PA-RISC is "special".
 15It actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc
 16terminology it's Execute, promote to PL0) in the page map.  So anything
 17executing on this page executes with kernel level privilege (there's more to it
 18than that: to have this happen, you also have to use a branch with a ,gate
 19completer to activate the privilege promotion).  The upshot is that everything
 20that runs on the gateway page runs at kernel privilege but with the current
 21user process address space (although you have access to kernel space via %sr2).
 22For the 0x100 syscall entry, we redo the space registers to point to the kernel
 23address space (preserving the user address space in %sr3), move to wide mode if
 24required, save the user registers and branch into the kernel syscall entry
 25point.  For all the other functions, we execute at kernel privilege but don't
 26flip address spaces. The basic upshot of this is that these code snippets are
 27executed atomically (because the kernel can't be pre-empted) and they may
 28perform architecturally forbidden (to PL3) operations (like setting control
 29registers).
 30*/
 31
 32
 33#include <asm/asm-offsets.h>
 34#include <asm/unistd.h>
 35#include <asm/errno.h>
 36#include <asm/page.h>
 37#include <asm/psw.h>
 38#include <asm/thread_info.h>
 39#include <asm/assembly.h>
 40#include <asm/processor.h>
 41#include <asm/cache.h>
 42
 43#include <linux/linkage.h>
 44
 45	/* We fill the empty parts of the gateway page with
 46 	 * something that will kill the kernel or a
 47 	 * userspace application.
 48	 */
 49#define KILL_INSN	break	0,0
 50
 51	.level          LEVEL
 52
 53	.text
 54
 55	.import syscall_exit,code
 56	.import syscall_exit_rfi,code
 57
 58	/* Linux gateway page is aliased to virtual page 0 in the kernel
 59	 * address space. Since it is a gateway page it cannot be
 60	 * dereferenced, so null pointers will still fault. We start
 61	 * the actual entry point at 0x100. We put break instructions
 62	 * at the beginning of the page to trap null indirect function
 63	 * pointers.
 64	 */
 65
 66	.align PAGE_SIZE
 67ENTRY(linux_gateway_page)
 68
 69        /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */
 70	.rept 44
 71	KILL_INSN
 72	.endr
 73
 74	/* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */
 75	/* Light-weight-syscall entry must always be located at 0xb0 */
 76	/* WARNING: Keep this number updated with table size changes */
 77#define __NR_lws_entries (3)
 78
 79lws_entry:
 80	gate	lws_start, %r0		/* increase privilege */
 81	depi	3, 31, 2, %r31		/* Ensure we return into user mode. */
 82
 83	/* Fill from 0xb8 to 0xe0 */
 84	.rept 10
 85	KILL_INSN
 86	.endr
 87
 88	/* This function MUST be located at 0xe0 for glibc's threading 
 89	mechanism to work. DO NOT MOVE THIS CODE EVER! */
 90set_thread_pointer:
 91	gate	.+8, %r0		/* increase privilege */
 92	depi	3, 31, 2, %r31		/* Ensure we return into user mode. */
 93	be	0(%sr7,%r31)		/* return to user space */
 94	mtctl	%r26, %cr27		/* move arg0 to the control register */
 95
 96	/* Increase the chance of trapping if random jumps occur to this
 97	address, fill from 0xf0 to 0x100 */
 98	.rept 4
 99	KILL_INSN
100	.endr
101
102/* This address must remain fixed at 0x100 for glibc's syscalls to work */
103	.align 256
104linux_gateway_entry:
105	gate	.+8, %r0			/* become privileged */
106	mtsp	%r0,%sr4			/* get kernel space into sr4 */
107	mtsp	%r0,%sr5			/* get kernel space into sr5 */
108	mtsp	%r0,%sr6			/* get kernel space into sr6 */
109	mfsp    %sr7,%r1                        /* save user sr7 */
110	mtsp    %r1,%sr3                        /* and store it in sr3 */
111
112#ifdef CONFIG_64BIT
113	/* for now we can *always* set the W bit on entry to the syscall
114	 * since we don't support wide userland processes.  We could
115	 * also save the current SM other than in r0 and restore it on
116	 * exit from the syscall, and also use that value to know
117	 * whether to do narrow or wide syscalls. -PB
118	 */
119	ssm	PSW_SM_W, %r1
120	extrd,u	%r1,PSW_W_BIT,1,%r1
121	/* sp must be aligned on 4, so deposit the W bit setting into
122	 * the bottom of sp temporarily */
123	or,ev	%r1,%r30,%r30
124	b,n	1f
125	/* The top halves of argument registers must be cleared on syscall
126	 * entry from narrow executable.
127	 */
128	depdi	0, 31, 32, %r26
129	depdi	0, 31, 32, %r25
130	depdi	0, 31, 32, %r24
131	depdi	0, 31, 32, %r23
132	depdi	0, 31, 32, %r22
133	depdi	0, 31, 32, %r21
1341:	
135#endif
 
 
 
 
 
 
 
 
136	mfctl   %cr30,%r1
137	xor     %r1,%r30,%r30                   /* ye olde xor trick */
138	xor     %r1,%r30,%r1
139	xor     %r1,%r30,%r30
140	
141	ldo     THREAD_SZ_ALGN+FRAME_SIZE(%r30),%r30  /* set up kernel stack */
142
143	/* N.B.: It is critical that we don't set sr7 to 0 until r30
144	 *       contains a valid kernel stack pointer. It is also
145	 *       critical that we don't start using the kernel stack
146	 *       until after sr7 has been set to 0.
147	 */
148
149	mtsp	%r0,%sr7			/* get kernel space into sr7 */
 
150	STREGM	%r1,FRAME_SIZE(%r30)		/* save r1 (usp) here for now */
151	mfctl	%cr30,%r1			/* get task ptr in %r1 */
152	LDREG	TI_TASK(%r1),%r1
153
154	/* Save some registers for sigcontext and potential task
155	   switch (see entry.S for the details of which ones are
156	   saved/restored).  TASK_PT_PSW is zeroed so we can see whether
157	   a process is on a syscall or not.  For an interrupt the real
158	   PSW value is stored.  This is needed for gdb and sys_ptrace. */
159	STREG	%r0,  TASK_PT_PSW(%r1)
160	STREG	%r2,  TASK_PT_GR2(%r1)		/* preserve rp */
161	STREG	%r19, TASK_PT_GR19(%r1)
162
163	LDREGM	-FRAME_SIZE(%r30), %r2		/* get users sp back */
164#ifdef CONFIG_64BIT
165	extrd,u	%r2,63,1,%r19			/* W hidden in bottom bit */
166#if 0
167	xor	%r19,%r2,%r2			/* clear bottom bit */
168	depd,z	%r19,1,1,%r19
169	std	%r19,TASK_PT_PSW(%r1)
170#endif
171#endif
172	STREG	%r2,  TASK_PT_GR30(%r1)		/* ... and save it */
173	
174	STREG	%r20, TASK_PT_GR20(%r1)		/* Syscall number */
175	STREG	%r21, TASK_PT_GR21(%r1)
176	STREG	%r22, TASK_PT_GR22(%r1)
177	STREG	%r23, TASK_PT_GR23(%r1)		/* 4th argument */
178	STREG	%r24, TASK_PT_GR24(%r1)		/* 3rd argument */
179	STREG	%r25, TASK_PT_GR25(%r1)		/* 2nd argument */
180	STREG	%r26, TASK_PT_GR26(%r1)	 	/* 1st argument */
181	STREG	%r27, TASK_PT_GR27(%r1)		/* user dp */
182	STREG   %r28, TASK_PT_GR28(%r1)         /* return value 0 */
183	STREG   %r0, TASK_PT_ORIG_R28(%r1)      /* don't prohibit restarts */
184	STREG	%r29, TASK_PT_GR29(%r1)		/* return value 1 */
185	STREG	%r31, TASK_PT_GR31(%r1)		/* preserve syscall return ptr */
186	
187	ldo	TASK_PT_FR0(%r1), %r27		/* save fpregs from the kernel */
188	save_fp	%r27				/* or potential task switch  */
189
190	mfctl	%cr11, %r27			/* i.e. SAR */
191	STREG	%r27, TASK_PT_SAR(%r1)
192
193	loadgp
194
195#ifdef CONFIG_64BIT
196	ldo	-16(%r30),%r29			/* Reference param save area */
197	copy	%r19,%r2			/* W bit back to r2 */
198#else
199	/* no need to save these on stack in wide mode because the first 8
200	 * args are passed in registers */
201	stw     %r22, -52(%r30)                 /* 5th argument */
202	stw     %r21, -56(%r30)                 /* 6th argument */
203#endif
204
205	/* Are we being ptraced? */
206	mfctl	%cr30, %r1
207	LDREG	TI_FLAGS(%r1),%r1
208	ldi	_TIF_SYSCALL_TRACE_MASK, %r19
209	and,COND(=) %r1, %r19, %r0
210	b,n	.Ltracesys
211	
212	/* Note!  We cannot use the syscall table that is mapped
213	nearby since the gateway page is mapped execute-only. */
214
215#ifdef CONFIG_64BIT
216	ldil	L%sys_call_table, %r1
217	or,=	%r2,%r2,%r2
218	addil	L%(sys_call_table64-sys_call_table), %r1
219	ldo	R%sys_call_table(%r1), %r19
220	or,=	%r2,%r2,%r2
221	ldo	R%sys_call_table64(%r1), %r19
222#else
223	ldil	L%sys_call_table, %r1
224	ldo     R%sys_call_table(%r1), %r19
225#endif	
226	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
227	b,n	.Lsyscall_nosys
228	
229	LDREGX  %r20(%r19), %r19
230
231	/* If this is a sys_rt_sigreturn call, and the signal was received
232	 * when not in_syscall, then we want to return via syscall_exit_rfi,
233	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
234	 * trampoline code in signal.c).
235	 */
236	ldi	__NR_rt_sigreturn,%r2
237	comb,=	%r2,%r20,.Lrt_sigreturn
238.Lin_syscall:
239	ldil	L%syscall_exit,%r2
240	be      0(%sr7,%r19)
241	ldo	R%syscall_exit(%r2),%r2
242.Lrt_sigreturn:
243	comib,<> 0,%r25,.Lin_syscall
244	ldil	L%syscall_exit_rfi,%r2
245	be      0(%sr7,%r19)
246	ldo	R%syscall_exit_rfi(%r2),%r2
247
248	/* Note!  Because we are not running where we were linked, any
249	calls to functions external to this file must be indirect.  To
250	be safe, we apply the opposite rule to functions within this
251	file, with local labels given to them to ensure correctness. */
252	
253.Lsyscall_nosys:
254syscall_nosys:
255	ldil	L%syscall_exit,%r1
256	be	R%syscall_exit(%sr7,%r1)
257	ldo	-ENOSYS(%r0),%r28		   /* set errno */
258
259
260/* Warning! This trace code is a virtual duplicate of the code above so be
261 * sure to maintain both! */
262.Ltracesys:
263tracesys:
264	/* Need to save more registers so the debugger can see where we
265	 * are.  This saves only the lower 8 bits of PSW, so that the C
266	 * bit is still clear on syscalls, and the D bit is set if this
267	 * full register save path has been executed.  We check the D
268	 * bit on syscall_return_rfi to determine which registers to
269	 * restore.  An interrupt results in a full PSW saved with the
270	 * C bit set, a non-straced syscall entry results in C and D clear
271	 * in the saved PSW.
272	 */
273	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
274	LDREG	TI_TASK(%r1), %r1
275	ssm	0,%r2
276	STREG	%r2,TASK_PT_PSW(%r1)		/* Lower 8 bits only!! */
277	mfsp	%sr0,%r2
278	STREG	%r2,TASK_PT_SR0(%r1)
279	mfsp	%sr1,%r2
280	STREG	%r2,TASK_PT_SR1(%r1)
281	mfsp	%sr2,%r2
282	STREG	%r2,TASK_PT_SR2(%r1)
283	mfsp	%sr3,%r2
284	STREG	%r2,TASK_PT_SR3(%r1)
285	STREG	%r2,TASK_PT_SR4(%r1)
286	STREG	%r2,TASK_PT_SR5(%r1)
287	STREG	%r2,TASK_PT_SR6(%r1)
288	STREG	%r2,TASK_PT_SR7(%r1)
289	STREG	%r2,TASK_PT_IASQ0(%r1)
290	STREG	%r2,TASK_PT_IASQ1(%r1)
291	LDREG	TASK_PT_GR31(%r1),%r2
292	STREG	%r2,TASK_PT_IAOQ0(%r1)
293	ldo	4(%r2),%r2
294	STREG	%r2,TASK_PT_IAOQ1(%r1)
295	ldo	TASK_REGS(%r1),%r2
296	/* reg_save %r2 */
297	STREG	%r3,PT_GR3(%r2)
298	STREG	%r4,PT_GR4(%r2)
299	STREG	%r5,PT_GR5(%r2)
300	STREG	%r6,PT_GR6(%r2)
301	STREG	%r7,PT_GR7(%r2)
302	STREG	%r8,PT_GR8(%r2)
303	STREG	%r9,PT_GR9(%r2)
304	STREG	%r10,PT_GR10(%r2)
305	STREG	%r11,PT_GR11(%r2)
306	STREG	%r12,PT_GR12(%r2)
307	STREG	%r13,PT_GR13(%r2)
308	STREG	%r14,PT_GR14(%r2)
309	STREG	%r15,PT_GR15(%r2)
310	STREG	%r16,PT_GR16(%r2)
311	STREG	%r17,PT_GR17(%r2)
312	STREG	%r18,PT_GR18(%r2)
313	/* Finished saving things for the debugger */
314
315	copy	%r2,%r26
316	ldil	L%do_syscall_trace_enter,%r1
317	ldil	L%tracesys_next,%r2
318	be	R%do_syscall_trace_enter(%sr7,%r1)
319	ldo	R%tracesys_next(%r2),%r2
320	
321tracesys_next:
322	/* do_syscall_trace_enter either returned the syscallno, or -1L,
323	 *  so we skip restoring the PT_GR20 below, since we pulled it from
324	 *  task->thread.regs.gr[20] above.
325	 */
326	copy	%ret0,%r20
327	ldil	L%sys_call_table,%r1
328	ldo     R%sys_call_table(%r1), %r19
329
330	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
331	LDREG	TI_TASK(%r1), %r1
332	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return value */
333	LDREG   TASK_PT_GR26(%r1), %r26		/* Restore the users args */
334	LDREG   TASK_PT_GR25(%r1), %r25
335	LDREG   TASK_PT_GR24(%r1), %r24
336	LDREG   TASK_PT_GR23(%r1), %r23
337	LDREG   TASK_PT_GR22(%r1), %r22
338	LDREG   TASK_PT_GR21(%r1), %r21
339#ifdef CONFIG_64BIT
340	ldo	-16(%r30),%r29			/* Reference param save area */
341#else
342	stw     %r22, -52(%r30)                 /* 5th argument */
343	stw     %r21, -56(%r30)                 /* 6th argument */
344#endif
345
346	cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */
347	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
348	b,n	.Ltracesys_nosys
349
350	LDREGX  %r20(%r19), %r19
351
352	/* If this is a sys_rt_sigreturn call, and the signal was received
353	 * when not in_syscall, then we want to return via syscall_exit_rfi,
354	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
355	 * trampoline code in signal.c).
356	 */
357	ldi	__NR_rt_sigreturn,%r2
358	comb,=	%r2,%r20,.Ltrace_rt_sigreturn
359.Ltrace_in_syscall:
360	ldil	L%tracesys_exit,%r2
361	be      0(%sr7,%r19)
362	ldo	R%tracesys_exit(%r2),%r2
363
364.Ltracesys_nosys:
365	ldo	-ENOSYS(%r0),%r28		/* set errno */
366
367	/* Do *not* call this function on the gateway page, because it
368	makes a direct call to syscall_trace. */
369	
370tracesys_exit:
371	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
372	LDREG	TI_TASK(%r1), %r1
373#ifdef CONFIG_64BIT
374	ldo	-16(%r30),%r29			/* Reference param save area */
375#endif
376	ldo	TASK_REGS(%r1),%r26
377	BL	do_syscall_trace_exit,%r2
378	STREG   %r28,TASK_PT_GR28(%r1)          /* save return value now */
379	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
380	LDREG	TI_TASK(%r1), %r1
381	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return val. */
382
383	ldil	L%syscall_exit,%r1
384	be,n	R%syscall_exit(%sr7,%r1)
385
386.Ltrace_rt_sigreturn:
387	comib,<> 0,%r25,.Ltrace_in_syscall
388	ldil	L%tracesys_sigexit,%r2
389	be      0(%sr7,%r19)
390	ldo	R%tracesys_sigexit(%r2),%r2
391
392tracesys_sigexit:
393	ldo     -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1      /* get task ptr */
394	LDREG	TI_TASK(%r1), %r1
395#ifdef CONFIG_64BIT
396	ldo	-16(%r30),%r29			/* Reference param save area */
397#endif
398	BL	do_syscall_trace_exit,%r2
399	ldo	TASK_REGS(%r1),%r26
400
401	ldil	L%syscall_exit_rfi,%r1
402	be,n	R%syscall_exit_rfi(%sr7,%r1)
403
404
405	/*********************************************************
406		32/64-bit Light-Weight-Syscall ABI
407
408		* - Indicates a hint for userspace inline asm
409		implementations.
410
411		Syscall number (caller-saves)
412	        - %r20
413	        * In asm clobber.
414
415		Argument registers (caller-saves)
416	        - %r26, %r25, %r24, %r23, %r22
417	        * In asm input.
418
419		Return registers (caller-saves)
420	        - %r28 (return), %r21 (errno)
421	        * In asm output.
422
423		Caller-saves registers
424	        - %r1, %r27, %r29
425	        - %r2 (return pointer)
426	        - %r31 (ble link register)
427	        * In asm clobber.
428
429		Callee-saves registers
430	        - %r3-%r18
431	        - %r30 (stack pointer)
432	        * Not in asm clobber.
433
434		If userspace is 32-bit:
435		Callee-saves registers
436	        - %r19 (32-bit PIC register)
437
438		Differences from 32-bit calling convention:
439		- Syscall number in %r20
440		- Additional argument register %r22 (arg4)
441		- Callee-saves %r19.
442
443		If userspace is 64-bit:
444		Callee-saves registers
445		- %r27 (64-bit PIC register)
446
447		Differences from 64-bit calling convention:
448		- Syscall number in %r20
449		- Additional argument register %r22 (arg4)
450		- Callee-saves %r27.
451
452		Error codes returned by entry path:
453
454		ENOSYS - r20 was an invalid LWS number.
455
456	*********************************************************/
457lws_start:
458
459#ifdef CONFIG_64BIT
460	/* FIXME: If we are a 64-bit kernel just
461	 *        turn this on unconditionally.
462	 */
463	ssm	PSW_SM_W, %r1
464	extrd,u	%r1,PSW_W_BIT,1,%r1
465	/* sp must be aligned on 4, so deposit the W bit setting into
466	 * the bottom of sp temporarily */
467	or,ev	%r1,%r30,%r30
468
469	/* Clip LWS number to a 32-bit value always */
470	depdi	0, 31, 32, %r20
471#endif	
472
473        /* Is the lws entry number valid? */
474	comiclr,>>	__NR_lws_entries, %r20, %r0
475	b,n	lws_exit_nosys
476
477	/* WARNING: Trashing sr2 and sr3 */
478	mfsp	%sr7,%r1			/* get userspace into sr3 */
479	mtsp	%r1,%sr3
480	mtsp	%r0,%sr2			/* get kernel space into sr2 */
481
482	/* Load table start */
483	ldil	L%lws_table, %r1
484	ldo	R%lws_table(%r1), %r28	/* Scratch use of r28 */
485	LDREGX	%r20(%sr2,r28), %r21	/* Scratch use of r21 */
486
487	/* Jump to lws, lws table pointers already relocated */
488	be,n	0(%sr2,%r21)
489
490lws_exit_nosys:
491	ldo	-ENOSYS(%r0),%r21		   /* set errno */
492	/* Fall through: Return to userspace */
493
494lws_exit:
495#ifdef CONFIG_64BIT
496	/* decide whether to reset the wide mode bit
497	 *
498	 * For a syscall, the W bit is stored in the lowest bit
499	 * of sp.  Extract it and reset W if it is zero */
500	extrd,u,*<>	%r30,63,1,%r1
501	rsm	PSW_SM_W, %r0
502	/* now reset the lowest bit of sp if it was set */
503	xor	%r30,%r1,%r30
504#endif
505	be,n	0(%sr7, %r31)
506
507
508	
509	/***************************************************
510		Implementing 32bit CAS as an atomic operation:
511
512		%r26 - Address to examine
513		%r25 - Old value to check (old)
514		%r24 - New value to set (new)
515		%r28 - Return prev through this register.
516		%r21 - Kernel error code
517
518		If debugging is DISabled:
519
520		%r21 has the following meanings:
521
522		EAGAIN - CAS is busy, ldcw failed, try again.
523		EFAULT - Read or write failed.		
524
525		If debugging is enabled:
526
527		EDEADLOCK - CAS called recursively.
528		EAGAIN && r28 == 1 - CAS is busy. Lock contended.
529		EAGAIN && r28 == 2 - CAS is busy. ldcw failed.
530		EFAULT - Read or write failed.
531
532		Scratch: r20, r28, r1
533
534	****************************************************/
535
536	/* Do not enable LWS debugging */
537#define ENABLE_LWS_DEBUG 0 
538
539	/* ELF64 Process entry path */
540lws_compare_and_swap64:
541#ifdef CONFIG_64BIT
542	b,n	lws_compare_and_swap
543#else
544	/* If we are not a 64-bit kernel, then we don't
545	 * have 64-bit input registers, and calling
546	 * the 64-bit LWS CAS returns ENOSYS.
547	 */
548	b,n	lws_exit_nosys
549#endif
550
551	/* ELF32 Process entry path */
552lws_compare_and_swap32:
553#ifdef CONFIG_64BIT
554	/* Clip all the input registers */
555	depdi	0, 31, 32, %r26
556	depdi	0, 31, 32, %r25
557	depdi	0, 31, 32, %r24
558#endif
559
560lws_compare_and_swap:
561	/* Load start of lock table */
562	ldil	L%lws_lock_start, %r20
563	ldo	R%lws_lock_start(%r20), %r28
564
565	/* Extract four bits from r26 and hash lock (Bits 4-7) */
566	extru  %r26, 27, 4, %r20
567
568	/* Find lock to use, the hash is either one of 0 to
569	   15, multiplied by 16 (keep it 16-byte aligned)
570	   and add to the lock table offset. */
571	shlw	%r20, 4, %r20
572	add	%r20, %r28, %r20
573
574# if ENABLE_LWS_DEBUG
575	/*	
576		DEBUG, check for deadlock! 
577		If the thread register values are the same
578		then we were the one that locked it last and
579		this is a recurisve call that will deadlock.
580		We *must* giveup this call and fail.
581	*/
582	ldw	4(%sr2,%r20), %r28			/* Load thread register */
583	/* WARNING: If cr27 cycles to the same value we have problems */
584	mfctl	%cr27, %r21				/* Get current thread register */
585	cmpb,<>,n	%r21, %r28, cas_lock		/* Called recursive? */
586	b	lws_exit				/* Return error! */
587	ldo	-EDEADLOCK(%r0), %r21
588cas_lock:
589	cmpb,=,n	%r0, %r28, cas_nocontend	/* Is nobody using it? */
590	ldo	1(%r0), %r28				/* 1st case */
591	b	lws_exit				/* Contended... */
592	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
593cas_nocontend:
594# endif
595/* ENABLE_LWS_DEBUG */
596
597	rsm	PSW_SM_I, %r0				/* Disable interrupts */
598	/* COW breaks can cause contention on UP systems */
599	LDCW	0(%sr2,%r20), %r28			/* Try to acquire the lock */
600	cmpb,<>,n	%r0, %r28, cas_action		/* Did we get it? */
601cas_wouldblock:
602	ldo	2(%r0), %r28				/* 2nd case */
603	ssm	PSW_SM_I, %r0
604	b	lws_exit				/* Contended... */
605	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
606
607	/*
608		prev = *addr;
609		if ( prev == old )
610		  *addr = new;
611		return prev;
612	*/
613
614	/* NOTES:
615		This all works becuse intr_do_signal
616		and schedule both check the return iasq
617		and see that we are on the kernel page
618		so this process is never scheduled off
619		or is ever sent any signal of any sort,
620		thus it is wholly atomic from usrspaces
621		perspective
622	*/
623cas_action:
624#if defined CONFIG_SMP && ENABLE_LWS_DEBUG
625	/* DEBUG */
626	mfctl	%cr27, %r1
627	stw	%r1, 4(%sr2,%r20)
628#endif
629	/* The load and store could fail */
6301:	ldw,ma	0(%sr3,%r26), %r28
631	sub,<>	%r28, %r25, %r0
6322:	stw,ma	%r24, 0(%sr3,%r26)
633	/* Free lock */
634	stw,ma	%r20, 0(%sr2,%r20)
635#if ENABLE_LWS_DEBUG
636	/* Clear thread register indicator */
637	stw	%r0, 4(%sr2,%r20)
638#endif
639	/* Enable interrupts */
640	ssm	PSW_SM_I, %r0
641	/* Return to userspace, set no error */
642	b	lws_exit
643	copy	%r0, %r21
644
6453:		
646	/* Error occurred on load or store */
647	/* Free lock */
648	stw	%r20, 0(%sr2,%r20)
649#if ENABLE_LWS_DEBUG
650	stw	%r0, 4(%sr2,%r20)
651#endif
652	ssm	PSW_SM_I, %r0
653	b	lws_exit
654	ldo	-EFAULT(%r0),%r21	/* set errno */
655	nop
656	nop
657	nop
658	nop
659
660	/* Two exception table entries, one for the load,
661	   the other for the store. Either return -EFAULT.
662	   Each of the entries must be relocated. */
663	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page)
664	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page)
665
666
667	/***************************************************
668		New CAS implementation which uses pointers and variable size
669		information. The value pointed by old and new MUST NOT change
670		while performing CAS. The lock only protect the value at %r26.
671
672		%r26 - Address to examine
673		%r25 - Pointer to the value to check (old)
674		%r24 - Pointer to the value to set (new)
675		%r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit)
676		%r28 - Return non-zero on failure
677		%r21 - Kernel error code
678
679		%r21 has the following meanings:
680
681		EAGAIN - CAS is busy, ldcw failed, try again.
682		EFAULT - Read or write failed.
683
684		Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only)
685
686	****************************************************/
687
688	/* ELF32 Process entry path */
689lws_compare_and_swap_2:
690#ifdef CONFIG_64BIT
691	/* Clip the input registers */
 
692	depdi	0, 31, 32, %r26
693	depdi	0, 31, 32, %r25
694	depdi	0, 31, 32, %r24
695	depdi	0, 31, 32, %r23
696#endif
697
698	/* Check the validity of the size pointer */
699	subi,>>= 4, %r23, %r0
700	b,n	lws_exit_nosys
701
702	/* Jump to the functions which will load the old and new values into
703	   registers depending on the their size */
704	shlw	%r23, 2, %r29
705	blr	%r29, %r0
706	nop
707
708	/* 8bit load */
7094:	ldb	0(%sr3,%r25), %r25
710	b	cas2_lock_start
7115:	ldb	0(%sr3,%r24), %r24
712	nop
713	nop
714	nop
715	nop
716	nop
717
718	/* 16bit load */
7196:	ldh	0(%sr3,%r25), %r25
720	b	cas2_lock_start
7217:	ldh	0(%sr3,%r24), %r24
722	nop
723	nop
724	nop
725	nop
726	nop
727
728	/* 32bit load */
7298:	ldw	0(%sr3,%r25), %r25
730	b	cas2_lock_start
7319:	ldw	0(%sr3,%r24), %r24
732	nop
733	nop
734	nop
735	nop
736	nop
737
738	/* 64bit load */
739#ifdef CONFIG_64BIT
74010:	ldd	0(%sr3,%r25), %r25
74111:	ldd	0(%sr3,%r24), %r24
742#else
743	/* Load new value into r22/r23 - high/low */
74410:	ldw	0(%sr3,%r25), %r22
74511:	ldw	4(%sr3,%r25), %r23
746	/* Load new value into fr4 for atomic store later */
74712:	flddx	0(%sr3,%r24), %fr4
748#endif
749
750cas2_lock_start:
751	/* Load start of lock table */
752	ldil	L%lws_lock_start, %r20
753	ldo	R%lws_lock_start(%r20), %r28
754
755	/* Extract four bits from r26 and hash lock (Bits 4-7) */
756	extru  %r26, 27, 4, %r20
757
758	/* Find lock to use, the hash is either one of 0 to
759	   15, multiplied by 16 (keep it 16-byte aligned)
760	   and add to the lock table offset. */
761	shlw	%r20, 4, %r20
762	add	%r20, %r28, %r20
763
764	rsm	PSW_SM_I, %r0			/* Disable interrupts */
765	/* COW breaks can cause contention on UP systems */
766	LDCW	0(%sr2,%r20), %r28		/* Try to acquire the lock */
767	cmpb,<>,n	%r0, %r28, cas2_action	/* Did we get it? */
768cas2_wouldblock:
769	ldo	2(%r0), %r28			/* 2nd case */
770	ssm	PSW_SM_I, %r0
771	b	lws_exit			/* Contended... */
772	ldo	-EAGAIN(%r0), %r21		/* Spin in userspace */
773
774	/*
775		prev = *addr;
776		if ( prev == old )
777		  *addr = new;
778		return prev;
779	*/
780
781	/* NOTES:
782		This all works becuse intr_do_signal
783		and schedule both check the return iasq
784		and see that we are on the kernel page
785		so this process is never scheduled off
786		or is ever sent any signal of any sort,
787		thus it is wholly atomic from usrspaces
788		perspective
789	*/
790cas2_action:
791	/* Jump to the correct function */
792	blr	%r29, %r0
793	/* Set %r28 as non-zero for now */
794	ldo	1(%r0),%r28
795
796	/* 8bit CAS */
79713:	ldb,ma	0(%sr3,%r26), %r29
798	sub,=	%r29, %r25, %r0
799	b,n	cas2_end
80014:	stb,ma	%r24, 0(%sr3,%r26)
801	b	cas2_end
802	copy	%r0, %r28
803	nop
804	nop
805
806	/* 16bit CAS */
80715:	ldh,ma	0(%sr3,%r26), %r29
808	sub,=	%r29, %r25, %r0
809	b,n	cas2_end
81016:	sth,ma	%r24, 0(%sr3,%r26)
811	b	cas2_end
812	copy	%r0, %r28
813	nop
814	nop
815
816	/* 32bit CAS */
81717:	ldw,ma	0(%sr3,%r26), %r29
818	sub,=	%r29, %r25, %r0
819	b,n	cas2_end
82018:	stw,ma	%r24, 0(%sr3,%r26)
821	b	cas2_end
822	copy	%r0, %r28
823	nop
824	nop
825
826	/* 64bit CAS */
827#ifdef CONFIG_64BIT
82819:	ldd,ma	0(%sr3,%r26), %r29
829	sub,*=	%r29, %r25, %r0
830	b,n	cas2_end
83120:	std,ma	%r24, 0(%sr3,%r26)
832	copy	%r0, %r28
833#else
834	/* Compare first word */
83519:	ldw,ma	0(%sr3,%r26), %r29
836	sub,=	%r29, %r22, %r0
837	b,n	cas2_end
838	/* Compare second word */
83920:	ldw,ma	4(%sr3,%r26), %r29
840	sub,=	%r29, %r23, %r0
841	b,n	cas2_end
842	/* Perform the store */
84321:	fstdx	%fr4, 0(%sr3,%r26)
844	copy	%r0, %r28
845#endif
846
847cas2_end:
848	/* Free lock */
849	stw,ma	%r20, 0(%sr2,%r20)
850	/* Enable interrupts */
851	ssm	PSW_SM_I, %r0
852	/* Return to userspace, set no error */
853	b	lws_exit
854	copy	%r0, %r21
855
85622:
857	/* Error occurred on load or store */
858	/* Free lock */
859	stw	%r20, 0(%sr2,%r20)
860	ssm	PSW_SM_I, %r0
861	ldo	1(%r0),%r28
862	b	lws_exit
863	ldo	-EFAULT(%r0),%r21	/* set errno */
864	nop
865	nop
866	nop
867
868	/* Exception table entries, for the load and store, return EFAULT.
869	   Each of the entries must be relocated. */
870	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page)
871	ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page)
872	ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page)
873	ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page)
874	ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page)
875	ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page)
876	ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page)
877	ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page)
878	ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page)
879	ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page)
880	ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page)
881	ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page)
882	ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page)
883	ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page)
884	ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page)
885	ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page)
886#ifndef CONFIG_64BIT
887	ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page)
888	ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
889#endif
890
891	/* Make sure nothing else is placed on this page */
892	.align PAGE_SIZE
893END(linux_gateway_page)
894ENTRY(end_linux_gateway_page)
895
896	/* Relocate symbols assuming linux_gateway_page is mapped
897	   to virtual address 0x0 */
898
899#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page)
900
901	.section .rodata,"a"
902
903	.align 8
904	/* Light-weight-syscall table */
905	/* Start of lws table. */
906ENTRY(lws_table)
907	LWS_ENTRY(compare_and_swap32)		/* 0 - ELF32 Atomic 32bit CAS */
908	LWS_ENTRY(compare_and_swap64)		/* 1 - ELF64 Atomic 32bit CAS */
909	LWS_ENTRY(compare_and_swap_2)		/* 2 - ELF32 Atomic 64bit CAS */
910END(lws_table)
911	/* End of lws table */
912
913	.align 8
914ENTRY(sys_call_table)
 
915#include "syscall_table.S"
916END(sys_call_table)
917
918#ifdef CONFIG_64BIT
919	.align 8
920ENTRY(sys_call_table64)
921#define SYSCALL_TABLE_64BIT
922#include "syscall_table.S"
923END(sys_call_table64)
924#endif
925
926	/*
927		All light-weight-syscall atomic operations 
928		will use this set of locks 
929
930		NOTE: The lws_lock_start symbol must be
931		at least 16-byte aligned for safe use
932		with ldcw.
933	*/
934	.section .data
935	.align	L1_CACHE_BYTES
936ENTRY(lws_lock_start)
937	/* lws locks */
938	.rept 16
939	/* Keep locks aligned at 16-bytes */
940	.word 1
941	.word 0 
942	.word 0
943	.word 0
944	.endr
945END(lws_lock_start)
946	.previous
947
948.end
949
950