xref: /openbmc/linux/arch/parisc/kernel/syscall.S (revision ee8ec048) 
1/*
2 * Linux/PA-RISC Project (http://www.parisc-linux.org/)
3 *
4 * System call entry code / Linux gateway page
5 * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org>
6 * Licensed under the GNU GPL.
7 * thanks to Philipp Rumpf, Mike Shaver and various others
8 * sorry about the wall, puffin..
9 */
10
11/*
12How does the Linux gateway page on PA-RISC work?
13------------------------------------------------
14The Linux gateway page on PA-RISC is "special".
15It actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc
16terminology it's Execute, promote to PL0) in the page map.  So anything
17executing on this page executes with kernel level privilege (there's more to it
18than that: to have this happen, you also have to use a branch with a ,gate
19completer to activate the privilege promotion).  The upshot is that everything
20that runs on the gateway page runs at kernel privilege but with the current
21user process address space (although you have access to kernel space via %sr2).
22For the 0x100 syscall entry, we redo the space registers to point to the kernel
23address space (preserving the user address space in %sr3), move to wide mode if
24required, save the user registers and branch into the kernel syscall entry
25point.  For all the other functions, we execute at kernel privilege but don't
26flip address spaces. The basic upshot of this is that these code snippets are
27executed atomically (because the kernel can't be pre-empted) and they may
28perform architecturally forbidden (to PL3) operations (like setting control
29registers).
30*/
31
32
33#include <asm/asm-offsets.h>
34#include <asm/unistd.h>
35#include <asm/errno.h>
36#include <asm/page.h>
37#include <asm/psw.h>
38#include <asm/thread_info.h>
39#include <asm/assembly.h>
40#include <asm/processor.h>
41#include <asm/cache.h>
42
43#include <linux/linkage.h>
44
45	/* We fill the empty parts of the gateway page with
46 	 * something that will kill the kernel or a
47 	 * userspace application.
48	 */
49#define KILL_INSN	break	0,0
50
51	.level          PA_ASM_LEVEL
52
53	.text
54
55	.import syscall_exit,code
56	.import syscall_exit_rfi,code
57
58	/* Linux gateway page is aliased to virtual page 0 in the kernel
59	 * address space. Since it is a gateway page it cannot be
60	 * dereferenced, so null pointers will still fault. We start
61	 * the actual entry point at 0x100. We put break instructions
62	 * at the beginning of the page to trap null indirect function
63	 * pointers.
64	 */
65
66	.align PAGE_SIZE
67ENTRY(linux_gateway_page)
68
69        /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */
70	.rept 44
71	KILL_INSN
72	.endr
73
74	/* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */
75	/* Light-weight-syscall entry must always be located at 0xb0 */
76	/* WARNING: Keep this number updated with table size changes */
77#define __NR_lws_entries (3)
78
79lws_entry:
80	gate	lws_start, %r0		/* increase privilege */
81	depi	PRIV_USER, 31, 2, %r31	/* Ensure we return into user mode. */
82
83	/* Fill from 0xb8 to 0xe0 */
84	.rept 10
85	KILL_INSN
86	.endr
87
88	/* This function MUST be located at 0xe0 for glibc's threading
89	mechanism to work. DO NOT MOVE THIS CODE EVER! */
90set_thread_pointer:
91	gate	.+8, %r0		/* increase privilege */
92	depi	PRIV_USER, 31, 2, %r31	/* Ensure we return into user mode. */
93	be	0(%sr7,%r31)		/* return to user space */
94	mtctl	%r26, %cr27		/* move arg0 to the control register */
95
96	/* Increase the chance of trapping if random jumps occur to this
97	address, fill from 0xf0 to 0x100 */
98	.rept 4
99	KILL_INSN
100	.endr
101
102/* This address must remain fixed at 0x100 for glibc's syscalls to work */
103	.align LINUX_GATEWAY_ADDR
104linux_gateway_entry:
105	gate	.+8, %r0			/* become privileged */
106	mtsp	%r0,%sr4			/* get kernel space into sr4 */
107	mtsp	%r0,%sr5			/* get kernel space into sr5 */
108	mtsp	%r0,%sr6			/* get kernel space into sr6 */
109
110#ifdef CONFIG_64BIT
111	/* Store W bit on entry to the syscall in case it's a wide userland
112	 * process. */
113	ssm	PSW_SM_W, %r1
114	extrd,u	%r1,PSW_W_BIT,1,%r1
115	/* sp must be aligned on 4, so deposit the W bit setting into
116	 * the bottom of sp temporarily */
117	or,ev	%r1,%r30,%r30
118	b,n	1f
119	/* The top halves of argument registers must be cleared on syscall
120	 * entry from narrow executable.
121	 */
122	depdi	0, 31, 32, %r26
123	depdi	0, 31, 32, %r25
124	depdi	0, 31, 32, %r24
125	depdi	0, 31, 32, %r23
126	depdi	0, 31, 32, %r22
127	depdi	0, 31, 32, %r21
1281:
129#endif
130
131	/* We use a rsm/ssm pair to prevent sr3 from being clobbered
132	 * by external interrupts.
133	 */
134	mfsp    %sr7,%r1                        /* save user sr7 */
135	rsm	PSW_SM_I, %r0			/* disable interrupts */
136	mtsp    %r1,%sr3                        /* and store it in sr3 */
137
138	mfctl   %cr30,%r1
139	xor     %r1,%r30,%r30                   /* ye olde xor trick */
140	xor     %r1,%r30,%r1
141	xor     %r1,%r30,%r30
142
143	LDREG	TASK_STACK(%r30),%r30		/* set up kernel stack */
144	ldo	FRAME_SIZE(%r30),%r30
145	/* N.B.: It is critical that we don't set sr7 to 0 until r30
146	 *       contains a valid kernel stack pointer. It is also
147	 *       critical that we don't start using the kernel stack
148	 *       until after sr7 has been set to 0.
149	 */
150
151	mtsp	%r0,%sr7			/* get kernel space into sr7 */
152	ssm	PSW_SM_I, %r0			/* enable interrupts */
153	STREGM	%r1,FRAME_SIZE(%r30)		/* save r1 (usp) here for now */
154	mfctl	%cr30,%r1			/* get task ptr in %r1 */
155
156	/* Save some registers for sigcontext and potential task
157	   switch (see entry.S for the details of which ones are
158	   saved/restored).  TASK_PT_PSW is zeroed so we can see whether
159	   a process is on a syscall or not.  For an interrupt the real
160	   PSW value is stored.  This is needed for gdb and sys_ptrace. */
161	STREG	%r0,  TASK_PT_PSW(%r1)
162	STREG	%r2,  TASK_PT_GR2(%r1)		/* preserve rp */
163	STREG	%r19, TASK_PT_GR19(%r1)
164
165	LDREGM	-FRAME_SIZE(%r30), %r2		/* get users sp back */
166#ifdef CONFIG_64BIT
167	extrd,u	%r2,63,1,%r19			/* W hidden in bottom bit */
168#if 0
169	xor	%r19,%r2,%r2			/* clear bottom bit */
170	depd,z	%r19,1,1,%r19
171	std	%r19,TASK_PT_PSW(%r1)
172#endif
173#endif
174	STREG	%r2,  TASK_PT_GR30(%r1)		/* ... and save it */
175
176	STREG	%r20, TASK_PT_GR20(%r1)		/* Syscall number */
177	STREG	%r21, TASK_PT_GR21(%r1)
178	STREG	%r22, TASK_PT_GR22(%r1)
179	STREG	%r23, TASK_PT_GR23(%r1)		/* 4th argument */
180	STREG	%r24, TASK_PT_GR24(%r1)		/* 3rd argument */
181	STREG	%r25, TASK_PT_GR25(%r1)		/* 2nd argument */
182	STREG	%r26, TASK_PT_GR26(%r1)	 	/* 1st argument */
183	STREG	%r27, TASK_PT_GR27(%r1)		/* user dp */
184	STREG   %r28, TASK_PT_GR28(%r1)         /* return value 0 */
185	STREG   %r0, TASK_PT_ORIG_R28(%r1)      /* don't prohibit restarts */
186	STREG	%r29, TASK_PT_GR29(%r1)		/* return value 1 */
187	STREG	%r31, TASK_PT_GR31(%r1)		/* preserve syscall return ptr */
188
189	ldo	TASK_PT_FR0(%r1), %r27		/* save fpregs from the kernel */
190	save_fp	%r27				/* or potential task switch  */
191
192	mfctl	%cr11, %r27			/* i.e. SAR */
193	STREG	%r27, TASK_PT_SAR(%r1)
194
195	loadgp
196
197#ifdef CONFIG_64BIT
198	ldo	-16(%r30),%r29			/* Reference param save area */
199	copy	%r19,%r2			/* W bit back to r2 */
200#else
201	/* no need to save these on stack in wide mode because the first 8
202	 * args are passed in registers */
203	stw     %r22, -52(%r30)                 /* 5th argument */
204	stw     %r21, -56(%r30)                 /* 6th argument */
205#endif
206
207	/* Are we being ptraced? */
208	mfctl	%cr30, %r1
209	LDREG	TASK_TI_FLAGS(%r1),%r1
210	ldi	_TIF_SYSCALL_TRACE_MASK, %r19
211	and,COND(=) %r1, %r19, %r0
212	b,n	.Ltracesys
213
214	/* Note!  We cannot use the syscall table that is mapped
215	nearby since the gateway page is mapped execute-only. */
216
217#ifdef CONFIG_64BIT
218	ldil	L%sys_call_table, %r1
219	or,=	%r2,%r2,%r2
220	addil	L%(sys_call_table64-sys_call_table), %r1
221	ldo	R%sys_call_table(%r1), %r19
222	or,=	%r2,%r2,%r2
223	ldo	R%sys_call_table64(%r1), %r19
224#else
225	load32	sys_call_table, %r19
226#endif
227	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
228	b,n	.Lsyscall_nosys
229
230	LDREGX  %r20(%r19), %r19
231
232	/* If this is a sys_rt_sigreturn call, and the signal was received
233	 * when not in_syscall, then we want to return via syscall_exit_rfi,
234	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
235	 * trampoline code in signal.c).
236	 */
237	ldi	__NR_rt_sigreturn,%r2
238	comb,=	%r2,%r20,.Lrt_sigreturn
239.Lin_syscall:
240	ldil	L%syscall_exit,%r2
241	be      0(%sr7,%r19)
242	ldo	R%syscall_exit(%r2),%r2
243.Lrt_sigreturn:
244	comib,<> 0,%r25,.Lin_syscall
245	ldil	L%syscall_exit_rfi,%r2
246	be      0(%sr7,%r19)
247	ldo	R%syscall_exit_rfi(%r2),%r2
248
249	/* Note!  Because we are not running where we were linked, any
250	calls to functions external to this file must be indirect.  To
251	be safe, we apply the opposite rule to functions within this
252	file, with local labels given to them to ensure correctness. */
253
254.Lsyscall_nosys:
255syscall_nosys:
256	ldil	L%syscall_exit,%r1
257	be	R%syscall_exit(%sr7,%r1)
258	ldo	-ENOSYS(%r0),%r28		   /* set errno */
259
260
261/* Warning! This trace code is a virtual duplicate of the code above so be
262 * sure to maintain both! */
263.Ltracesys:
264tracesys:
265	/* Need to save more registers so the debugger can see where we
266	 * are.  This saves only the lower 8 bits of PSW, so that the C
267	 * bit is still clear on syscalls, and the D bit is set if this
268	 * full register save path has been executed.  We check the D
269	 * bit on syscall_return_rfi to determine which registers to
270	 * restore.  An interrupt results in a full PSW saved with the
271	 * C bit set, a non-straced syscall entry results in C and D clear
272	 * in the saved PSW.
273	 */
274	mfctl	%cr30,%r1			/* get task ptr */
275	ssm	0,%r2
276	STREG	%r2,TASK_PT_PSW(%r1)		/* Lower 8 bits only!! */
277	mfsp	%sr0,%r2
278	STREG	%r2,TASK_PT_SR0(%r1)
279	mfsp	%sr1,%r2
280	STREG	%r2,TASK_PT_SR1(%r1)
281	mfsp	%sr2,%r2
282	STREG	%r2,TASK_PT_SR2(%r1)
283	mfsp	%sr3,%r2
284	STREG	%r2,TASK_PT_SR3(%r1)
285	STREG	%r2,TASK_PT_SR4(%r1)
286	STREG	%r2,TASK_PT_SR5(%r1)
287	STREG	%r2,TASK_PT_SR6(%r1)
288	STREG	%r2,TASK_PT_SR7(%r1)
289	STREG	%r2,TASK_PT_IASQ0(%r1)
290	STREG	%r2,TASK_PT_IASQ1(%r1)
291	LDREG	TASK_PT_GR31(%r1),%r2
292	STREG	%r2,TASK_PT_IAOQ0(%r1)
293	ldo	4(%r2),%r2
294	STREG	%r2,TASK_PT_IAOQ1(%r1)
295	ldo	TASK_REGS(%r1),%r2
296	/* reg_save %r2 */
297	STREG	%r3,PT_GR3(%r2)
298	STREG	%r4,PT_GR4(%r2)
299	STREG	%r5,PT_GR5(%r2)
300	STREG	%r6,PT_GR6(%r2)
301	STREG	%r7,PT_GR7(%r2)
302	STREG	%r8,PT_GR8(%r2)
303	STREG	%r9,PT_GR9(%r2)
304	STREG	%r10,PT_GR10(%r2)
305	STREG	%r11,PT_GR11(%r2)
306	STREG	%r12,PT_GR12(%r2)
307	STREG	%r13,PT_GR13(%r2)
308	STREG	%r14,PT_GR14(%r2)
309	STREG	%r15,PT_GR15(%r2)
310	STREG	%r16,PT_GR16(%r2)
311	STREG	%r17,PT_GR17(%r2)
312	STREG	%r18,PT_GR18(%r2)
313	/* Finished saving things for the debugger */
314
315	copy	%r2,%r26
316	ldil	L%do_syscall_trace_enter,%r1
317	ldil	L%tracesys_next,%r2
318	be	R%do_syscall_trace_enter(%sr7,%r1)
319	ldo	R%tracesys_next(%r2),%r2
320
321tracesys_next:
322	/* do_syscall_trace_enter either returned the syscallno, or -1L,
323	 *  so we skip restoring the PT_GR20 below, since we pulled it from
324	 *  task->thread.regs.gr[20] above.
325	 */
326	copy	%ret0,%r20
327
328	mfctl	%cr30,%r1			/* get task ptr */
329	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return value */
330	LDREG   TASK_PT_GR26(%r1), %r26		/* Restore the users args */
331	LDREG   TASK_PT_GR25(%r1), %r25
332	LDREG   TASK_PT_GR24(%r1), %r24
333	LDREG   TASK_PT_GR23(%r1), %r23
334	LDREG   TASK_PT_GR22(%r1), %r22
335	LDREG   TASK_PT_GR21(%r1), %r21
336#ifdef CONFIG_64BIT
337	ldo	-16(%r30),%r29			/* Reference param save area */
338#else
339	stw     %r22, -52(%r30)                 /* 5th argument */
340	stw     %r21, -56(%r30)                 /* 6th argument */
341#endif
342
343	cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */
344	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
345	b,n	.Ltracesys_nosys
346
347	/* Note!  We cannot use the syscall table that is mapped
348	nearby since the gateway page is mapped execute-only. */
349
350#ifdef CONFIG_64BIT
351	LDREG	TASK_PT_GR30(%r1), %r19		/* get users sp back */
352	extrd,u	%r19,63,1,%r2			/* W hidden in bottom bit */
353
354	ldil	L%sys_call_table, %r1
355	or,=	%r2,%r2,%r2
356	addil	L%(sys_call_table64-sys_call_table), %r1
357	ldo	R%sys_call_table(%r1), %r19
358	or,=	%r2,%r2,%r2
359	ldo	R%sys_call_table64(%r1), %r19
360#else
361	load32	sys_call_table, %r19
362#endif
363
364	LDREGX  %r20(%r19), %r19
365
366	/* If this is a sys_rt_sigreturn call, and the signal was received
367	 * when not in_syscall, then we want to return via syscall_exit_rfi,
368	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
369	 * trampoline code in signal.c).
370	 */
371	ldi	__NR_rt_sigreturn,%r2
372	comb,=	%r2,%r20,.Ltrace_rt_sigreturn
373.Ltrace_in_syscall:
374	ldil	L%tracesys_exit,%r2
375	be      0(%sr7,%r19)
376	ldo	R%tracesys_exit(%r2),%r2
377
378.Ltracesys_nosys:
379	ldo	-ENOSYS(%r0),%r28		/* set errno */
380
381	/* Do *not* call this function on the gateway page, because it
382	makes a direct call to syscall_trace. */
383
384tracesys_exit:
385	mfctl	%cr30,%r1			/* get task ptr */
386#ifdef CONFIG_64BIT
387	ldo	-16(%r30),%r29			/* Reference param save area */
388#endif
389	ldo	TASK_REGS(%r1),%r26
390	BL	do_syscall_trace_exit,%r2
391	STREG   %r28,TASK_PT_GR28(%r1)          /* save return value now */
392	mfctl	%cr30,%r1			/* get task ptr */
393	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return val. */
394
395	ldil	L%syscall_exit,%r1
396	be,n	R%syscall_exit(%sr7,%r1)
397
398.Ltrace_rt_sigreturn:
399	comib,<> 0,%r25,.Ltrace_in_syscall
400	ldil	L%tracesys_sigexit,%r2
401	be      0(%sr7,%r19)
402	ldo	R%tracesys_sigexit(%r2),%r2
403
404tracesys_sigexit:
405	mfctl	%cr30,%r1			/* get task ptr */
406#ifdef CONFIG_64BIT
407	ldo	-16(%r30),%r29			/* Reference param save area */
408#endif
409	BL	do_syscall_trace_exit,%r2
410	ldo	TASK_REGS(%r1),%r26
411
412	ldil	L%syscall_exit_rfi,%r1
413	be,n	R%syscall_exit_rfi(%sr7,%r1)
414
415
416	/*********************************************************
417		32/64-bit Light-Weight-Syscall ABI
418
419		* - Indicates a hint for userspace inline asm
420		implementations.
421
422		Syscall number (caller-saves)
423	        - %r20
424	        * In asm clobber.
425
426		Argument registers (caller-saves)
427	        - %r26, %r25, %r24, %r23, %r22
428	        * In asm input.
429
430		Return registers (caller-saves)
431	        - %r28 (return), %r21 (errno)
432	        * In asm output.
433
434		Caller-saves registers
435	        - %r1, %r27, %r29
436	        - %r2 (return pointer)
437	        - %r31 (ble link register)
438	        * In asm clobber.
439
440		Callee-saves registers
441	        - %r3-%r18
442	        - %r30 (stack pointer)
443	        * Not in asm clobber.
444
445		If userspace is 32-bit:
446		Callee-saves registers
447	        - %r19 (32-bit PIC register)
448
449		Differences from 32-bit calling convention:
450		- Syscall number in %r20
451		- Additional argument register %r22 (arg4)
452		- Callee-saves %r19.
453
454		If userspace is 64-bit:
455		Callee-saves registers
456		- %r27 (64-bit PIC register)
457
458		Differences from 64-bit calling convention:
459		- Syscall number in %r20
460		- Additional argument register %r22 (arg4)
461		- Callee-saves %r27.
462
463		Error codes returned by entry path:
464
465		ENOSYS - r20 was an invalid LWS number.
466
467	*********************************************************/
468lws_start:
469
470#ifdef CONFIG_64BIT
471	ssm	PSW_SM_W, %r1
472	extrd,u	%r1,PSW_W_BIT,1,%r1
473	/* sp must be aligned on 4, so deposit the W bit setting into
474	 * the bottom of sp temporarily */
475	or,ev	%r1,%r30,%r30
476
477	/* Clip LWS number to a 32-bit value for 32-bit processes */
478	depdi	0, 31, 32, %r20
479#endif
480
481        /* Is the lws entry number valid? */
482	comiclr,>>	__NR_lws_entries, %r20, %r0
483	b,n	lws_exit_nosys
484
485	/* Load table start */
486	ldil	L%lws_table, %r1
487	ldo	R%lws_table(%r1), %r28	/* Scratch use of r28 */
488	LDREGX	%r20(%sr2,r28), %r21	/* Scratch use of r21 */
489
490	/* Jump to lws, lws table pointers already relocated */
491	be,n	0(%sr2,%r21)
492
493lws_exit_nosys:
494	ldo	-ENOSYS(%r0),%r21		   /* set errno */
495	/* Fall through: Return to userspace */
496
497lws_exit:
498#ifdef CONFIG_64BIT
499	/* decide whether to reset the wide mode bit
500	 *
501	 * For a syscall, the W bit is stored in the lowest bit
502	 * of sp.  Extract it and reset W if it is zero */
503	extrd,u,*<>	%r30,63,1,%r1
504	rsm	PSW_SM_W, %r0
505	/* now reset the lowest bit of sp if it was set */
506	xor	%r30,%r1,%r30
507#endif
508	be,n	0(%sr7, %r31)
509
510
511
512	/***************************************************
513		Implementing 32bit CAS as an atomic operation:
514
515		%r26 - Address to examine
516		%r25 - Old value to check (old)
517		%r24 - New value to set (new)
518		%r28 - Return prev through this register.
519		%r21 - Kernel error code
520
521		If debugging is DISabled:
522
523		%r21 has the following meanings:
524
525		EAGAIN - CAS is busy, ldcw failed, try again.
526		EFAULT - Read or write failed.
527
528		If debugging is enabled:
529
530		EDEADLOCK - CAS called recursively.
531		EAGAIN && r28 == 1 - CAS is busy. Lock contended.
532		EAGAIN && r28 == 2 - CAS is busy. ldcw failed.
533		EFAULT - Read or write failed.
534
535		Scratch: r20, r28, r1
536
537	****************************************************/
538
539	/* Do not enable LWS debugging */
540#define ENABLE_LWS_DEBUG 0
541
542	/* ELF64 Process entry path */
543lws_compare_and_swap64:
544#ifdef CONFIG_64BIT
545	b,n	lws_compare_and_swap
546#else
547	/* If we are not a 64-bit kernel, then we don't
548	 * have 64-bit input registers, and calling
549	 * the 64-bit LWS CAS returns ENOSYS.
550	 */
551	b,n	lws_exit_nosys
552#endif
553
554	/* ELF32 Process entry path */
555lws_compare_and_swap32:
556#ifdef CONFIG_64BIT
557	/* Clip all the input registers */
558	depdi	0, 31, 32, %r26
559	depdi	0, 31, 32, %r25
560	depdi	0, 31, 32, %r24
561#endif
562
563lws_compare_and_swap:
564	/* Load start of lock table */
565	ldil	L%lws_lock_start, %r20
566	ldo	R%lws_lock_start(%r20), %r28
567
568	/* Extract eight bits from r26 and hash lock (Bits 3-11) */
569	extru  %r26, 28, 8, %r20
570
571	/* Find lock to use, the hash is either one of 0 to
572	   15, multiplied by 16 (keep it 16-byte aligned)
573	   and add to the lock table offset. */
574	shlw	%r20, 4, %r20
575	add	%r20, %r28, %r20
576
577# if ENABLE_LWS_DEBUG
578	/*
579		DEBUG, check for deadlock!
580		If the thread register values are the same
581		then we were the one that locked it last and
582		this is a recurisve call that will deadlock.
583		We *must* giveup this call and fail.
584	*/
585	ldw	4(%sr2,%r20), %r28			/* Load thread register */
586	/* WARNING: If cr27 cycles to the same value we have problems */
587	mfctl	%cr27, %r21				/* Get current thread register */
588	cmpb,<>,n	%r21, %r28, cas_lock		/* Called recursive? */
589	b	lws_exit				/* Return error! */
590	ldo	-EDEADLOCK(%r0), %r21
591cas_lock:
592	cmpb,=,n	%r0, %r28, cas_nocontend	/* Is nobody using it? */
593	ldo	1(%r0), %r28				/* 1st case */
594	b	lws_exit				/* Contended... */
595	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
596cas_nocontend:
597# endif
598/* ENABLE_LWS_DEBUG */
599
600	/* COW breaks can cause contention on UP systems */
601	LDCW	0(%sr2,%r20), %r28			/* Try to acquire the lock */
602	cmpb,<>,n	%r0, %r28, cas_action		/* Did we get it? */
603cas_wouldblock:
604	ldo	2(%r0), %r28				/* 2nd case */
605	b	lws_exit				/* Contended... */
606	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
607
608	/*
609		prev = *addr;
610		if ( prev == old )
611		  *addr = new;
612		return prev;
613	*/
614
615	/* NOTES:
616		This all works becuse intr_do_signal
617		and schedule both check the return iasq
618		and see that we are on the kernel page
619		so this process is never scheduled off
620		or is ever sent any signal of any sort,
621		thus it is wholly atomic from usrspaces
622		perspective
623	*/
624cas_action:
625#if defined CONFIG_SMP && ENABLE_LWS_DEBUG
626	/* DEBUG */
627	mfctl	%cr27, %r1
628	stw	%r1, 4(%sr2,%r20)
629#endif
630	/* The load and store could fail */
6311:	ldw	0(%r26), %r28
632	sub,<>	%r28, %r25, %r0
6332:	stw	%r24, 0(%r26)
634	/* Free lock */
635	stw,ma	%r20, 0(%sr2,%r20)
636#if ENABLE_LWS_DEBUG
637	/* Clear thread register indicator */
638	stw	%r0, 4(%sr2,%r20)
639#endif
640	/* Return to userspace, set no error */
641	b	lws_exit
642	copy	%r0, %r21
643
6443:
645	/* Error occurred on load or store */
646	/* Free lock */
647	stw,ma	%r20, 0(%sr2,%r20)
648#if ENABLE_LWS_DEBUG
649	stw	%r0, 4(%sr2,%r20)
650#endif
651	b	lws_exit
652	ldo	-EFAULT(%r0),%r21	/* set errno */
653	nop
654	nop
655	nop
656	nop
657
658	/* Two exception table entries, one for the load,
659	   the other for the store. Either return -EFAULT.
660	   Each of the entries must be relocated. */
661	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page)
662	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page)
663
664
665	/***************************************************
666		New CAS implementation which uses pointers and variable size
667		information. The value pointed by old and new MUST NOT change
668		while performing CAS. The lock only protect the value at %r26.
669
670		%r26 - Address to examine
671		%r25 - Pointer to the value to check (old)
672		%r24 - Pointer to the value to set (new)
673		%r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit)
674		%r28 - Return non-zero on failure
675		%r21 - Kernel error code
676
677		%r21 has the following meanings:
678
679		EAGAIN - CAS is busy, ldcw failed, try again.
680		EFAULT - Read or write failed.
681
682		Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only)
683
684	****************************************************/
685
686	/* ELF32 Process entry path */
687lws_compare_and_swap_2:
688#ifdef CONFIG_64BIT
689	/* Clip the input registers. We don't need to clip %r23 as we
690	   only use it for word operations */
691	depdi	0, 31, 32, %r26
692	depdi	0, 31, 32, %r25
693	depdi	0, 31, 32, %r24
694#endif
695
696	/* Check the validity of the size pointer */
697	subi,>>= 3, %r23, %r0
698	b,n	lws_exit_nosys
699
700	/* Jump to the functions which will load the old and new values into
701	   registers depending on the their size */
702	shlw	%r23, 2, %r29
703	blr	%r29, %r0
704	nop
705
706	/* 8bit load */
7074:	ldb	0(%r25), %r25
708	b	cas2_lock_start
7095:	ldb	0(%r24), %r24
710	nop
711	nop
712	nop
713	nop
714	nop
715
716	/* 16bit load */
7176:	ldh	0(%r25), %r25
718	b	cas2_lock_start
7197:	ldh	0(%r24), %r24
720	nop
721	nop
722	nop
723	nop
724	nop
725
726	/* 32bit load */
7278:	ldw	0(%r25), %r25
728	b	cas2_lock_start
7299:	ldw	0(%r24), %r24
730	nop
731	nop
732	nop
733	nop
734	nop
735
736	/* 64bit load */
737#ifdef CONFIG_64BIT
73810:	ldd	0(%r25), %r25
73911:	ldd	0(%r24), %r24
740#else
741	/* Load old value into r22/r23 - high/low */
74210:	ldw	0(%r25), %r22
74311:	ldw	4(%r25), %r23
744	/* Load new value into fr4 for atomic store later */
74512:	flddx	0(%r24), %fr4
746#endif
747
748cas2_lock_start:
749	/* Load start of lock table */
750	ldil	L%lws_lock_start, %r20
751	ldo	R%lws_lock_start(%r20), %r28
752
753	/* Extract eight bits from r26 and hash lock (Bits 3-11) */
754	extru  %r26, 28, 8, %r20
755
756	/* Find lock to use, the hash is either one of 0 to
757	   15, multiplied by 16 (keep it 16-byte aligned)
758	   and add to the lock table offset. */
759	shlw	%r20, 4, %r20
760	add	%r20, %r28, %r20
761
762	/* COW breaks can cause contention on UP systems */
763	LDCW	0(%sr2,%r20), %r28		/* Try to acquire the lock */
764	cmpb,<>,n	%r0, %r28, cas2_action	/* Did we get it? */
765cas2_wouldblock:
766	ldo	2(%r0), %r28			/* 2nd case */
767	b	lws_exit			/* Contended... */
768	ldo	-EAGAIN(%r0), %r21		/* Spin in userspace */
769
770	/*
771		prev = *addr;
772		if ( prev == old )
773		  *addr = new;
774		return prev;
775	*/
776
777	/* NOTES:
778		This all works becuse intr_do_signal
779		and schedule both check the return iasq
780		and see that we are on the kernel page
781		so this process is never scheduled off
782		or is ever sent any signal of any sort,
783		thus it is wholly atomic from usrspaces
784		perspective
785	*/
786cas2_action:
787	/* Jump to the correct function */
788	blr	%r29, %r0
789	/* Set %r28 as non-zero for now */
790	ldo	1(%r0),%r28
791
792	/* 8bit CAS */
79313:	ldb	0(%r26), %r29
794	sub,=	%r29, %r25, %r0
795	b,n	cas2_end
79614:	stb	%r24, 0(%r26)
797	b	cas2_end
798	copy	%r0, %r28
799	nop
800	nop
801
802	/* 16bit CAS */
80315:	ldh	0(%r26), %r29
804	sub,=	%r29, %r25, %r0
805	b,n	cas2_end
80616:	sth	%r24, 0(%r26)
807	b	cas2_end
808	copy	%r0, %r28
809	nop
810	nop
811
812	/* 32bit CAS */
81317:	ldw	0(%r26), %r29
814	sub,=	%r29, %r25, %r0
815	b,n	cas2_end
81618:	stw	%r24, 0(%r26)
817	b	cas2_end
818	copy	%r0, %r28
819	nop
820	nop
821
822	/* 64bit CAS */
823#ifdef CONFIG_64BIT
82419:	ldd	0(%r26), %r29
825	sub,*=	%r29, %r25, %r0
826	b,n	cas2_end
82720:	std	%r24, 0(%r26)
828	copy	%r0, %r28
829#else
830	/* Compare first word */
83119:	ldw	0(%r26), %r29
832	sub,=	%r29, %r22, %r0
833	b,n	cas2_end
834	/* Compare second word */
83520:	ldw	4(%r26), %r29
836	sub,=	%r29, %r23, %r0
837	b,n	cas2_end
838	/* Perform the store */
83921:	fstdx	%fr4, 0(%r26)
840	copy	%r0, %r28
841#endif
842
843cas2_end:
844	/* Free lock */
845	stw,ma	%r20, 0(%sr2,%r20)
846	/* Return to userspace, set no error */
847	b	lws_exit
848	copy	%r0, %r21
849
85022:
851	/* Error occurred on load or store */
852	/* Free lock */
853	stw,ma	%r20, 0(%sr2,%r20)
854	ldo	1(%r0),%r28
855	b	lws_exit
856	ldo	-EFAULT(%r0),%r21	/* set errno */
857	nop
858	nop
859	nop
860
861	/* Exception table entries, for the load and store, return EFAULT.
862	   Each of the entries must be relocated. */
863	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page)
864	ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page)
865	ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page)
866	ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page)
867	ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page)
868	ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page)
869	ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page)
870	ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page)
871	ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page)
872	ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page)
873	ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page)
874	ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page)
875	ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page)
876	ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page)
877	ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page)
878	ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page)
879#ifndef CONFIG_64BIT
880	ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page)
881	ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
882#endif
883
884	/* Make sure nothing else is placed on this page */
885	.align PAGE_SIZE
886END(linux_gateway_page)
887ENTRY(end_linux_gateway_page)
888
889	/* Relocate symbols assuming linux_gateway_page is mapped
890	   to virtual address 0x0 */
891
892#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page)
893
894	.section .rodata,"a"
895
896	.align 8
897	/* Light-weight-syscall table */
898	/* Start of lws table. */
899ENTRY(lws_table)
900	LWS_ENTRY(compare_and_swap32)		/* 0 - ELF32 Atomic 32bit CAS */
901	LWS_ENTRY(compare_and_swap64)		/* 1 - ELF64 Atomic 32bit CAS */
902	LWS_ENTRY(compare_and_swap_2)		/* 2 - ELF32 Atomic 64bit CAS */
903END(lws_table)
904	/* End of lws table */
905
906#ifdef CONFIG_64BIT
907#define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, compat)
908#else
909#define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, native)
910#endif
911#define __SYSCALL(nr, entry)	ASM_ULONG_INSN entry
912	.align 8
913ENTRY(sys_call_table)
914	.export sys_call_table,data
915#include <asm/syscall_table_32.h>    /* 32-bit syscalls */
916END(sys_call_table)
917
918#ifdef CONFIG_64BIT
919	.align 8
920ENTRY(sys_call_table64)
921#include <asm/syscall_table_64.h>    /* 64-bit syscalls */
922END(sys_call_table64)
923#endif
924
925	/*
926		All light-weight-syscall atomic operations
927		will use this set of locks
928
929		NOTE: The lws_lock_start symbol must be
930		at least 16-byte aligned for safe use
931		with ldcw.
932	*/
933	.section .data
934	.align	L1_CACHE_BYTES
935ENTRY(lws_lock_start)
936	/* lws locks */
937	.rept 256
938	/* Keep locks aligned at 16-bytes */
939	.word 1
940	.word 0
941	.word 0
942	.word 0
943	.endr
944END(lws_lock_start)
945	.previous
946
947.end
948