xref: /openbmc/linux/arch/parisc/kernel/syscall.S (revision 827beb77) 
1/*
2 * Linux/PA-RISC Project (http://www.parisc-linux.org/)
3 *
4 * System call entry code / Linux gateway page
5 * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org>
6 * Licensed under the GNU GPL.
7 * thanks to Philipp Rumpf, Mike Shaver and various others
8 * sorry about the wall, puffin..
9 */
10
11/*
12How does the Linux gateway page on PA-RISC work?
13------------------------------------------------
14The Linux gateway page on PA-RISC is "special".
15It actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc
16terminology it's Execute, promote to PL0) in the page map.  So anything
17executing on this page executes with kernel level privilege (there's more to it
18than that: to have this happen, you also have to use a branch with a ,gate
19completer to activate the privilege promotion).  The upshot is that everything
20that runs on the gateway page runs at kernel privilege but with the current
21user process address space (although you have access to kernel space via %sr2).
22For the 0x100 syscall entry, we redo the space registers to point to the kernel
23address space (preserving the user address space in %sr3), move to wide mode if
24required, save the user registers and branch into the kernel syscall entry
25point.  For all the other functions, we execute at kernel privilege but don't
26flip address spaces. The basic upshot of this is that these code snippets are
27executed atomically (because the kernel can't be pre-empted) and they may
28perform architecturally forbidden (to PL3) operations (like setting control
29registers).
30*/
31
32
33#include <asm/asm-offsets.h>
34#include <asm/unistd.h>
35#include <asm/errno.h>
36#include <asm/page.h>
37#include <asm/psw.h>
38#include <asm/thread_info.h>
39#include <asm/assembly.h>
40#include <asm/processor.h>
41#include <asm/cache.h>
42
43#include <linux/linkage.h>
44
45	/* We fill the empty parts of the gateway page with
46 	 * something that will kill the kernel or a
47 	 * userspace application.
48	 */
49#define KILL_INSN	break	0,0
50
51	.level          PA_ASM_LEVEL
52
53	.text
54
55	.import syscall_exit,code
56	.import syscall_exit_rfi,code
57
58	/* Linux gateway page is aliased to virtual page 0 in the kernel
59	 * address space. Since it is a gateway page it cannot be
60	 * dereferenced, so null pointers will still fault. We start
61	 * the actual entry point at 0x100. We put break instructions
62	 * at the beginning of the page to trap null indirect function
63	 * pointers.
64	 */
65
66	.align PAGE_SIZE
67ENTRY(linux_gateway_page)
68
69        /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */
70	.rept 44
71	KILL_INSN
72	.endr
73
74	/* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */
75	/* Light-weight-syscall entry must always be located at 0xb0 */
76	/* WARNING: Keep this number updated with table size changes */
77#define __NR_lws_entries (3)
78
79lws_entry:
80	gate	lws_start, %r0		/* increase privilege */
81	depi	PRIV_USER, 31, 2, %r31	/* Ensure we return into user mode. */
82
83	/* Fill from 0xb8 to 0xe0 */
84	.rept 10
85	KILL_INSN
86	.endr
87
88	/* This function MUST be located at 0xe0 for glibc's threading
89	mechanism to work. DO NOT MOVE THIS CODE EVER! */
90set_thread_pointer:
91	gate	.+8, %r0		/* increase privilege */
92	depi	PRIV_USER, 31, 2, %r31	/* Ensure we return into user mode. */
93	be	0(%sr7,%r31)		/* return to user space */
94	mtctl	%r26, %cr27		/* move arg0 to the control register */
95
96	/* Increase the chance of trapping if random jumps occur to this
97	address, fill from 0xf0 to 0x100 */
98	.rept 4
99	KILL_INSN
100	.endr
101
102/* This address must remain fixed at 0x100 for glibc's syscalls to work */
103	.align LINUX_GATEWAY_ADDR
104linux_gateway_entry:
105	gate	.+8, %r0			/* become privileged */
106	mtsp	%r0,%sr4			/* get kernel space into sr4 */
107	mtsp	%r0,%sr5			/* get kernel space into sr5 */
108	mtsp	%r0,%sr6			/* get kernel space into sr6 */
109
110#ifdef CONFIG_64BIT
111	/* Store W bit on entry to the syscall in case it's a wide userland
112	 * process. */
113	ssm	PSW_SM_W, %r1
114	extrd,u	%r1,PSW_W_BIT,1,%r1
115	/* sp must be aligned on 4, so deposit the W bit setting into
116	 * the bottom of sp temporarily */
117	or,ev	%r1,%r30,%r30
118	b,n	1f
119	/* The top halves of argument registers must be cleared on syscall
120	 * entry from narrow executable.
121	 */
122	depdi	0, 31, 32, %r26
123	depdi	0, 31, 32, %r25
124	depdi	0, 31, 32, %r24
125	depdi	0, 31, 32, %r23
126	depdi	0, 31, 32, %r22
127	depdi	0, 31, 32, %r21
1281:
129#endif
130
131	/* We use a rsm/ssm pair to prevent sr3 from being clobbered
132	 * by external interrupts.
133	 */
134	mfsp    %sr7,%r1                        /* save user sr7 */
135	rsm	PSW_SM_I, %r0			/* disable interrupts */
136	mtsp    %r1,%sr3                        /* and store it in sr3 */
137
138	mfctl   %cr30,%r1
139	xor     %r1,%r30,%r30                   /* ye olde xor trick */
140	xor     %r1,%r30,%r1
141	xor     %r1,%r30,%r30
142
143	LDREG	TASK_STACK(%r30),%r30		/* set up kernel stack */
144	ldo	FRAME_SIZE(%r30),%r30
145	/* N.B.: It is critical that we don't set sr7 to 0 until r30
146	 *       contains a valid kernel stack pointer. It is also
147	 *       critical that we don't start using the kernel stack
148	 *       until after sr7 has been set to 0.
149	 */
150
151	mtsp	%r0,%sr7			/* get kernel space into sr7 */
152	ssm	PSW_SM_I, %r0			/* enable interrupts */
153	STREGM	%r1,FRAME_SIZE(%r30)		/* save r1 (usp) here for now */
154	mfctl	%cr30,%r1			/* get task ptr in %r1 */
155
156	/* Save some registers for sigcontext and potential task
157	   switch (see entry.S for the details of which ones are
158	   saved/restored).  TASK_PT_PSW is zeroed so we can see whether
159	   a process is on a syscall or not.  For an interrupt the real
160	   PSW value is stored.  This is needed for gdb and sys_ptrace. */
161	STREG	%r0,  TASK_PT_PSW(%r1)
162	STREG	%r2,  TASK_PT_GR2(%r1)		/* preserve rp */
163	STREG	%r19, TASK_PT_GR19(%r1)
164
165	LDREGM	-FRAME_SIZE(%r30), %r2		/* get users sp back */
166#ifdef CONFIG_64BIT
167	extrd,u	%r2,63,1,%r19			/* W hidden in bottom bit */
168#if 0
169	xor	%r19,%r2,%r2			/* clear bottom bit */
170	depd,z	%r19,1,1,%r19
171	std	%r19,TASK_PT_PSW(%r1)
172#endif
173#endif
174	STREG	%r2,  TASK_PT_GR30(%r1)		/* ... and save it */
175
176	STREG	%r20, TASK_PT_GR20(%r1)		/* Syscall number */
177	STREG	%r21, TASK_PT_GR21(%r1)
178	STREG	%r22, TASK_PT_GR22(%r1)
179	STREG	%r23, TASK_PT_GR23(%r1)		/* 4th argument */
180	STREG	%r24, TASK_PT_GR24(%r1)		/* 3rd argument */
181	STREG	%r25, TASK_PT_GR25(%r1)		/* 2nd argument */
182	STREG	%r26, TASK_PT_GR26(%r1)	 	/* 1st argument */
183	STREG	%r27, TASK_PT_GR27(%r1)		/* user dp */
184	STREG   %r28, TASK_PT_GR28(%r1)         /* return value 0 */
185	STREG   %r0, TASK_PT_ORIG_R28(%r1)      /* don't prohibit restarts */
186	STREG	%r29, TASK_PT_GR29(%r1)		/* return value 1 */
187	STREG	%r31, TASK_PT_GR31(%r1)		/* preserve syscall return ptr */
188
189	ldo	TASK_PT_FR0(%r1), %r27		/* save fpregs from the kernel */
190	save_fp	%r27				/* or potential task switch  */
191
192	mfctl	%cr11, %r27			/* i.e. SAR */
193	STREG	%r27, TASK_PT_SAR(%r1)
194
195	loadgp
196
197#ifdef CONFIG_64BIT
198	ldo	-16(%r30),%r29			/* Reference param save area */
199	copy	%r19,%r2			/* W bit back to r2 */
200#else
201	/* no need to save these on stack in wide mode because the first 8
202	 * args are passed in registers */
203	stw     %r22, -52(%r30)                 /* 5th argument */
204	stw     %r21, -56(%r30)                 /* 6th argument */
205#endif
206
207	/* Are we being ptraced? */
208	mfctl	%cr30, %r1
209	LDREG	TASK_TI_FLAGS(%r1),%r1
210	ldi	_TIF_SYSCALL_TRACE_MASK, %r19
211	and,COND(=) %r1, %r19, %r0
212	b,n	.Ltracesys
213
214	/* Note!  We cannot use the syscall table that is mapped
215	nearby since the gateway page is mapped execute-only. */
216
217#ifdef CONFIG_64BIT
218	ldil	L%sys_call_table, %r1
219	or,=	%r2,%r2,%r2
220	addil	L%(sys_call_table64-sys_call_table), %r1
221	ldo	R%sys_call_table(%r1), %r19
222	or,=	%r2,%r2,%r2
223	ldo	R%sys_call_table64(%r1), %r19
224#else
225	load32	sys_call_table, %r19
226#endif
227	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
228	b,n	.Lsyscall_nosys
229
230	LDREGX  %r20(%r19), %r19
231
232	/* If this is a sys_rt_sigreturn call, and the signal was received
233	 * when not in_syscall, then we want to return via syscall_exit_rfi,
234	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
235	 * trampoline code in signal.c).
236	 */
237	ldi	__NR_rt_sigreturn,%r2
238	comb,=	%r2,%r20,.Lrt_sigreturn
239.Lin_syscall:
240	ldil	L%syscall_exit,%r2
241	be      0(%sr7,%r19)
242	ldo	R%syscall_exit(%r2),%r2
243.Lrt_sigreturn:
244	comib,<> 0,%r25,.Lin_syscall
245	ldil	L%syscall_exit_rfi,%r2
246	be      0(%sr7,%r19)
247	ldo	R%syscall_exit_rfi(%r2),%r2
248
249	/* Note!  Because we are not running where we were linked, any
250	calls to functions external to this file must be indirect.  To
251	be safe, we apply the opposite rule to functions within this
252	file, with local labels given to them to ensure correctness. */
253
254.Lsyscall_nosys:
255syscall_nosys:
256	ldil	L%syscall_exit,%r1
257	be	R%syscall_exit(%sr7,%r1)
258	ldo	-ENOSYS(%r0),%r28		   /* set errno */
259
260
261/* Warning! This trace code is a virtual duplicate of the code above so be
262 * sure to maintain both! */
263.Ltracesys:
264tracesys:
265	/* Need to save more registers so the debugger can see where we
266	 * are.  This saves only the lower 8 bits of PSW, so that the C
267	 * bit is still clear on syscalls, and the D bit is set if this
268	 * full register save path has been executed.  We check the D
269	 * bit on syscall_return_rfi to determine which registers to
270	 * restore.  An interrupt results in a full PSW saved with the
271	 * C bit set, a non-straced syscall entry results in C and D clear
272	 * in the saved PSW.
273	 */
274	mfctl	%cr30,%r1			/* get task ptr */
275	ssm	0,%r2
276	STREG	%r2,TASK_PT_PSW(%r1)		/* Lower 8 bits only!! */
277	mfsp	%sr0,%r2
278	STREG	%r2,TASK_PT_SR0(%r1)
279	mfsp	%sr1,%r2
280	STREG	%r2,TASK_PT_SR1(%r1)
281	mfsp	%sr2,%r2
282	STREG	%r2,TASK_PT_SR2(%r1)
283	mfsp	%sr3,%r2
284	STREG	%r2,TASK_PT_SR3(%r1)
285	STREG	%r2,TASK_PT_SR4(%r1)
286	STREG	%r2,TASK_PT_SR5(%r1)
287	STREG	%r2,TASK_PT_SR6(%r1)
288	STREG	%r2,TASK_PT_SR7(%r1)
289	STREG	%r2,TASK_PT_IASQ0(%r1)
290	STREG	%r2,TASK_PT_IASQ1(%r1)
291	LDREG	TASK_PT_GR31(%r1),%r2
292	STREG	%r2,TASK_PT_IAOQ0(%r1)
293	ldo	4(%r2),%r2
294	STREG	%r2,TASK_PT_IAOQ1(%r1)
295	ldo	TASK_REGS(%r1),%r2
296	/* reg_save %r2 */
297	STREG	%r3,PT_GR3(%r2)
298	STREG	%r4,PT_GR4(%r2)
299	STREG	%r5,PT_GR5(%r2)
300	STREG	%r6,PT_GR6(%r2)
301	STREG	%r7,PT_GR7(%r2)
302	STREG	%r8,PT_GR8(%r2)
303	STREG	%r9,PT_GR9(%r2)
304	STREG	%r10,PT_GR10(%r2)
305	STREG	%r11,PT_GR11(%r2)
306	STREG	%r12,PT_GR12(%r2)
307	STREG	%r13,PT_GR13(%r2)
308	STREG	%r14,PT_GR14(%r2)
309	STREG	%r15,PT_GR15(%r2)
310	STREG	%r16,PT_GR16(%r2)
311	STREG	%r17,PT_GR17(%r2)
312	STREG	%r18,PT_GR18(%r2)
313	/* Finished saving things for the debugger */
314
315	copy	%r2,%r26
316	ldil	L%do_syscall_trace_enter,%r1
317	ldil	L%tracesys_next,%r2
318	be	R%do_syscall_trace_enter(%sr7,%r1)
319	ldo	R%tracesys_next(%r2),%r2
320
321tracesys_next:
322	/* do_syscall_trace_enter either returned the syscallno, or -1L,
323	 *  so we skip restoring the PT_GR20 below, since we pulled it from
324	 *  task->thread.regs.gr[20] above.
325	 */
326	copy	%ret0,%r20
327
328	mfctl	%cr30,%r1			/* get task ptr */
329	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return value */
330	LDREG   TASK_PT_GR26(%r1), %r26		/* Restore the users args */
331	LDREG   TASK_PT_GR25(%r1), %r25
332	LDREG   TASK_PT_GR24(%r1), %r24
333	LDREG   TASK_PT_GR23(%r1), %r23
334	LDREG   TASK_PT_GR22(%r1), %r22
335	LDREG   TASK_PT_GR21(%r1), %r21
336#ifdef CONFIG_64BIT
337	ldo	-16(%r30),%r29			/* Reference param save area */
338#else
339	stw     %r22, -52(%r30)                 /* 5th argument */
340	stw     %r21, -56(%r30)                 /* 6th argument */
341#endif
342
343	cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */
344	comiclr,>>	__NR_Linux_syscalls, %r20, %r0
345	b,n	.Ltracesys_nosys
346
347	/* Note!  We cannot use the syscall table that is mapped
348	nearby since the gateway page is mapped execute-only. */
349
350#ifdef CONFIG_64BIT
351	LDREG	TASK_PT_GR30(%r1), %r19		/* get users sp back */
352	extrd,u	%r19,63,1,%r2			/* W hidden in bottom bit */
353
354	ldil	L%sys_call_table, %r1
355	or,=	%r2,%r2,%r2
356	addil	L%(sys_call_table64-sys_call_table), %r1
357	ldo	R%sys_call_table(%r1), %r19
358	or,=	%r2,%r2,%r2
359	ldo	R%sys_call_table64(%r1), %r19
360#else
361	load32	sys_call_table, %r19
362#endif
363
364	LDREGX  %r20(%r19), %r19
365
366	/* If this is a sys_rt_sigreturn call, and the signal was received
367	 * when not in_syscall, then we want to return via syscall_exit_rfi,
368	 * not syscall_exit.  Signal no. in r20, in_syscall in r25 (see
369	 * trampoline code in signal.c).
370	 */
371	ldi	__NR_rt_sigreturn,%r2
372	comb,=	%r2,%r20,.Ltrace_rt_sigreturn
373.Ltrace_in_syscall:
374	ldil	L%tracesys_exit,%r2
375	be      0(%sr7,%r19)
376	ldo	R%tracesys_exit(%r2),%r2
377
378.Ltracesys_nosys:
379	ldo	-ENOSYS(%r0),%r28		/* set errno */
380
381	/* Do *not* call this function on the gateway page, because it
382	makes a direct call to syscall_trace. */
383
384tracesys_exit:
385	mfctl	%cr30,%r1			/* get task ptr */
386#ifdef CONFIG_64BIT
387	ldo	-16(%r30),%r29			/* Reference param save area */
388#endif
389	ldo	TASK_REGS(%r1),%r26
390	BL	do_syscall_trace_exit,%r2
391	STREG   %r28,TASK_PT_GR28(%r1)          /* save return value now */
392	mfctl	%cr30,%r1			/* get task ptr */
393	LDREG   TASK_PT_GR28(%r1), %r28		/* Restore return val. */
394
395	ldil	L%syscall_exit,%r1
396	be,n	R%syscall_exit(%sr7,%r1)
397
398.Ltrace_rt_sigreturn:
399	comib,<> 0,%r25,.Ltrace_in_syscall
400	ldil	L%tracesys_sigexit,%r2
401	be      0(%sr7,%r19)
402	ldo	R%tracesys_sigexit(%r2),%r2
403
404tracesys_sigexit:
405	mfctl	%cr30,%r1			/* get task ptr */
406#ifdef CONFIG_64BIT
407	ldo	-16(%r30),%r29			/* Reference param save area */
408#endif
409	BL	do_syscall_trace_exit,%r2
410	ldo	TASK_REGS(%r1),%r26
411
412	ldil	L%syscall_exit_rfi,%r1
413	be,n	R%syscall_exit_rfi(%sr7,%r1)
414
415
416	/*********************************************************
417		32/64-bit Light-Weight-Syscall ABI
418
419		* - Indicates a hint for userspace inline asm
420		implementations.
421
422		Syscall number (caller-saves)
423	        - %r20
424	        * In asm clobber.
425
426		Argument registers (caller-saves)
427	        - %r26, %r25, %r24, %r23, %r22
428	        * In asm input.
429
430		Return registers (caller-saves)
431	        - %r28 (return), %r21 (errno)
432	        * In asm output.
433
434		Caller-saves registers
435	        - %r1, %r27, %r29
436	        - %r2 (return pointer)
437	        - %r31 (ble link register)
438	        * In asm clobber.
439
440		Callee-saves registers
441	        - %r3-%r18
442	        - %r30 (stack pointer)
443	        * Not in asm clobber.
444
445		If userspace is 32-bit:
446		Callee-saves registers
447	        - %r19 (32-bit PIC register)
448
449		Differences from 32-bit calling convention:
450		- Syscall number in %r20
451		- Additional argument register %r22 (arg4)
452		- Callee-saves %r19.
453
454		If userspace is 64-bit:
455		Callee-saves registers
456		- %r27 (64-bit PIC register)
457
458		Differences from 64-bit calling convention:
459		- Syscall number in %r20
460		- Additional argument register %r22 (arg4)
461		- Callee-saves %r27.
462
463		Error codes returned by entry path:
464
465		ENOSYS - r20 was an invalid LWS number.
466
467	*********************************************************/
468lws_start:
469
470#ifdef CONFIG_64BIT
471	ssm	PSW_SM_W, %r1
472	extrd,u	%r1,PSW_W_BIT,1,%r1
473	/* sp must be aligned on 4, so deposit the W bit setting into
474	 * the bottom of sp temporarily */
475	or,ev	%r1,%r30,%r30
476
477	/* Clip LWS number to a 32-bit value for 32-bit processes */
478	depdi	0, 31, 32, %r20
479#endif
480
481        /* Is the lws entry number valid? */
482	comiclr,>>	__NR_lws_entries, %r20, %r0
483	b,n	lws_exit_nosys
484
485	/* Load table start */
486	ldil	L%lws_table, %r1
487	ldo	R%lws_table(%r1), %r28	/* Scratch use of r28 */
488	LDREGX	%r20(%sr2,r28), %r21	/* Scratch use of r21 */
489
490	/* Jump to lws, lws table pointers already relocated */
491	be,n	0(%sr2,%r21)
492
493lws_exit_nosys:
494	ldo	-ENOSYS(%r0),%r21		   /* set errno */
495	/* Fall through: Return to userspace */
496
497lws_exit:
498#ifdef CONFIG_64BIT
499	/* decide whether to reset the wide mode bit
500	 *
501	 * For a syscall, the W bit is stored in the lowest bit
502	 * of sp.  Extract it and reset W if it is zero */
503	extrd,u,*<>	%r30,63,1,%r1
504	rsm	PSW_SM_W, %r0
505	/* now reset the lowest bit of sp if it was set */
506	xor	%r30,%r1,%r30
507#endif
508	be,n	0(%sr7, %r31)
509
510
511
512	/***************************************************
513		Implementing 32bit CAS as an atomic operation:
514
515		%r26 - Address to examine
516		%r25 - Old value to check (old)
517		%r24 - New value to set (new)
518		%r28 - Return prev through this register.
519		%r21 - Kernel error code
520
521		If debugging is DISabled:
522
523		%r21 has the following meanings:
524
525		EAGAIN - CAS is busy, ldcw failed, try again.
526		EFAULT - Read or write failed.
527
528		If debugging is enabled:
529
530		EDEADLOCK - CAS called recursively.
531		EAGAIN && r28 == 1 - CAS is busy. Lock contended.
532		EAGAIN && r28 == 2 - CAS is busy. ldcw failed.
533		EFAULT - Read or write failed.
534
535		Scratch: r20, r28, r1
536
537	****************************************************/
538
539	/* Do not enable LWS debugging */
540#define ENABLE_LWS_DEBUG 0
541
542	/* ELF64 Process entry path */
543lws_compare_and_swap64:
544#ifdef CONFIG_64BIT
545	b,n	lws_compare_and_swap
546#else
547	/* If we are not a 64-bit kernel, then we don't
548	 * have 64-bit input registers, and calling
549	 * the 64-bit LWS CAS returns ENOSYS.
550	 */
551	b,n	lws_exit_nosys
552#endif
553
554	/* ELF32 Process entry path */
555lws_compare_and_swap32:
556#ifdef CONFIG_64BIT
557	/* Clip all the input registers */
558	depdi	0, 31, 32, %r26
559	depdi	0, 31, 32, %r25
560	depdi	0, 31, 32, %r24
561#endif
562
563lws_compare_and_swap:
564	/* Load start of lock table */
565	ldil	L%lws_lock_start, %r20
566	ldo	R%lws_lock_start(%r20), %r28
567
568	/* Extract eight bits from r26 and hash lock (Bits 3-11) */
569	extru  %r26, 28, 8, %r20
570
571	/* Find lock to use, the hash is either one of 0 to
572	   15, multiplied by 16 (keep it 16-byte aligned)
573	   and add to the lock table offset. */
574	shlw	%r20, 4, %r20
575	add	%r20, %r28, %r20
576
577# if ENABLE_LWS_DEBUG
578	/*
579		DEBUG, check for deadlock!
580		If the thread register values are the same
581		then we were the one that locked it last and
582		this is a recurisve call that will deadlock.
583		We *must* giveup this call and fail.
584	*/
585	ldw	4(%sr2,%r20), %r28			/* Load thread register */
586	/* WARNING: If cr27 cycles to the same value we have problems */
587	mfctl	%cr27, %r21				/* Get current thread register */
588	cmpb,<>,n	%r21, %r28, cas_lock		/* Called recursive? */
589	b	lws_exit				/* Return error! */
590	ldo	-EDEADLOCK(%r0), %r21
591cas_lock:
592	cmpb,=,n	%r0, %r28, cas_nocontend	/* Is nobody using it? */
593	ldo	1(%r0), %r28				/* 1st case */
594	b	lws_exit				/* Contended... */
595	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
596cas_nocontend:
597# endif
598/* ENABLE_LWS_DEBUG */
599
600	rsm	PSW_SM_I, %r0				/* Disable interrupts */
601	/* COW breaks can cause contention on UP systems */
602	LDCW	0(%sr2,%r20), %r28			/* Try to acquire the lock */
603	cmpb,<>,n	%r0, %r28, cas_action		/* Did we get it? */
604cas_wouldblock:
605	ldo	2(%r0), %r28				/* 2nd case */
606	ssm	PSW_SM_I, %r0
607	b	lws_exit				/* Contended... */
608	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
609
610	/*
611		prev = *addr;
612		if ( prev == old )
613		  *addr = new;
614		return prev;
615	*/
616
617	/* NOTES:
618		This all works becuse intr_do_signal
619		and schedule both check the return iasq
620		and see that we are on the kernel page
621		so this process is never scheduled off
622		or is ever sent any signal of any sort,
623		thus it is wholly atomic from usrspaces
624		perspective
625	*/
626cas_action:
627#if defined CONFIG_SMP && ENABLE_LWS_DEBUG
628	/* DEBUG */
629	mfctl	%cr27, %r1
630	stw	%r1, 4(%sr2,%r20)
631#endif
632	/* The load and store could fail */
6331:	ldw	0(%r26), %r28
634	sub,<>	%r28, %r25, %r0
6352:	stw	%r24, 0(%r26)
636	/* Free lock */
637	stw,ma	%r20, 0(%sr2,%r20)
638#if ENABLE_LWS_DEBUG
639	/* Clear thread register indicator */
640	stw	%r0, 4(%sr2,%r20)
641#endif
642	/* Enable interrupts */
643	ssm	PSW_SM_I, %r0
644	/* Return to userspace, set no error */
645	b	lws_exit
646	copy	%r0, %r21
647
6483:
649	/* Error occurred on load or store */
650	/* Free lock */
651	stw,ma	%r20, 0(%sr2,%r20)
652#if ENABLE_LWS_DEBUG
653	stw	%r0, 4(%sr2,%r20)
654#endif
655	ssm	PSW_SM_I, %r0
656	b	lws_exit
657	ldo	-EFAULT(%r0),%r21	/* set errno */
658	nop
659	nop
660	nop
661	nop
662
663	/* Two exception table entries, one for the load,
664	   the other for the store. Either return -EFAULT.
665	   Each of the entries must be relocated. */
666	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page)
667	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page)
668
669
670	/***************************************************
671		New CAS implementation which uses pointers and variable size
672		information. The value pointed by old and new MUST NOT change
673		while performing CAS. The lock only protect the value at %r26.
674
675		%r26 - Address to examine
676		%r25 - Pointer to the value to check (old)
677		%r24 - Pointer to the value to set (new)
678		%r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit)
679		%r28 - Return non-zero on failure
680		%r21 - Kernel error code
681
682		%r21 has the following meanings:
683
684		EAGAIN - CAS is busy, ldcw failed, try again.
685		EFAULT - Read or write failed.
686
687		Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only)
688
689	****************************************************/
690
691	/* ELF32 Process entry path */
692lws_compare_and_swap_2:
693#ifdef CONFIG_64BIT
694	/* Clip the input registers. We don't need to clip %r23 as we
695	   only use it for word operations */
696	depdi	0, 31, 32, %r26
697	depdi	0, 31, 32, %r25
698	depdi	0, 31, 32, %r24
699#endif
700
701	/* Check the validity of the size pointer */
702	subi,>>= 3, %r23, %r0
703	b,n	lws_exit_nosys
704
705	/* Jump to the functions which will load the old and new values into
706	   registers depending on the their size */
707	shlw	%r23, 2, %r29
708	blr	%r29, %r0
709	nop
710
711	/* 8bit load */
7124:	ldb	0(%r25), %r25
713	b	cas2_lock_start
7145:	ldb	0(%r24), %r24
715	nop
716	nop
717	nop
718	nop
719	nop
720
721	/* 16bit load */
7226:	ldh	0(%r25), %r25
723	b	cas2_lock_start
7247:	ldh	0(%r24), %r24
725	nop
726	nop
727	nop
728	nop
729	nop
730
731	/* 32bit load */
7328:	ldw	0(%r25), %r25
733	b	cas2_lock_start
7349:	ldw	0(%r24), %r24
735	nop
736	nop
737	nop
738	nop
739	nop
740
741	/* 64bit load */
742#ifdef CONFIG_64BIT
74310:	ldd	0(%r25), %r25
74411:	ldd	0(%r24), %r24
745#else
746	/* Load old value into r22/r23 - high/low */
74710:	ldw	0(%r25), %r22
74811:	ldw	4(%r25), %r23
749	/* Load new value into fr4 for atomic store later */
75012:	flddx	0(%r24), %fr4
751#endif
752
753cas2_lock_start:
754	/* Load start of lock table */
755	ldil	L%lws_lock_start, %r20
756	ldo	R%lws_lock_start(%r20), %r28
757
758	/* Extract eight bits from r26 and hash lock (Bits 3-11) */
759	extru  %r26, 28, 8, %r20
760
761	/* Find lock to use, the hash is either one of 0 to
762	   15, multiplied by 16 (keep it 16-byte aligned)
763	   and add to the lock table offset. */
764	shlw	%r20, 4, %r20
765	add	%r20, %r28, %r20
766
767	rsm	PSW_SM_I, %r0			/* Disable interrupts */
768	/* COW breaks can cause contention on UP systems */
769	LDCW	0(%sr2,%r20), %r28		/* Try to acquire the lock */
770	cmpb,<>,n	%r0, %r28, cas2_action	/* Did we get it? */
771cas2_wouldblock:
772	ldo	2(%r0), %r28			/* 2nd case */
773	ssm	PSW_SM_I, %r0
774	b	lws_exit			/* Contended... */
775	ldo	-EAGAIN(%r0), %r21		/* Spin in userspace */
776
777	/*
778		prev = *addr;
779		if ( prev == old )
780		  *addr = new;
781		return prev;
782	*/
783
784	/* NOTES:
785		This all works becuse intr_do_signal
786		and schedule both check the return iasq
787		and see that we are on the kernel page
788		so this process is never scheduled off
789		or is ever sent any signal of any sort,
790		thus it is wholly atomic from usrspaces
791		perspective
792	*/
793cas2_action:
794	/* Jump to the correct function */
795	blr	%r29, %r0
796	/* Set %r28 as non-zero for now */
797	ldo	1(%r0),%r28
798
799	/* 8bit CAS */
80013:	ldb	0(%r26), %r29
801	sub,=	%r29, %r25, %r0
802	b,n	cas2_end
80314:	stb	%r24, 0(%r26)
804	b	cas2_end
805	copy	%r0, %r28
806	nop
807	nop
808
809	/* 16bit CAS */
81015:	ldh	0(%r26), %r29
811	sub,=	%r29, %r25, %r0
812	b,n	cas2_end
81316:	sth	%r24, 0(%r26)
814	b	cas2_end
815	copy	%r0, %r28
816	nop
817	nop
818
819	/* 32bit CAS */
82017:	ldw	0(%r26), %r29
821	sub,=	%r29, %r25, %r0
822	b,n	cas2_end
82318:	stw	%r24, 0(%r26)
824	b	cas2_end
825	copy	%r0, %r28
826	nop
827	nop
828
829	/* 64bit CAS */
830#ifdef CONFIG_64BIT
83119:	ldd	0(%r26), %r29
832	sub,*=	%r29, %r25, %r0
833	b,n	cas2_end
83420:	std	%r24, 0(%r26)
835	copy	%r0, %r28
836#else
837	/* Compare first word */
83819:	ldw	0(%r26), %r29
839	sub,=	%r29, %r22, %r0
840	b,n	cas2_end
841	/* Compare second word */
84220:	ldw	4(%r26), %r29
843	sub,=	%r29, %r23, %r0
844	b,n	cas2_end
845	/* Perform the store */
84621:	fstdx	%fr4, 0(%r26)
847	copy	%r0, %r28
848#endif
849
850cas2_end:
851	/* Free lock */
852	stw,ma	%r20, 0(%sr2,%r20)
853	/* Enable interrupts */
854	ssm	PSW_SM_I, %r0
855	/* Return to userspace, set no error */
856	b	lws_exit
857	copy	%r0, %r21
858
85922:
860	/* Error occurred on load or store */
861	/* Free lock */
862	stw,ma	%r20, 0(%sr2,%r20)
863	ssm	PSW_SM_I, %r0
864	ldo	1(%r0),%r28
865	b	lws_exit
866	ldo	-EFAULT(%r0),%r21	/* set errno */
867	nop
868	nop
869	nop
870
871	/* Exception table entries, for the load and store, return EFAULT.
872	   Each of the entries must be relocated. */
873	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page)
874	ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page)
875	ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page)
876	ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page)
877	ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page)
878	ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page)
879	ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page)
880	ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page)
881	ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page)
882	ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page)
883	ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page)
884	ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page)
885	ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page)
886	ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page)
887	ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page)
888	ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page)
889#ifndef CONFIG_64BIT
890	ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page)
891	ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
892#endif
893
894	/* Make sure nothing else is placed on this page */
895	.align PAGE_SIZE
896END(linux_gateway_page)
897ENTRY(end_linux_gateway_page)
898
899	/* Relocate symbols assuming linux_gateway_page is mapped
900	   to virtual address 0x0 */
901
902#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page)
903
904	.section .rodata,"a"
905
906	.align 8
907	/* Light-weight-syscall table */
908	/* Start of lws table. */
909ENTRY(lws_table)
910	LWS_ENTRY(compare_and_swap32)		/* 0 - ELF32 Atomic 32bit CAS */
911	LWS_ENTRY(compare_and_swap64)		/* 1 - ELF64 Atomic 32bit CAS */
912	LWS_ENTRY(compare_and_swap_2)		/* 2 - ELF32 Atomic 64bit CAS */
913END(lws_table)
914	/* End of lws table */
915
916#ifdef CONFIG_64BIT
917#define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, compat)
918#else
919#define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, native)
920#endif
921#define __SYSCALL(nr, entry)	ASM_ULONG_INSN entry
922	.align 8
923ENTRY(sys_call_table)
924	.export sys_call_table,data
925#include <asm/syscall_table_32.h>    /* 32-bit syscalls */
926END(sys_call_table)
927
928#ifdef CONFIG_64BIT
929	.align 8
930ENTRY(sys_call_table64)
931#include <asm/syscall_table_64.h>    /* 64-bit syscalls */
932END(sys_call_table64)
933#endif
934
935	/*
936		All light-weight-syscall atomic operations
937		will use this set of locks
938
939		NOTE: The lws_lock_start symbol must be
940		at least 16-byte aligned for safe use
941		with ldcw.
942	*/
943	.section .data
944	.align	L1_CACHE_BYTES
945ENTRY(lws_lock_start)
946	/* lws locks */
947	.rept 256
948	/* Keep locks aligned at 16-bytes */
949	.word 1
950	.word 0
951	.word 0
952	.word 0
953	.endr
954END(lws_lock_start)
955	.previous
956
957.end
958