11da177e4SLinus Torvalds/* 21da177e4SLinus Torvalds * Linux/PA-RISC Project (http://www.parisc-linux.org/) 31da177e4SLinus Torvalds * 4dde39798SHelge Deller * System call entry code / Linux gateway page 53d0186bbSMatthew Wilcox * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org> 61da177e4SLinus Torvalds * Licensed under the GNU GPL. 71da177e4SLinus Torvalds * thanks to Philipp Rumpf, Mike Shaver and various others 81da177e4SLinus Torvalds * sorry about the wall, puffin.. 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11dde39798SHelge Deller/* 12dde39798SHelge DellerHow does the Linux gateway page on PA-RISC work? 13dde39798SHelge Deller------------------------------------------------ 14dde39798SHelge DellerThe Linux gateway page on PA-RISC is "special". 15dde39798SHelge DellerIt actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc 16dde39798SHelge Dellerterminology it's Execute, promote to PL0) in the page map. So anything 17dde39798SHelge Dellerexecuting on this page executes with kernel level privilege (there's more to it 18dde39798SHelge Dellerthan that: to have this happen, you also have to use a branch with a ,gate 19dde39798SHelge Dellercompleter to activate the privilege promotion). The upshot is that everything 20dde39798SHelge Dellerthat runs on the gateway page runs at kernel privilege but with the current 21dde39798SHelge Delleruser process address space (although you have access to kernel space via %sr2). 22dde39798SHelge DellerFor the 0x100 syscall entry, we redo the space registers to point to the kernel 23dde39798SHelge Delleraddress space (preserving the user address space in %sr3), move to wide mode if 24dde39798SHelge Dellerrequired, save the user registers and branch into the kernel syscall entry 25dde39798SHelge Dellerpoint. For all the other functions, we execute at kernel privilege but don't 26dde39798SHelge Dellerflip address spaces. The basic upshot of this is that these code snippets are 27dde39798SHelge Dellerexecuted atomically (because the kernel can't be pre-empted) and they may 28dde39798SHelge Dellerperform architecturally forbidden (to PL3) operations (like setting control 29dde39798SHelge Dellerregisters). 30dde39798SHelge Deller*/ 31dde39798SHelge Deller 32dde39798SHelge Deller 330013a854SSam Ravnborg#include <asm/asm-offsets.h> 341da177e4SLinus Torvalds#include <asm/unistd.h> 351da177e4SLinus Torvalds#include <asm/errno.h> 361c593571SSam Ravnborg#include <asm/page.h> 371da177e4SLinus Torvalds#include <asm/psw.h> 381da177e4SLinus Torvalds#include <asm/thread_info.h> 391da177e4SLinus Torvalds#include <asm/assembly.h> 401da177e4SLinus Torvalds#include <asm/processor.h> 416a45716aSHelge Deller#include <asm/cache.h> 421da177e4SLinus Torvalds 438e9e9844SHelge Deller#include <linux/linkage.h> 448e9e9844SHelge Deller 451da177e4SLinus Torvalds /* We fill the empty parts of the gateway page with 461da177e4SLinus Torvalds * something that will kill the kernel or a 471da177e4SLinus Torvalds * userspace application. 481da177e4SLinus Torvalds */ 491da177e4SLinus Torvalds#define KILL_INSN break 0,0 501da177e4SLinus Torvalds 511829dda0SHelge Deller .level PA_ASM_LEVEL 528e9e9844SHelge Deller 53dfcf753bSKyle McMartin .text 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds .import syscall_exit,code 561da177e4SLinus Torvalds .import syscall_exit_rfi,code 571da177e4SLinus Torvalds 581da177e4SLinus Torvalds /* Linux gateway page is aliased to virtual page 0 in the kernel 591da177e4SLinus Torvalds * address space. Since it is a gateway page it cannot be 601da177e4SLinus Torvalds * dereferenced, so null pointers will still fault. We start 611da177e4SLinus Torvalds * the actual entry point at 0x100. We put break instructions 621da177e4SLinus Torvalds * at the beginning of the page to trap null indirect function 631da177e4SLinus Torvalds * pointers. 641da177e4SLinus Torvalds */ 651da177e4SLinus Torvalds 661c593571SSam Ravnborg .align PAGE_SIZE 678e9e9844SHelge DellerENTRY(linux_gateway_page) 681da177e4SLinus Torvalds 691da177e4SLinus Torvalds /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */ 701da177e4SLinus Torvalds .rept 44 711da177e4SLinus Torvalds KILL_INSN 721da177e4SLinus Torvalds .endr 731da177e4SLinus Torvalds 74f4c0346cSJohn David Anglin /* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */ 751da177e4SLinus Torvalds /* Light-weight-syscall entry must always be located at 0xb0 */ 761da177e4SLinus Torvalds /* WARNING: Keep this number updated with table size changes */ 7789206491SGuy Martin#define __NR_lws_entries (3) 781da177e4SLinus Torvalds 791da177e4SLinus Torvaldslws_entry: 80f4c0346cSJohn David Anglin gate lws_start, %r0 /* increase privilege */ 81f4c0346cSJohn David Anglin depi 3, 31, 2, %r31 /* Ensure we return into user mode. */ 821da177e4SLinus Torvalds 83f4c0346cSJohn David Anglin /* Fill from 0xb8 to 0xe0 */ 84f4c0346cSJohn David Anglin .rept 10 851da177e4SLinus Torvalds KILL_INSN 861da177e4SLinus Torvalds .endr 871da177e4SLinus Torvalds 881da177e4SLinus Torvalds /* This function MUST be located at 0xe0 for glibc's threading 891da177e4SLinus Torvalds mechanism to work. DO NOT MOVE THIS CODE EVER! */ 901da177e4SLinus Torvaldsset_thread_pointer: 911da177e4SLinus Torvalds gate .+8, %r0 /* increase privilege */ 921da177e4SLinus Torvalds depi 3, 31, 2, %r31 /* Ensure we return into user mode. */ 931da177e4SLinus Torvalds be 0(%sr7,%r31) /* return to user space */ 941da177e4SLinus Torvalds mtctl %r26, %cr27 /* move arg0 to the control register */ 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds /* Increase the chance of trapping if random jumps occur to this 971da177e4SLinus Torvalds address, fill from 0xf0 to 0x100 */ 981da177e4SLinus Torvalds .rept 4 991da177e4SLinus Torvalds KILL_INSN 1001da177e4SLinus Torvalds .endr 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds/* This address must remain fixed at 0x100 for glibc's syscalls to work */ 1036a6e2a14SHelge Deller .align LINUX_GATEWAY_ADDR 1041da177e4SLinus Torvaldslinux_gateway_entry: 1051da177e4SLinus Torvalds gate .+8, %r0 /* become privileged */ 1061da177e4SLinus Torvalds mtsp %r0,%sr4 /* get kernel space into sr4 */ 1071da177e4SLinus Torvalds mtsp %r0,%sr5 /* get kernel space into sr5 */ 1081da177e4SLinus Torvalds mtsp %r0,%sr6 /* get kernel space into sr6 */ 1091da177e4SLinus Torvalds 110413059f2SGrant Grundler#ifdef CONFIG_64BIT 11154c770daSHelge Deller /* Store W bit on entry to the syscall in case it's a wide userland 11254c770daSHelge Deller * process. */ 1131da177e4SLinus Torvalds ssm PSW_SM_W, %r1 1141da177e4SLinus Torvalds extrd,u %r1,PSW_W_BIT,1,%r1 1151da177e4SLinus Torvalds /* sp must be aligned on 4, so deposit the W bit setting into 1161da177e4SLinus Torvalds * the bottom of sp temporarily */ 1171da177e4SLinus Torvalds or,ev %r1,%r30,%r30 1181da177e4SLinus Torvalds b,n 1f 1191da177e4SLinus Torvalds /* The top halves of argument registers must be cleared on syscall 1201da177e4SLinus Torvalds * entry from narrow executable. 1211da177e4SLinus Torvalds */ 1221da177e4SLinus Torvalds depdi 0, 31, 32, %r26 1231da177e4SLinus Torvalds depdi 0, 31, 32, %r25 1241da177e4SLinus Torvalds depdi 0, 31, 32, %r24 1251da177e4SLinus Torvalds depdi 0, 31, 32, %r23 1261da177e4SLinus Torvalds depdi 0, 31, 32, %r22 1271da177e4SLinus Torvalds depdi 0, 31, 32, %r21 1281da177e4SLinus Torvalds1: 1291da177e4SLinus Torvalds#endif 1306ed51832SJohn David Anglin 1316ed51832SJohn David Anglin /* We use a rsm/ssm pair to prevent sr3 from being clobbered 1326ed51832SJohn David Anglin * by external interrupts. 1336ed51832SJohn David Anglin */ 1346ed51832SJohn David Anglin mfsp %sr7,%r1 /* save user sr7 */ 1356ed51832SJohn David Anglin rsm PSW_SM_I, %r0 /* disable interrupts */ 1366ed51832SJohn David Anglin mtsp %r1,%sr3 /* and store it in sr3 */ 1376ed51832SJohn David Anglin 1381da177e4SLinus Torvalds mfctl %cr30,%r1 1391da177e4SLinus Torvalds xor %r1,%r30,%r30 /* ye olde xor trick */ 1401da177e4SLinus Torvalds xor %r1,%r30,%r1 1411da177e4SLinus Torvalds xor %r1,%r30,%r30 1421da177e4SLinus Torvalds 1431da177e4SLinus Torvalds ldo THREAD_SZ_ALGN+FRAME_SIZE(%r30),%r30 /* set up kernel stack */ 1441da177e4SLinus Torvalds 1451da177e4SLinus Torvalds /* N.B.: It is critical that we don't set sr7 to 0 until r30 1461da177e4SLinus Torvalds * contains a valid kernel stack pointer. It is also 1471da177e4SLinus Torvalds * critical that we don't start using the kernel stack 1481da177e4SLinus Torvalds * until after sr7 has been set to 0. 1491da177e4SLinus Torvalds */ 1501da177e4SLinus Torvalds 1511da177e4SLinus Torvalds mtsp %r0,%sr7 /* get kernel space into sr7 */ 1526ed51832SJohn David Anglin ssm PSW_SM_I, %r0 /* enable interrupts */ 1531da177e4SLinus Torvalds STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ 1541da177e4SLinus Torvalds mfctl %cr30,%r1 /* get task ptr in %r1 */ 1551da177e4SLinus Torvalds LDREG TI_TASK(%r1),%r1 1561da177e4SLinus Torvalds 1571da177e4SLinus Torvalds /* Save some registers for sigcontext and potential task 1581da177e4SLinus Torvalds switch (see entry.S for the details of which ones are 1591da177e4SLinus Torvalds saved/restored). TASK_PT_PSW is zeroed so we can see whether 1601da177e4SLinus Torvalds a process is on a syscall or not. For an interrupt the real 1611da177e4SLinus Torvalds PSW value is stored. This is needed for gdb and sys_ptrace. */ 1621da177e4SLinus Torvalds STREG %r0, TASK_PT_PSW(%r1) 1631da177e4SLinus Torvalds STREG %r2, TASK_PT_GR2(%r1) /* preserve rp */ 1641da177e4SLinus Torvalds STREG %r19, TASK_PT_GR19(%r1) 1651da177e4SLinus Torvalds 1661da177e4SLinus Torvalds LDREGM -FRAME_SIZE(%r30), %r2 /* get users sp back */ 167413059f2SGrant Grundler#ifdef CONFIG_64BIT 1681da177e4SLinus Torvalds extrd,u %r2,63,1,%r19 /* W hidden in bottom bit */ 1691da177e4SLinus Torvalds#if 0 1701da177e4SLinus Torvalds xor %r19,%r2,%r2 /* clear bottom bit */ 1711da177e4SLinus Torvalds depd,z %r19,1,1,%r19 1721da177e4SLinus Torvalds std %r19,TASK_PT_PSW(%r1) 1731da177e4SLinus Torvalds#endif 1741da177e4SLinus Torvalds#endif 1751da177e4SLinus Torvalds STREG %r2, TASK_PT_GR30(%r1) /* ... and save it */ 1761da177e4SLinus Torvalds 177aa0eecb0SCarlos O'Donell STREG %r20, TASK_PT_GR20(%r1) /* Syscall number */ 1781da177e4SLinus Torvalds STREG %r21, TASK_PT_GR21(%r1) 1791da177e4SLinus Torvalds STREG %r22, TASK_PT_GR22(%r1) 1801da177e4SLinus Torvalds STREG %r23, TASK_PT_GR23(%r1) /* 4th argument */ 1811da177e4SLinus Torvalds STREG %r24, TASK_PT_GR24(%r1) /* 3rd argument */ 1821da177e4SLinus Torvalds STREG %r25, TASK_PT_GR25(%r1) /* 2nd argument */ 1831da177e4SLinus Torvalds STREG %r26, TASK_PT_GR26(%r1) /* 1st argument */ 1841da177e4SLinus Torvalds STREG %r27, TASK_PT_GR27(%r1) /* user dp */ 1851da177e4SLinus Torvalds STREG %r28, TASK_PT_GR28(%r1) /* return value 0 */ 18600df111eSAl Viro STREG %r0, TASK_PT_ORIG_R28(%r1) /* don't prohibit restarts */ 1871da177e4SLinus Torvalds STREG %r29, TASK_PT_GR29(%r1) /* return value 1 */ 1881da177e4SLinus Torvalds STREG %r31, TASK_PT_GR31(%r1) /* preserve syscall return ptr */ 1891da177e4SLinus Torvalds 1901da177e4SLinus Torvalds ldo TASK_PT_FR0(%r1), %r27 /* save fpregs from the kernel */ 1911da177e4SLinus Torvalds save_fp %r27 /* or potential task switch */ 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds mfctl %cr11, %r27 /* i.e. SAR */ 1941da177e4SLinus Torvalds STREG %r27, TASK_PT_SAR(%r1) 1951da177e4SLinus Torvalds 1961da177e4SLinus Torvalds loadgp 1971da177e4SLinus Torvalds 198413059f2SGrant Grundler#ifdef CONFIG_64BIT 1991da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 2001da177e4SLinus Torvalds copy %r19,%r2 /* W bit back to r2 */ 2011da177e4SLinus Torvalds#else 2021da177e4SLinus Torvalds /* no need to save these on stack in wide mode because the first 8 2031da177e4SLinus Torvalds * args are passed in registers */ 2041da177e4SLinus Torvalds stw %r22, -52(%r30) /* 5th argument */ 2051da177e4SLinus Torvalds stw %r21, -56(%r30) /* 6th argument */ 2061da177e4SLinus Torvalds#endif 2071da177e4SLinus Torvalds 2081da177e4SLinus Torvalds /* Are we being ptraced? */ 2091da177e4SLinus Torvalds mfctl %cr30, %r1 21064482bd8SAl Viro LDREG TI_FLAGS(%r1),%r1 21164482bd8SAl Viro ldi _TIF_SYSCALL_TRACE_MASK, %r19 21264482bd8SAl Viro and,COND(=) %r1, %r19, %r0 21364482bd8SAl Viro b,n .Ltracesys 2141da177e4SLinus Torvalds 2151da177e4SLinus Torvalds /* Note! We cannot use the syscall table that is mapped 2161da177e4SLinus Torvalds nearby since the gateway page is mapped execute-only. */ 2171da177e4SLinus Torvalds 218413059f2SGrant Grundler#ifdef CONFIG_64BIT 2191da177e4SLinus Torvalds ldil L%sys_call_table, %r1 2201da177e4SLinus Torvalds or,= %r2,%r2,%r2 2211da177e4SLinus Torvalds addil L%(sys_call_table64-sys_call_table), %r1 2221da177e4SLinus Torvalds ldo R%sys_call_table(%r1), %r19 2231da177e4SLinus Torvalds or,= %r2,%r2,%r2 2241da177e4SLinus Torvalds ldo R%sys_call_table64(%r1), %r19 2251da177e4SLinus Torvalds#else 226b6fc0cccSHelge Deller load32 sys_call_table, %r19 2271da177e4SLinus Torvalds#endif 2283bb457afSKyle McMartin comiclr,>> __NR_Linux_syscalls, %r20, %r0 2291da177e4SLinus Torvalds b,n .Lsyscall_nosys 2301da177e4SLinus Torvalds 2311da177e4SLinus Torvalds LDREGX %r20(%r19), %r19 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds /* If this is a sys_rt_sigreturn call, and the signal was received 2341da177e4SLinus Torvalds * when not in_syscall, then we want to return via syscall_exit_rfi, 2351da177e4SLinus Torvalds * not syscall_exit. Signal no. in r20, in_syscall in r25 (see 2361da177e4SLinus Torvalds * trampoline code in signal.c). 2371da177e4SLinus Torvalds */ 2381da177e4SLinus Torvalds ldi __NR_rt_sigreturn,%r2 2391da177e4SLinus Torvalds comb,= %r2,%r20,.Lrt_sigreturn 2401da177e4SLinus Torvalds.Lin_syscall: 2411da177e4SLinus Torvalds ldil L%syscall_exit,%r2 2421da177e4SLinus Torvalds be 0(%sr7,%r19) 2431da177e4SLinus Torvalds ldo R%syscall_exit(%r2),%r2 2441da177e4SLinus Torvalds.Lrt_sigreturn: 2451da177e4SLinus Torvalds comib,<> 0,%r25,.Lin_syscall 2461da177e4SLinus Torvalds ldil L%syscall_exit_rfi,%r2 2471da177e4SLinus Torvalds be 0(%sr7,%r19) 2481da177e4SLinus Torvalds ldo R%syscall_exit_rfi(%r2),%r2 2491da177e4SLinus Torvalds 2501da177e4SLinus Torvalds /* Note! Because we are not running where we were linked, any 2511da177e4SLinus Torvalds calls to functions external to this file must be indirect. To 2521da177e4SLinus Torvalds be safe, we apply the opposite rule to functions within this 2531da177e4SLinus Torvalds file, with local labels given to them to ensure correctness. */ 2541da177e4SLinus Torvalds 2551da177e4SLinus Torvalds.Lsyscall_nosys: 2561da177e4SLinus Torvaldssyscall_nosys: 2571da177e4SLinus Torvalds ldil L%syscall_exit,%r1 2581da177e4SLinus Torvalds be R%syscall_exit(%sr7,%r1) 2591da177e4SLinus Torvalds ldo -ENOSYS(%r0),%r28 /* set errno */ 2601da177e4SLinus Torvalds 2611da177e4SLinus Torvalds 2621da177e4SLinus Torvalds/* Warning! This trace code is a virtual duplicate of the code above so be 2631da177e4SLinus Torvalds * sure to maintain both! */ 2641da177e4SLinus Torvalds.Ltracesys: 2651da177e4SLinus Torvaldstracesys: 2661da177e4SLinus Torvalds /* Need to save more registers so the debugger can see where we 2671da177e4SLinus Torvalds * are. This saves only the lower 8 bits of PSW, so that the C 2681da177e4SLinus Torvalds * bit is still clear on syscalls, and the D bit is set if this 2691da177e4SLinus Torvalds * full register save path has been executed. We check the D 2701da177e4SLinus Torvalds * bit on syscall_return_rfi to determine which registers to 2711da177e4SLinus Torvalds * restore. An interrupt results in a full PSW saved with the 2721da177e4SLinus Torvalds * C bit set, a non-straced syscall entry results in C and D clear 2731da177e4SLinus Torvalds * in the saved PSW. 2741da177e4SLinus Torvalds */ 2751da177e4SLinus Torvalds ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ 2761da177e4SLinus Torvalds LDREG TI_TASK(%r1), %r1 2771da177e4SLinus Torvalds ssm 0,%r2 2781da177e4SLinus Torvalds STREG %r2,TASK_PT_PSW(%r1) /* Lower 8 bits only!! */ 2791da177e4SLinus Torvalds mfsp %sr0,%r2 2801da177e4SLinus Torvalds STREG %r2,TASK_PT_SR0(%r1) 2811da177e4SLinus Torvalds mfsp %sr1,%r2 2821da177e4SLinus Torvalds STREG %r2,TASK_PT_SR1(%r1) 2831da177e4SLinus Torvalds mfsp %sr2,%r2 2841da177e4SLinus Torvalds STREG %r2,TASK_PT_SR2(%r1) 2851da177e4SLinus Torvalds mfsp %sr3,%r2 2861da177e4SLinus Torvalds STREG %r2,TASK_PT_SR3(%r1) 2871da177e4SLinus Torvalds STREG %r2,TASK_PT_SR4(%r1) 2881da177e4SLinus Torvalds STREG %r2,TASK_PT_SR5(%r1) 2891da177e4SLinus Torvalds STREG %r2,TASK_PT_SR6(%r1) 2901da177e4SLinus Torvalds STREG %r2,TASK_PT_SR7(%r1) 2911da177e4SLinus Torvalds STREG %r2,TASK_PT_IASQ0(%r1) 2921da177e4SLinus Torvalds STREG %r2,TASK_PT_IASQ1(%r1) 2931da177e4SLinus Torvalds LDREG TASK_PT_GR31(%r1),%r2 2941da177e4SLinus Torvalds STREG %r2,TASK_PT_IAOQ0(%r1) 2951da177e4SLinus Torvalds ldo 4(%r2),%r2 2961da177e4SLinus Torvalds STREG %r2,TASK_PT_IAOQ1(%r1) 2971da177e4SLinus Torvalds ldo TASK_REGS(%r1),%r2 2981da177e4SLinus Torvalds /* reg_save %r2 */ 2991da177e4SLinus Torvalds STREG %r3,PT_GR3(%r2) 3001da177e4SLinus Torvalds STREG %r4,PT_GR4(%r2) 3011da177e4SLinus Torvalds STREG %r5,PT_GR5(%r2) 3021da177e4SLinus Torvalds STREG %r6,PT_GR6(%r2) 3031da177e4SLinus Torvalds STREG %r7,PT_GR7(%r2) 3041da177e4SLinus Torvalds STREG %r8,PT_GR8(%r2) 3051da177e4SLinus Torvalds STREG %r9,PT_GR9(%r2) 3061da177e4SLinus Torvalds STREG %r10,PT_GR10(%r2) 3071da177e4SLinus Torvalds STREG %r11,PT_GR11(%r2) 3081da177e4SLinus Torvalds STREG %r12,PT_GR12(%r2) 3091da177e4SLinus Torvalds STREG %r13,PT_GR13(%r2) 3101da177e4SLinus Torvalds STREG %r14,PT_GR14(%r2) 3111da177e4SLinus Torvalds STREG %r15,PT_GR15(%r2) 3121da177e4SLinus Torvalds STREG %r16,PT_GR16(%r2) 3131da177e4SLinus Torvalds STREG %r17,PT_GR17(%r2) 3141da177e4SLinus Torvalds STREG %r18,PT_GR18(%r2) 3151da177e4SLinus Torvalds /* Finished saving things for the debugger */ 3161da177e4SLinus Torvalds 3172798af1aSKyle McMartin copy %r2,%r26 3182798af1aSKyle McMartin ldil L%do_syscall_trace_enter,%r1 3191da177e4SLinus Torvalds ldil L%tracesys_next,%r2 3202798af1aSKyle McMartin be R%do_syscall_trace_enter(%sr7,%r1) 3211da177e4SLinus Torvalds ldo R%tracesys_next(%r2),%r2 3221da177e4SLinus Torvalds 3231da177e4SLinus Torvaldstracesys_next: 3242798af1aSKyle McMartin /* do_syscall_trace_enter either returned the syscallno, or -1L, 3252798af1aSKyle McMartin * so we skip restoring the PT_GR20 below, since we pulled it from 3262798af1aSKyle McMartin * task->thread.regs.gr[20] above. 3272798af1aSKyle McMartin */ 3282798af1aSKyle McMartin copy %ret0,%r20 3291da177e4SLinus Torvalds 3301da177e4SLinus Torvalds ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ 3311da177e4SLinus Torvalds LDREG TI_TASK(%r1), %r1 332910cd32eSHelge Deller LDREG TASK_PT_GR28(%r1), %r28 /* Restore return value */ 3331da177e4SLinus Torvalds LDREG TASK_PT_GR26(%r1), %r26 /* Restore the users args */ 3341da177e4SLinus Torvalds LDREG TASK_PT_GR25(%r1), %r25 3351da177e4SLinus Torvalds LDREG TASK_PT_GR24(%r1), %r24 3361da177e4SLinus Torvalds LDREG TASK_PT_GR23(%r1), %r23 3371da177e4SLinus Torvalds LDREG TASK_PT_GR22(%r1), %r22 3381da177e4SLinus Torvalds LDREG TASK_PT_GR21(%r1), %r21 33952ab532eSAl Viro#ifdef CONFIG_64BIT 3401da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 34152ab532eSAl Viro#else 34252ab532eSAl Viro stw %r22, -52(%r30) /* 5th argument */ 34352ab532eSAl Viro stw %r21, -56(%r30) /* 6th argument */ 3441da177e4SLinus Torvalds#endif 3451da177e4SLinus Torvalds 346910cd32eSHelge Deller cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */ 347f0b22d1bSDmitry V. Levin comiclr,>> __NR_Linux_syscalls, %r20, %r0 34898e8b6c9SHelge Deller b,n .Ltracesys_nosys 3491da177e4SLinus Torvalds 350b6fc0cccSHelge Deller /* Note! We cannot use the syscall table that is mapped 351b6fc0cccSHelge Deller nearby since the gateway page is mapped execute-only. */ 352b6fc0cccSHelge Deller 353b6fc0cccSHelge Deller#ifdef CONFIG_64BIT 354b6fc0cccSHelge Deller LDREG TASK_PT_GR30(%r1), %r19 /* get users sp back */ 355b6fc0cccSHelge Deller extrd,u %r19,63,1,%r2 /* W hidden in bottom bit */ 356b6fc0cccSHelge Deller 357b6fc0cccSHelge Deller ldil L%sys_call_table, %r1 358b6fc0cccSHelge Deller or,= %r2,%r2,%r2 359b6fc0cccSHelge Deller addil L%(sys_call_table64-sys_call_table), %r1 360b6fc0cccSHelge Deller ldo R%sys_call_table(%r1), %r19 361b6fc0cccSHelge Deller or,= %r2,%r2,%r2 362b6fc0cccSHelge Deller ldo R%sys_call_table64(%r1), %r19 363b6fc0cccSHelge Deller#else 364b6fc0cccSHelge Deller load32 sys_call_table, %r19 365b6fc0cccSHelge Deller#endif 366b6fc0cccSHelge Deller 3671da177e4SLinus Torvalds LDREGX %r20(%r19), %r19 3681da177e4SLinus Torvalds 3691da177e4SLinus Torvalds /* If this is a sys_rt_sigreturn call, and the signal was received 3701da177e4SLinus Torvalds * when not in_syscall, then we want to return via syscall_exit_rfi, 3711da177e4SLinus Torvalds * not syscall_exit. Signal no. in r20, in_syscall in r25 (see 3721da177e4SLinus Torvalds * trampoline code in signal.c). 3731da177e4SLinus Torvalds */ 3741da177e4SLinus Torvalds ldi __NR_rt_sigreturn,%r2 3751da177e4SLinus Torvalds comb,= %r2,%r20,.Ltrace_rt_sigreturn 3761da177e4SLinus Torvalds.Ltrace_in_syscall: 3771da177e4SLinus Torvalds ldil L%tracesys_exit,%r2 3781da177e4SLinus Torvalds be 0(%sr7,%r19) 3791da177e4SLinus Torvalds ldo R%tracesys_exit(%r2),%r2 3801da177e4SLinus Torvalds 38198e8b6c9SHelge Deller.Ltracesys_nosys: 38298e8b6c9SHelge Deller ldo -ENOSYS(%r0),%r28 /* set errno */ 38398e8b6c9SHelge Deller 3841da177e4SLinus Torvalds /* Do *not* call this function on the gateway page, because it 3851da177e4SLinus Torvalds makes a direct call to syscall_trace. */ 3861da177e4SLinus Torvalds 3871da177e4SLinus Torvaldstracesys_exit: 3881da177e4SLinus Torvalds ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ 3891da177e4SLinus Torvalds LDREG TI_TASK(%r1), %r1 390413059f2SGrant Grundler#ifdef CONFIG_64BIT 3911da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 3921da177e4SLinus Torvalds#endif 3932798af1aSKyle McMartin ldo TASK_REGS(%r1),%r26 394337685e5SHelge Deller BL do_syscall_trace_exit,%r2 3951da177e4SLinus Torvalds STREG %r28,TASK_PT_GR28(%r1) /* save return value now */ 3961da177e4SLinus Torvalds ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ 3971da177e4SLinus Torvalds LDREG TI_TASK(%r1), %r1 3981da177e4SLinus Torvalds LDREG TASK_PT_GR28(%r1), %r28 /* Restore return val. */ 3991da177e4SLinus Torvalds 4001da177e4SLinus Torvalds ldil L%syscall_exit,%r1 4011da177e4SLinus Torvalds be,n R%syscall_exit(%sr7,%r1) 4021da177e4SLinus Torvalds 4031da177e4SLinus Torvalds.Ltrace_rt_sigreturn: 4041da177e4SLinus Torvalds comib,<> 0,%r25,.Ltrace_in_syscall 4051da177e4SLinus Torvalds ldil L%tracesys_sigexit,%r2 4061da177e4SLinus Torvalds be 0(%sr7,%r19) 4071da177e4SLinus Torvalds ldo R%tracesys_sigexit(%r2),%r2 4081da177e4SLinus Torvalds 4091da177e4SLinus Torvaldstracesys_sigexit: 4101da177e4SLinus Torvalds ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */ 4115837d42fSKyle McMartin LDREG TI_TASK(%r1), %r1 412413059f2SGrant Grundler#ifdef CONFIG_64BIT 4131da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 4141da177e4SLinus Torvalds#endif 415337685e5SHelge Deller BL do_syscall_trace_exit,%r2 4162798af1aSKyle McMartin ldo TASK_REGS(%r1),%r26 4171da177e4SLinus Torvalds 4181da177e4SLinus Torvalds ldil L%syscall_exit_rfi,%r1 4191da177e4SLinus Torvalds be,n R%syscall_exit_rfi(%sr7,%r1) 4201da177e4SLinus Torvalds 4211da177e4SLinus Torvalds 4221da177e4SLinus Torvalds /********************************************************* 423c84c3a69SHelge Deller 32/64-bit Light-Weight-Syscall ABI 4241da177e4SLinus Torvalds 425c84c3a69SHelge Deller * - Indicates a hint for userspace inline asm 426c84c3a69SHelge Deller implementations. 4271da177e4SLinus Torvalds 428c84c3a69SHelge Deller Syscall number (caller-saves) 429c84c3a69SHelge Deller - %r20 430c84c3a69SHelge Deller * In asm clobber. 4311da177e4SLinus Torvalds 432c84c3a69SHelge Deller Argument registers (caller-saves) 433c84c3a69SHelge Deller - %r26, %r25, %r24, %r23, %r22 434c84c3a69SHelge Deller * In asm input. 435c84c3a69SHelge Deller 436c84c3a69SHelge Deller Return registers (caller-saves) 437c84c3a69SHelge Deller - %r28 (return), %r21 (errno) 438c84c3a69SHelge Deller * In asm output. 439c84c3a69SHelge Deller 440c84c3a69SHelge Deller Caller-saves registers 441c84c3a69SHelge Deller - %r1, %r27, %r29 442c84c3a69SHelge Deller - %r2 (return pointer) 443c84c3a69SHelge Deller - %r31 (ble link register) 444c84c3a69SHelge Deller * In asm clobber. 445c84c3a69SHelge Deller 446c84c3a69SHelge Deller Callee-saves registers 447c84c3a69SHelge Deller - %r3-%r18 448c84c3a69SHelge Deller - %r30 (stack pointer) 449c84c3a69SHelge Deller * Not in asm clobber. 450c84c3a69SHelge Deller 451c84c3a69SHelge Deller If userspace is 32-bit: 452c84c3a69SHelge Deller Callee-saves registers 453c84c3a69SHelge Deller - %r19 (32-bit PIC register) 454c84c3a69SHelge Deller 455c84c3a69SHelge Deller Differences from 32-bit calling convention: 456c84c3a69SHelge Deller - Syscall number in %r20 457c84c3a69SHelge Deller - Additional argument register %r22 (arg4) 458c84c3a69SHelge Deller - Callee-saves %r19. 459c84c3a69SHelge Deller 460c84c3a69SHelge Deller If userspace is 64-bit: 461c84c3a69SHelge Deller Callee-saves registers 462c84c3a69SHelge Deller - %r27 (64-bit PIC register) 463c84c3a69SHelge Deller 464c84c3a69SHelge Deller Differences from 64-bit calling convention: 465c84c3a69SHelge Deller - Syscall number in %r20 466c84c3a69SHelge Deller - Additional argument register %r22 (arg4) 467c84c3a69SHelge Deller - Callee-saves %r27. 4681da177e4SLinus Torvalds 4691da177e4SLinus Torvalds Error codes returned by entry path: 4701da177e4SLinus Torvalds 4711da177e4SLinus Torvalds ENOSYS - r20 was an invalid LWS number. 4721da177e4SLinus Torvalds 4731da177e4SLinus Torvalds *********************************************************/ 4741da177e4SLinus Torvaldslws_start: 4751da177e4SLinus Torvalds 476413059f2SGrant Grundler#ifdef CONFIG_64BIT 4771da177e4SLinus Torvalds ssm PSW_SM_W, %r1 4781da177e4SLinus Torvalds extrd,u %r1,PSW_W_BIT,1,%r1 4791da177e4SLinus Torvalds /* sp must be aligned on 4, so deposit the W bit setting into 4801da177e4SLinus Torvalds * the bottom of sp temporarily */ 4811da177e4SLinus Torvalds or,ev %r1,%r30,%r30 4821da177e4SLinus Torvalds 48354c770daSHelge Deller /* Clip LWS number to a 32-bit value for 32-bit processes */ 4841da177e4SLinus Torvalds depdi 0, 31, 32, %r20 4851da177e4SLinus Torvalds#endif 4861da177e4SLinus Torvalds 4871da177e4SLinus Torvalds /* Is the lws entry number valid? */ 488f4c0346cSJohn David Anglin comiclr,>> __NR_lws_entries, %r20, %r0 4891da177e4SLinus Torvalds b,n lws_exit_nosys 4901da177e4SLinus Torvalds 4911da177e4SLinus Torvalds /* Load table start */ 4921da177e4SLinus Torvalds ldil L%lws_table, %r1 4931da177e4SLinus Torvalds ldo R%lws_table(%r1), %r28 /* Scratch use of r28 */ 4941da177e4SLinus Torvalds LDREGX %r20(%sr2,r28), %r21 /* Scratch use of r21 */ 4951da177e4SLinus Torvalds 4961da177e4SLinus Torvalds /* Jump to lws, lws table pointers already relocated */ 4971da177e4SLinus Torvalds be,n 0(%sr2,%r21) 4981da177e4SLinus Torvalds 4991da177e4SLinus Torvaldslws_exit_nosys: 5001da177e4SLinus Torvalds ldo -ENOSYS(%r0),%r21 /* set errno */ 5011da177e4SLinus Torvalds /* Fall through: Return to userspace */ 5021da177e4SLinus Torvalds 5031da177e4SLinus Torvaldslws_exit: 504413059f2SGrant Grundler#ifdef CONFIG_64BIT 5051da177e4SLinus Torvalds /* decide whether to reset the wide mode bit 5061da177e4SLinus Torvalds * 5071da177e4SLinus Torvalds * For a syscall, the W bit is stored in the lowest bit 5081da177e4SLinus Torvalds * of sp. Extract it and reset W if it is zero */ 5091da177e4SLinus Torvalds extrd,u,*<> %r30,63,1,%r1 5101da177e4SLinus Torvalds rsm PSW_SM_W, %r0 5111da177e4SLinus Torvalds /* now reset the lowest bit of sp if it was set */ 5121da177e4SLinus Torvalds xor %r30,%r1,%r30 5131da177e4SLinus Torvalds#endif 514f4c0346cSJohn David Anglin be,n 0(%sr7, %r31) 5151da177e4SLinus Torvalds 5161da177e4SLinus Torvalds 5171da177e4SLinus Torvalds 5181da177e4SLinus Torvalds /*************************************************** 51989206491SGuy Martin Implementing 32bit CAS as an atomic operation: 5201da177e4SLinus Torvalds 5211da177e4SLinus Torvalds %r26 - Address to examine 5221da177e4SLinus Torvalds %r25 - Old value to check (old) 5231da177e4SLinus Torvalds %r24 - New value to set (new) 5241da177e4SLinus Torvalds %r28 - Return prev through this register. 5251da177e4SLinus Torvalds %r21 - Kernel error code 5261da177e4SLinus Torvalds 5271da177e4SLinus Torvalds If debugging is DISabled: 5281da177e4SLinus Torvalds 5291da177e4SLinus Torvalds %r21 has the following meanings: 5301da177e4SLinus Torvalds 5311da177e4SLinus Torvalds EAGAIN - CAS is busy, ldcw failed, try again. 5321da177e4SLinus Torvalds EFAULT - Read or write failed. 5331da177e4SLinus Torvalds 5341da177e4SLinus Torvalds If debugging is enabled: 5351da177e4SLinus Torvalds 5361da177e4SLinus Torvalds EDEADLOCK - CAS called recursively. 5371da177e4SLinus Torvalds EAGAIN && r28 == 1 - CAS is busy. Lock contended. 5381da177e4SLinus Torvalds EAGAIN && r28 == 2 - CAS is busy. ldcw failed. 5391da177e4SLinus Torvalds EFAULT - Read or write failed. 5401da177e4SLinus Torvalds 5411da177e4SLinus Torvalds Scratch: r20, r28, r1 5421da177e4SLinus Torvalds 5431da177e4SLinus Torvalds ****************************************************/ 5441da177e4SLinus Torvalds 5451da177e4SLinus Torvalds /* Do not enable LWS debugging */ 5461da177e4SLinus Torvalds#define ENABLE_LWS_DEBUG 0 5471da177e4SLinus Torvalds 5481da177e4SLinus Torvalds /* ELF64 Process entry path */ 5491da177e4SLinus Torvaldslws_compare_and_swap64: 550413059f2SGrant Grundler#ifdef CONFIG_64BIT 5511da177e4SLinus Torvalds b,n lws_compare_and_swap 5521da177e4SLinus Torvalds#else 5531da177e4SLinus Torvalds /* If we are not a 64-bit kernel, then we don't 554c84c3a69SHelge Deller * have 64-bit input registers, and calling 555c84c3a69SHelge Deller * the 64-bit LWS CAS returns ENOSYS. 5561da177e4SLinus Torvalds */ 5571da177e4SLinus Torvalds b,n lws_exit_nosys 5581da177e4SLinus Torvalds#endif 5591da177e4SLinus Torvalds 5601da177e4SLinus Torvalds /* ELF32 Process entry path */ 5611da177e4SLinus Torvaldslws_compare_and_swap32: 562413059f2SGrant Grundler#ifdef CONFIG_64BIT 5631da177e4SLinus Torvalds /* Clip all the input registers */ 5641da177e4SLinus Torvalds depdi 0, 31, 32, %r26 5651da177e4SLinus Torvalds depdi 0, 31, 32, %r25 5661da177e4SLinus Torvalds depdi 0, 31, 32, %r24 5671da177e4SLinus Torvalds#endif 5681da177e4SLinus Torvalds 5691da177e4SLinus Torvaldslws_compare_and_swap: 5701da177e4SLinus Torvalds /* Load start of lock table */ 5711da177e4SLinus Torvalds ldil L%lws_lock_start, %r20 5721da177e4SLinus Torvalds ldo R%lws_lock_start(%r20), %r28 5731da177e4SLinus Torvalds 5741da177e4SLinus Torvalds /* Extract four bits from r26 and hash lock (Bits 4-7) */ 5751da177e4SLinus Torvalds extru %r26, 27, 4, %r20 5761da177e4SLinus Torvalds 5771da177e4SLinus Torvalds /* Find lock to use, the hash is either one of 0 to 5781da177e4SLinus Torvalds 15, multiplied by 16 (keep it 16-byte aligned) 5791da177e4SLinus Torvalds and add to the lock table offset. */ 5801da177e4SLinus Torvalds shlw %r20, 4, %r20 5811da177e4SLinus Torvalds add %r20, %r28, %r20 5821da177e4SLinus Torvalds 583b5e8b733SAurelien Jarno# if ENABLE_LWS_DEBUG 5841da177e4SLinus Torvalds /* 5851da177e4SLinus Torvalds DEBUG, check for deadlock! 5861da177e4SLinus Torvalds If the thread register values are the same 5871da177e4SLinus Torvalds then we were the one that locked it last and 5881da177e4SLinus Torvalds this is a recurisve call that will deadlock. 5891da177e4SLinus Torvalds We *must* giveup this call and fail. 5901da177e4SLinus Torvalds */ 5911da177e4SLinus Torvalds ldw 4(%sr2,%r20), %r28 /* Load thread register */ 592aa0eecb0SCarlos O'Donell /* WARNING: If cr27 cycles to the same value we have problems */ 5931da177e4SLinus Torvalds mfctl %cr27, %r21 /* Get current thread register */ 5941da177e4SLinus Torvalds cmpb,<>,n %r21, %r28, cas_lock /* Called recursive? */ 5951da177e4SLinus Torvalds b lws_exit /* Return error! */ 5961da177e4SLinus Torvalds ldo -EDEADLOCK(%r0), %r21 5971da177e4SLinus Torvaldscas_lock: 5981da177e4SLinus Torvalds cmpb,=,n %r0, %r28, cas_nocontend /* Is nobody using it? */ 5991da177e4SLinus Torvalds ldo 1(%r0), %r28 /* 1st case */ 6001da177e4SLinus Torvalds b lws_exit /* Contended... */ 6011da177e4SLinus Torvalds ldo -EAGAIN(%r0), %r21 /* Spin in userspace */ 6021da177e4SLinus Torvaldscas_nocontend: 6031da177e4SLinus Torvalds# endif 6041da177e4SLinus Torvalds/* ENABLE_LWS_DEBUG */ 6051da177e4SLinus Torvalds 606c776cd89SJohn David Anglin rsm PSW_SM_I, %r0 /* Disable interrupts */ 607c776cd89SJohn David Anglin /* COW breaks can cause contention on UP systems */ 60864f49532SKyle McMartin LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */ 6091da177e4SLinus Torvalds cmpb,<>,n %r0, %r28, cas_action /* Did we get it? */ 6101da177e4SLinus Torvaldscas_wouldblock: 6111da177e4SLinus Torvalds ldo 2(%r0), %r28 /* 2nd case */ 612c776cd89SJohn David Anglin ssm PSW_SM_I, %r0 6131da177e4SLinus Torvalds b lws_exit /* Contended... */ 6141da177e4SLinus Torvalds ldo -EAGAIN(%r0), %r21 /* Spin in userspace */ 6151da177e4SLinus Torvalds 6161da177e4SLinus Torvalds /* 6171da177e4SLinus Torvalds prev = *addr; 6181da177e4SLinus Torvalds if ( prev == old ) 6191da177e4SLinus Torvalds *addr = new; 6201da177e4SLinus Torvalds return prev; 6211da177e4SLinus Torvalds */ 6221da177e4SLinus Torvalds 6231da177e4SLinus Torvalds /* NOTES: 6241da177e4SLinus Torvalds This all works becuse intr_do_signal 6251da177e4SLinus Torvalds and schedule both check the return iasq 6261da177e4SLinus Torvalds and see that we are on the kernel page 6271da177e4SLinus Torvalds so this process is never scheduled off 6281da177e4SLinus Torvalds or is ever sent any signal of any sort, 6291da177e4SLinus Torvalds thus it is wholly atomic from usrspaces 6301da177e4SLinus Torvalds perspective 6311da177e4SLinus Torvalds */ 6321da177e4SLinus Torvaldscas_action: 633b5e8b733SAurelien Jarno#if defined CONFIG_SMP && ENABLE_LWS_DEBUG 6341da177e4SLinus Torvalds /* DEBUG */ 6351da177e4SLinus Torvalds mfctl %cr27, %r1 6361da177e4SLinus Torvalds stw %r1, 4(%sr2,%r20) 6371da177e4SLinus Torvalds#endif 6381da177e4SLinus Torvalds /* The load and store could fail */ 6397797167fSJohn David Anglin1: ldw 0(%r26), %r28 6401da177e4SLinus Torvalds sub,<> %r28, %r25, %r0 6417797167fSJohn David Anglin2: stw %r24, 0(%r26) 6421da177e4SLinus Torvalds /* Free lock */ 643157e9afcSHelge Deller stw,ma %r20, 0(%sr2,%r20) 644b5e8b733SAurelien Jarno#if ENABLE_LWS_DEBUG 6451da177e4SLinus Torvalds /* Clear thread register indicator */ 6461da177e4SLinus Torvalds stw %r0, 4(%sr2,%r20) 6471da177e4SLinus Torvalds#endif 648c776cd89SJohn David Anglin /* Enable interrupts */ 649c776cd89SJohn David Anglin ssm PSW_SM_I, %r0 6501da177e4SLinus Torvalds /* Return to userspace, set no error */ 6511da177e4SLinus Torvalds b lws_exit 6521da177e4SLinus Torvalds copy %r0, %r21 6531da177e4SLinus Torvalds 6541da177e4SLinus Torvalds3: 65525985edcSLucas De Marchi /* Error occurred on load or store */ 6561da177e4SLinus Torvalds /* Free lock */ 657157e9afcSHelge Deller stw,ma %r20, 0(%sr2,%r20) 658b5e8b733SAurelien Jarno#if ENABLE_LWS_DEBUG 6591da177e4SLinus Torvalds stw %r0, 4(%sr2,%r20) 6601da177e4SLinus Torvalds#endif 661c776cd89SJohn David Anglin ssm PSW_SM_I, %r0 6621da177e4SLinus Torvalds b lws_exit 6631da177e4SLinus Torvalds ldo -EFAULT(%r0),%r21 /* set errno */ 6641da177e4SLinus Torvalds nop 6651da177e4SLinus Torvalds nop 6661da177e4SLinus Torvalds nop 6671da177e4SLinus Torvalds nop 6681da177e4SLinus Torvalds 6691da177e4SLinus Torvalds /* Two exception table entries, one for the load, 6701da177e4SLinus Torvalds the other for the store. Either return -EFAULT. 6711da177e4SLinus Torvalds Each of the entries must be relocated. */ 67261dbbaebSHelge Deller ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page) 67361dbbaebSHelge Deller ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page) 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds 67689206491SGuy Martin /*************************************************** 67789206491SGuy Martin New CAS implementation which uses pointers and variable size 67889206491SGuy Martin information. The value pointed by old and new MUST NOT change 67989206491SGuy Martin while performing CAS. The lock only protect the value at %r26. 68089206491SGuy Martin 68189206491SGuy Martin %r26 - Address to examine 68289206491SGuy Martin %r25 - Pointer to the value to check (old) 68389206491SGuy Martin %r24 - Pointer to the value to set (new) 68489206491SGuy Martin %r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 68589206491SGuy Martin %r28 - Return non-zero on failure 68689206491SGuy Martin %r21 - Kernel error code 68789206491SGuy Martin 68889206491SGuy Martin %r21 has the following meanings: 68989206491SGuy Martin 69089206491SGuy Martin EAGAIN - CAS is busy, ldcw failed, try again. 69189206491SGuy Martin EFAULT - Read or write failed. 69289206491SGuy Martin 69389206491SGuy Martin Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only) 69489206491SGuy Martin 69589206491SGuy Martin ****************************************************/ 69689206491SGuy Martin 69789206491SGuy Martin /* ELF32 Process entry path */ 69889206491SGuy Martinlws_compare_and_swap_2: 69989206491SGuy Martin#ifdef CONFIG_64BIT 70005f016d2SJohn David Anglin /* Clip the input registers. We don't need to clip %r23 as we 70105f016d2SJohn David Anglin only use it for word operations */ 70289206491SGuy Martin depdi 0, 31, 32, %r26 70389206491SGuy Martin depdi 0, 31, 32, %r25 70489206491SGuy Martin depdi 0, 31, 32, %r24 70589206491SGuy Martin#endif 70689206491SGuy Martin 70789206491SGuy Martin /* Check the validity of the size pointer */ 70805f016d2SJohn David Anglin subi,>>= 3, %r23, %r0 70989206491SGuy Martin b,n lws_exit_nosys 71089206491SGuy Martin 71189206491SGuy Martin /* Jump to the functions which will load the old and new values into 71289206491SGuy Martin registers depending on the their size */ 71389206491SGuy Martin shlw %r23, 2, %r29 71489206491SGuy Martin blr %r29, %r0 71589206491SGuy Martin nop 71689206491SGuy Martin 71789206491SGuy Martin /* 8bit load */ 718f4125cfdSJohn David Anglin4: ldb 0(%r25), %r25 71989206491SGuy Martin b cas2_lock_start 720f4125cfdSJohn David Anglin5: ldb 0(%r24), %r24 72189206491SGuy Martin nop 72289206491SGuy Martin nop 72389206491SGuy Martin nop 72489206491SGuy Martin nop 72589206491SGuy Martin nop 72689206491SGuy Martin 72789206491SGuy Martin /* 16bit load */ 728f4125cfdSJohn David Anglin6: ldh 0(%r25), %r25 72989206491SGuy Martin b cas2_lock_start 730f4125cfdSJohn David Anglin7: ldh 0(%r24), %r24 73189206491SGuy Martin nop 73289206491SGuy Martin nop 73389206491SGuy Martin nop 73489206491SGuy Martin nop 73589206491SGuy Martin nop 73689206491SGuy Martin 73789206491SGuy Martin /* 32bit load */ 738f4125cfdSJohn David Anglin8: ldw 0(%r25), %r25 73989206491SGuy Martin b cas2_lock_start 740f4125cfdSJohn David Anglin9: ldw 0(%r24), %r24 74189206491SGuy Martin nop 74289206491SGuy Martin nop 74389206491SGuy Martin nop 74489206491SGuy Martin nop 74589206491SGuy Martin nop 74689206491SGuy Martin 74789206491SGuy Martin /* 64bit load */ 74889206491SGuy Martin#ifdef CONFIG_64BIT 749f4125cfdSJohn David Anglin10: ldd 0(%r25), %r25 750f4125cfdSJohn David Anglin11: ldd 0(%r24), %r24 75189206491SGuy Martin#else 752374b3bf8SJohn David Anglin /* Load old value into r22/r23 - high/low */ 753f4125cfdSJohn David Anglin10: ldw 0(%r25), %r22 754f4125cfdSJohn David Anglin11: ldw 4(%r25), %r23 75589206491SGuy Martin /* Load new value into fr4 for atomic store later */ 756f4125cfdSJohn David Anglin12: flddx 0(%r24), %fr4 75789206491SGuy Martin#endif 75889206491SGuy Martin 75989206491SGuy Martincas2_lock_start: 76089206491SGuy Martin /* Load start of lock table */ 76189206491SGuy Martin ldil L%lws_lock_start, %r20 76289206491SGuy Martin ldo R%lws_lock_start(%r20), %r28 76389206491SGuy Martin 76489206491SGuy Martin /* Extract four bits from r26 and hash lock (Bits 4-7) */ 76589206491SGuy Martin extru %r26, 27, 4, %r20 76689206491SGuy Martin 76789206491SGuy Martin /* Find lock to use, the hash is either one of 0 to 76889206491SGuy Martin 15, multiplied by 16 (keep it 16-byte aligned) 76989206491SGuy Martin and add to the lock table offset. */ 77089206491SGuy Martin shlw %r20, 4, %r20 77189206491SGuy Martin add %r20, %r28, %r20 77289206491SGuy Martin 77389206491SGuy Martin rsm PSW_SM_I, %r0 /* Disable interrupts */ 77489206491SGuy Martin /* COW breaks can cause contention on UP systems */ 77589206491SGuy Martin LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */ 77689206491SGuy Martin cmpb,<>,n %r0, %r28, cas2_action /* Did we get it? */ 77789206491SGuy Martincas2_wouldblock: 77889206491SGuy Martin ldo 2(%r0), %r28 /* 2nd case */ 77989206491SGuy Martin ssm PSW_SM_I, %r0 78089206491SGuy Martin b lws_exit /* Contended... */ 78189206491SGuy Martin ldo -EAGAIN(%r0), %r21 /* Spin in userspace */ 78289206491SGuy Martin 78389206491SGuy Martin /* 78489206491SGuy Martin prev = *addr; 78589206491SGuy Martin if ( prev == old ) 78689206491SGuy Martin *addr = new; 78789206491SGuy Martin return prev; 78889206491SGuy Martin */ 78989206491SGuy Martin 79089206491SGuy Martin /* NOTES: 79189206491SGuy Martin This all works becuse intr_do_signal 79289206491SGuy Martin and schedule both check the return iasq 79389206491SGuy Martin and see that we are on the kernel page 79489206491SGuy Martin so this process is never scheduled off 79589206491SGuy Martin or is ever sent any signal of any sort, 79689206491SGuy Martin thus it is wholly atomic from usrspaces 79789206491SGuy Martin perspective 79889206491SGuy Martin */ 79989206491SGuy Martincas2_action: 80089206491SGuy Martin /* Jump to the correct function */ 80189206491SGuy Martin blr %r29, %r0 80289206491SGuy Martin /* Set %r28 as non-zero for now */ 80389206491SGuy Martin ldo 1(%r0),%r28 80489206491SGuy Martin 80589206491SGuy Martin /* 8bit CAS */ 8067797167fSJohn David Anglin13: ldb 0(%r26), %r29 80789206491SGuy Martin sub,= %r29, %r25, %r0 80889206491SGuy Martin b,n cas2_end 8097797167fSJohn David Anglin14: stb %r24, 0(%r26) 81089206491SGuy Martin b cas2_end 81189206491SGuy Martin copy %r0, %r28 81289206491SGuy Martin nop 81389206491SGuy Martin nop 81489206491SGuy Martin 81589206491SGuy Martin /* 16bit CAS */ 8167797167fSJohn David Anglin15: ldh 0(%r26), %r29 81789206491SGuy Martin sub,= %r29, %r25, %r0 81889206491SGuy Martin b,n cas2_end 8197797167fSJohn David Anglin16: sth %r24, 0(%r26) 82089206491SGuy Martin b cas2_end 82189206491SGuy Martin copy %r0, %r28 82289206491SGuy Martin nop 82389206491SGuy Martin nop 82489206491SGuy Martin 82589206491SGuy Martin /* 32bit CAS */ 8267797167fSJohn David Anglin17: ldw 0(%r26), %r29 82789206491SGuy Martin sub,= %r29, %r25, %r0 82889206491SGuy Martin b,n cas2_end 8297797167fSJohn David Anglin18: stw %r24, 0(%r26) 83089206491SGuy Martin b cas2_end 83189206491SGuy Martin copy %r0, %r28 83289206491SGuy Martin nop 83389206491SGuy Martin nop 83489206491SGuy Martin 83589206491SGuy Martin /* 64bit CAS */ 83689206491SGuy Martin#ifdef CONFIG_64BIT 8377797167fSJohn David Anglin19: ldd 0(%r26), %r29 8381b59ddfcSJohn David Anglin sub,*= %r29, %r25, %r0 83989206491SGuy Martin b,n cas2_end 8407797167fSJohn David Anglin20: std %r24, 0(%r26) 84189206491SGuy Martin copy %r0, %r28 84289206491SGuy Martin#else 84389206491SGuy Martin /* Compare first word */ 844374b3bf8SJohn David Anglin19: ldw 0(%r26), %r29 84589206491SGuy Martin sub,= %r29, %r22, %r0 84689206491SGuy Martin b,n cas2_end 84789206491SGuy Martin /* Compare second word */ 848374b3bf8SJohn David Anglin20: ldw 4(%r26), %r29 84989206491SGuy Martin sub,= %r29, %r23, %r0 85089206491SGuy Martin b,n cas2_end 85189206491SGuy Martin /* Perform the store */ 852f4125cfdSJohn David Anglin21: fstdx %fr4, 0(%r26) 85389206491SGuy Martin copy %r0, %r28 85489206491SGuy Martin#endif 85589206491SGuy Martin 85689206491SGuy Martincas2_end: 85789206491SGuy Martin /* Free lock */ 858157e9afcSHelge Deller stw,ma %r20, 0(%sr2,%r20) 85989206491SGuy Martin /* Enable interrupts */ 86089206491SGuy Martin ssm PSW_SM_I, %r0 86189206491SGuy Martin /* Return to userspace, set no error */ 86289206491SGuy Martin b lws_exit 86389206491SGuy Martin copy %r0, %r21 86489206491SGuy Martin 86589206491SGuy Martin22: 86689206491SGuy Martin /* Error occurred on load or store */ 86789206491SGuy Martin /* Free lock */ 868157e9afcSHelge Deller stw,ma %r20, 0(%sr2,%r20) 86989206491SGuy Martin ssm PSW_SM_I, %r0 87089206491SGuy Martin ldo 1(%r0),%r28 87189206491SGuy Martin b lws_exit 87289206491SGuy Martin ldo -EFAULT(%r0),%r21 /* set errno */ 87389206491SGuy Martin nop 87489206491SGuy Martin nop 87589206491SGuy Martin nop 87689206491SGuy Martin 87789206491SGuy Martin /* Exception table entries, for the load and store, return EFAULT. 87889206491SGuy Martin Each of the entries must be relocated. */ 87989206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page) 88089206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page) 88189206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page) 88289206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page) 88389206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page) 88489206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page) 88589206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page) 88689206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page) 88789206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page) 88889206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page) 88989206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page) 89089206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page) 89189206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page) 89289206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page) 89389206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page) 89489206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page) 89589206491SGuy Martin#ifndef CONFIG_64BIT 89689206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page) 89789206491SGuy Martin ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page) 89889206491SGuy Martin#endif 89989206491SGuy Martin 9001da177e4SLinus Torvalds /* Make sure nothing else is placed on this page */ 9011c593571SSam Ravnborg .align PAGE_SIZE 9028e9e9844SHelge DellerEND(linux_gateway_page) 9038e9e9844SHelge DellerENTRY(end_linux_gateway_page) 9041da177e4SLinus Torvalds 9051da177e4SLinus Torvalds /* Relocate symbols assuming linux_gateway_page is mapped 9061da177e4SLinus Torvalds to virtual address 0x0 */ 9078e9e9844SHelge Deller 9080b3d643fSHelge Deller#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page) 9091da177e4SLinus Torvalds 9101bcdd854SHelge Deller .section .rodata,"a" 9111bcdd854SHelge Deller 9126a45716aSHelge Deller .align 8 9131da177e4SLinus Torvalds /* Light-weight-syscall table */ 9141da177e4SLinus Torvalds /* Start of lws table. */ 9158e9e9844SHelge DellerENTRY(lws_table) 91689206491SGuy Martin LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic 32bit CAS */ 91789206491SGuy Martin LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic 32bit CAS */ 91889206491SGuy Martin LWS_ENTRY(compare_and_swap_2) /* 2 - ELF32 Atomic 64bit CAS */ 9198e9e9844SHelge DellerEND(lws_table) 9201da177e4SLinus Torvalds /* End of lws table */ 9211da177e4SLinus Torvalds 922106c9092SFiroz Khan#define __SYSCALL(nr, entry) ASM_ULONG_INSN entry 9236a45716aSHelge Deller .align 8 9248e9e9844SHelge DellerENTRY(sys_call_table) 925fc79168aSHelge Deller .export sys_call_table,data 926575afc4dSFiroz Khan#ifdef CONFIG_64BIT 927575afc4dSFiroz Khan#include <asm/syscall_table_c32.h> /* Compat syscalls */ 928575afc4dSFiroz Khan#else 929575afc4dSFiroz Khan#include <asm/syscall_table_32.h> /* 32-bit native syscalls */ 930575afc4dSFiroz Khan#endif 9318e9e9844SHelge DellerEND(sys_call_table) 9321da177e4SLinus Torvalds 933413059f2SGrant Grundler#ifdef CONFIG_64BIT 9346a45716aSHelge Deller .align 8 9358e9e9844SHelge DellerENTRY(sys_call_table64) 936575afc4dSFiroz Khan#include <asm/syscall_table_64.h> /* 64-bit native syscalls */ 9378e9e9844SHelge DellerEND(sys_call_table64) 9381da177e4SLinus Torvalds#endif 939575afc4dSFiroz Khan#undef __SYSCALL 9401da177e4SLinus Torvalds 9411da177e4SLinus Torvalds /* 9421da177e4SLinus Torvalds All light-weight-syscall atomic operations 9431da177e4SLinus Torvalds will use this set of locks 944c84c3a69SHelge Deller 945c84c3a69SHelge Deller NOTE: The lws_lock_start symbol must be 946c84c3a69SHelge Deller at least 16-byte aligned for safe use 947c84c3a69SHelge Deller with ldcw. 9481da177e4SLinus Torvalds */ 949dfcf753bSKyle McMartin .section .data 9506a45716aSHelge Deller .align L1_CACHE_BYTES 9518e9e9844SHelge DellerENTRY(lws_lock_start) 9521da177e4SLinus Torvalds /* lws locks */ 9531da177e4SLinus Torvalds .rept 16 9541da177e4SLinus Torvalds /* Keep locks aligned at 16-bytes */ 9551da177e4SLinus Torvalds .word 1 9561da177e4SLinus Torvalds .word 0 9571da177e4SLinus Torvalds .word 0 9581da177e4SLinus Torvalds .word 0 9591da177e4SLinus Torvalds .endr 9608e9e9844SHelge DellerEND(lws_lock_start) 9611da177e4SLinus Torvalds .previous 9621da177e4SLinus Torvalds 9631da177e4SLinus Torvalds.end 9641da177e4SLinus Torvalds 9651da177e4SLinus Torvalds 966