11da177e4SLinus Torvalds/* 21da177e4SLinus Torvalds * Linux/PA-RISC Project (http://www.parisc-linux.org/) 31da177e4SLinus Torvalds * 4dde39798SHelge Deller * System call entry code / Linux gateway page 53d0186bbSMatthew Wilcox * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org> 61da177e4SLinus Torvalds * Licensed under the GNU GPL. 71da177e4SLinus Torvalds * thanks to Philipp Rumpf, Mike Shaver and various others 81da177e4SLinus Torvalds * sorry about the wall, puffin.. 91da177e4SLinus Torvalds */ 101da177e4SLinus Torvalds 11dde39798SHelge Deller/* 12dde39798SHelge DellerHow does the Linux gateway page on PA-RISC work? 13dde39798SHelge Deller------------------------------------------------ 14dde39798SHelge DellerThe Linux gateway page on PA-RISC is "special". 15dde39798SHelge DellerIt actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc 16dde39798SHelge Dellerterminology it's Execute, promote to PL0) in the page map. So anything 17dde39798SHelge Dellerexecuting on this page executes with kernel level privilege (there's more to it 18dde39798SHelge Dellerthan that: to have this happen, you also have to use a branch with a ,gate 19dde39798SHelge Dellercompleter to activate the privilege promotion). The upshot is that everything 20dde39798SHelge Dellerthat runs on the gateway page runs at kernel privilege but with the current 21dde39798SHelge Delleruser process address space (although you have access to kernel space via %sr2). 22dde39798SHelge DellerFor the 0x100 syscall entry, we redo the space registers to point to the kernel 23dde39798SHelge Delleraddress space (preserving the user address space in %sr3), move to wide mode if 24dde39798SHelge Dellerrequired, save the user registers and branch into the kernel syscall entry 25dde39798SHelge Dellerpoint. For all the other functions, we execute at kernel privilege but don't 26dde39798SHelge Dellerflip address spaces. The basic upshot of this is that these code snippets are 27dde39798SHelge Dellerexecuted atomically (because the kernel can't be pre-empted) and they may 28dde39798SHelge Dellerperform architecturally forbidden (to PL3) operations (like setting control 29dde39798SHelge Dellerregisters). 30dde39798SHelge Deller*/ 31dde39798SHelge Deller 32dde39798SHelge Deller 330013a854SSam Ravnborg#include <asm/asm-offsets.h> 341da177e4SLinus Torvalds#include <asm/unistd.h> 351da177e4SLinus Torvalds#include <asm/errno.h> 361c593571SSam Ravnborg#include <asm/page.h> 371da177e4SLinus Torvalds#include <asm/psw.h> 381da177e4SLinus Torvalds#include <asm/thread_info.h> 391da177e4SLinus Torvalds#include <asm/assembly.h> 401da177e4SLinus Torvalds#include <asm/processor.h> 416a45716aSHelge Deller#include <asm/cache.h> 42*a0f4b787SHelge Deller#include <asm/spinlock_types.h> 431da177e4SLinus Torvalds 448e9e9844SHelge Deller#include <linux/linkage.h> 458e9e9844SHelge Deller 461da177e4SLinus Torvalds /* We fill the empty parts of the gateway page with 471da177e4SLinus Torvalds * something that will kill the kernel or a 481da177e4SLinus Torvalds * userspace application. 491da177e4SLinus Torvalds */ 501da177e4SLinus Torvalds#define KILL_INSN break 0,0 511da177e4SLinus Torvalds 521829dda0SHelge Deller .level PA_ASM_LEVEL 538e9e9844SHelge Deller 54d0585d74SJohn David Anglin .macro lws_pagefault_disable reg1,reg2 55d0585d74SJohn David Anglin mfctl %cr30, \reg2 56d0585d74SJohn David Anglin ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 57d0585d74SJohn David Anglin ldw 0(%sr2,\reg2), \reg1 58d0585d74SJohn David Anglin ldo 1(\reg1), \reg1 59d0585d74SJohn David Anglin stw \reg1, 0(%sr2,\reg2) 60d0585d74SJohn David Anglin .endm 61d0585d74SJohn David Anglin 62d0585d74SJohn David Anglin .macro lws_pagefault_enable reg1,reg2 63d0585d74SJohn David Anglin mfctl %cr30, \reg2 64d0585d74SJohn David Anglin ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 65d0585d74SJohn David Anglin ldw 0(%sr2,\reg2), \reg1 66d0585d74SJohn David Anglin ldo -1(\reg1), \reg1 67d0585d74SJohn David Anglin stw \reg1, 0(%sr2,\reg2) 68d0585d74SJohn David Anglin .endm 69d0585d74SJohn David Anglin 70*a0f4b787SHelge Deller /* raise exception if spinlock content is not zero or 71*a0f4b787SHelge Deller * __ARCH_SPIN_LOCK_UNLOCKED_VAL */ 72*a0f4b787SHelge Deller .macro spinlock_check spin_val,tmpreg 73*a0f4b787SHelge Deller#ifdef CONFIG_LIGHTWEIGHT_SPINLOCK_CHECK 74*a0f4b787SHelge Deller ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmpreg 75*a0f4b787SHelge Deller andcm,= \spin_val, \tmpreg, %r0 76*a0f4b787SHelge Deller .word SPINLOCK_BREAK_INSN 77*a0f4b787SHelge Deller#endif 78*a0f4b787SHelge Deller .endm 79*a0f4b787SHelge Deller 80dfcf753bSKyle McMartin .text 811da177e4SLinus Torvalds 821da177e4SLinus Torvalds .import syscall_exit,code 831da177e4SLinus Torvalds .import syscall_exit_rfi,code 841da177e4SLinus Torvalds 851da177e4SLinus Torvalds /* Linux gateway page is aliased to virtual page 0 in the kernel 861da177e4SLinus Torvalds * address space. Since it is a gateway page it cannot be 871da177e4SLinus Torvalds * dereferenced, so null pointers will still fault. We start 881da177e4SLinus Torvalds * the actual entry point at 0x100. We put break instructions 891da177e4SLinus Torvalds * at the beginning of the page to trap null indirect function 901da177e4SLinus Torvalds * pointers. 911da177e4SLinus Torvalds */ 921da177e4SLinus Torvalds 931c593571SSam Ravnborg .align PAGE_SIZE 948e9e9844SHelge DellerENTRY(linux_gateway_page) 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */ 971da177e4SLinus Torvalds .rept 44 981da177e4SLinus Torvalds KILL_INSN 991da177e4SLinus Torvalds .endr 1001da177e4SLinus Torvalds 101f4c0346cSJohn David Anglin /* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */ 1021da177e4SLinus Torvalds /* Light-weight-syscall entry must always be located at 0xb0 */ 1031da177e4SLinus Torvalds /* WARNING: Keep this number updated with table size changes */ 10472c3dd82SJohn David Anglin#define __NR_lws_entries (5) 1051da177e4SLinus Torvalds 1061da177e4SLinus Torvaldslws_entry: 107f4c0346cSJohn David Anglin gate lws_start, %r0 /* increase privilege */ 108fdc9e4e0SHelge Deller depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ 1091da177e4SLinus Torvalds 110f4c0346cSJohn David Anglin /* Fill from 0xb8 to 0xe0 */ 111f4c0346cSJohn David Anglin .rept 10 1121da177e4SLinus Torvalds KILL_INSN 1131da177e4SLinus Torvalds .endr 1141da177e4SLinus Torvalds 1151da177e4SLinus Torvalds /* This function MUST be located at 0xe0 for glibc's threading 1161da177e4SLinus Torvalds mechanism to work. DO NOT MOVE THIS CODE EVER! */ 1171da177e4SLinus Torvaldsset_thread_pointer: 1181da177e4SLinus Torvalds gate .+8, %r0 /* increase privilege */ 119fdc9e4e0SHelge Deller depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ 1201da177e4SLinus Torvalds be 0(%sr7,%r31) /* return to user space */ 1211da177e4SLinus Torvalds mtctl %r26, %cr27 /* move arg0 to the control register */ 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds /* Increase the chance of trapping if random jumps occur to this 1241da177e4SLinus Torvalds address, fill from 0xf0 to 0x100 */ 1251da177e4SLinus Torvalds .rept 4 1261da177e4SLinus Torvalds KILL_INSN 1271da177e4SLinus Torvalds .endr 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds/* This address must remain fixed at 0x100 for glibc's syscalls to work */ 1306a6e2a14SHelge Deller .align LINUX_GATEWAY_ADDR 1311da177e4SLinus Torvaldslinux_gateway_entry: 1321da177e4SLinus Torvalds gate .+8, %r0 /* become privileged */ 1331da177e4SLinus Torvalds mtsp %r0,%sr4 /* get kernel space into sr4 */ 1341da177e4SLinus Torvalds mtsp %r0,%sr5 /* get kernel space into sr5 */ 1351da177e4SLinus Torvalds mtsp %r0,%sr6 /* get kernel space into sr6 */ 1361da177e4SLinus Torvalds 137413059f2SGrant Grundler#ifdef CONFIG_64BIT 13854c770daSHelge Deller /* Store W bit on entry to the syscall in case it's a wide userland 13954c770daSHelge Deller * process. */ 1401da177e4SLinus Torvalds ssm PSW_SM_W, %r1 1411da177e4SLinus Torvalds extrd,u %r1,PSW_W_BIT,1,%r1 1421da177e4SLinus Torvalds /* sp must be aligned on 4, so deposit the W bit setting into 1431da177e4SLinus Torvalds * the bottom of sp temporarily */ 1441da177e4SLinus Torvalds or,ev %r1,%r30,%r30 1451da177e4SLinus Torvalds b,n 1f 1461da177e4SLinus Torvalds /* The top halves of argument registers must be cleared on syscall 1471da177e4SLinus Torvalds * entry from narrow executable. 1481da177e4SLinus Torvalds */ 1491da177e4SLinus Torvalds depdi 0, 31, 32, %r26 1501da177e4SLinus Torvalds depdi 0, 31, 32, %r25 1511da177e4SLinus Torvalds depdi 0, 31, 32, %r24 1521da177e4SLinus Torvalds depdi 0, 31, 32, %r23 1531da177e4SLinus Torvalds depdi 0, 31, 32, %r22 1541da177e4SLinus Torvalds depdi 0, 31, 32, %r21 1551da177e4SLinus Torvalds1: 1561da177e4SLinus Torvalds#endif 1576ed51832SJohn David Anglin 1586ed51832SJohn David Anglin /* We use a rsm/ssm pair to prevent sr3 from being clobbered 1596ed51832SJohn David Anglin * by external interrupts. 1606ed51832SJohn David Anglin */ 1616ed51832SJohn David Anglin mfsp %sr7,%r1 /* save user sr7 */ 1626ed51832SJohn David Anglin rsm PSW_SM_I, %r0 /* disable interrupts */ 1636ed51832SJohn David Anglin mtsp %r1,%sr3 /* and store it in sr3 */ 1646ed51832SJohn David Anglin 1651da177e4SLinus Torvalds mfctl %cr30,%r1 1661da177e4SLinus Torvalds xor %r1,%r30,%r30 /* ye olde xor trick */ 1671da177e4SLinus Torvalds xor %r1,%r30,%r1 1681da177e4SLinus Torvalds xor %r1,%r30,%r30 1691da177e4SLinus Torvalds 1702214c0e7SHelge Deller LDREG TASK_STACK(%r30),%r30 /* set up kernel stack */ 1712214c0e7SHelge Deller ldo FRAME_SIZE(%r30),%r30 1721da177e4SLinus Torvalds /* N.B.: It is critical that we don't set sr7 to 0 until r30 1731da177e4SLinus Torvalds * contains a valid kernel stack pointer. It is also 1741da177e4SLinus Torvalds * critical that we don't start using the kernel stack 1751da177e4SLinus Torvalds * until after sr7 has been set to 0. 1761da177e4SLinus Torvalds */ 1771da177e4SLinus Torvalds 1781da177e4SLinus Torvalds mtsp %r0,%sr7 /* get kernel space into sr7 */ 1796ed51832SJohn David Anglin ssm PSW_SM_I, %r0 /* enable interrupts */ 1801da177e4SLinus Torvalds STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ 1811da177e4SLinus Torvalds mfctl %cr30,%r1 /* get task ptr in %r1 */ 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds /* Save some registers for sigcontext and potential task 1841da177e4SLinus Torvalds switch (see entry.S for the details of which ones are 1851da177e4SLinus Torvalds saved/restored). TASK_PT_PSW is zeroed so we can see whether 1861da177e4SLinus Torvalds a process is on a syscall or not. For an interrupt the real 1871da177e4SLinus Torvalds PSW value is stored. This is needed for gdb and sys_ptrace. */ 1881da177e4SLinus Torvalds STREG %r0, TASK_PT_PSW(%r1) 1891da177e4SLinus Torvalds STREG %r2, TASK_PT_GR2(%r1) /* preserve rp */ 1901da177e4SLinus Torvalds STREG %r19, TASK_PT_GR19(%r1) 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds LDREGM -FRAME_SIZE(%r30), %r2 /* get users sp back */ 193413059f2SGrant Grundler#ifdef CONFIG_64BIT 1941da177e4SLinus Torvalds extrd,u %r2,63,1,%r19 /* W hidden in bottom bit */ 1951da177e4SLinus Torvalds#if 0 1961da177e4SLinus Torvalds xor %r19,%r2,%r2 /* clear bottom bit */ 1971da177e4SLinus Torvalds depd,z %r19,1,1,%r19 1981da177e4SLinus Torvalds std %r19,TASK_PT_PSW(%r1) 1991da177e4SLinus Torvalds#endif 2001da177e4SLinus Torvalds#endif 2011da177e4SLinus Torvalds STREG %r2, TASK_PT_GR30(%r1) /* ... and save it */ 2021da177e4SLinus Torvalds 203aa0eecb0SCarlos O'Donell STREG %r20, TASK_PT_GR20(%r1) /* Syscall number */ 2041da177e4SLinus Torvalds STREG %r21, TASK_PT_GR21(%r1) 2051da177e4SLinus Torvalds STREG %r22, TASK_PT_GR22(%r1) 2061da177e4SLinus Torvalds STREG %r23, TASK_PT_GR23(%r1) /* 4th argument */ 2071da177e4SLinus Torvalds STREG %r24, TASK_PT_GR24(%r1) /* 3rd argument */ 2081da177e4SLinus Torvalds STREG %r25, TASK_PT_GR25(%r1) /* 2nd argument */ 2091da177e4SLinus Torvalds STREG %r26, TASK_PT_GR26(%r1) /* 1st argument */ 2101da177e4SLinus Torvalds STREG %r27, TASK_PT_GR27(%r1) /* user dp */ 2111da177e4SLinus Torvalds STREG %r28, TASK_PT_GR28(%r1) /* return value 0 */ 21200df111eSAl Viro STREG %r0, TASK_PT_ORIG_R28(%r1) /* don't prohibit restarts */ 2131da177e4SLinus Torvalds STREG %r29, TASK_PT_GR29(%r1) /* return value 1 */ 2141da177e4SLinus Torvalds STREG %r31, TASK_PT_GR31(%r1) /* preserve syscall return ptr */ 2151da177e4SLinus Torvalds 2161da177e4SLinus Torvalds ldo TASK_PT_FR0(%r1), %r27 /* save fpregs from the kernel */ 2171da177e4SLinus Torvalds save_fp %r27 /* or potential task switch */ 2181da177e4SLinus Torvalds 2191da177e4SLinus Torvalds mfctl %cr11, %r27 /* i.e. SAR */ 2201da177e4SLinus Torvalds STREG %r27, TASK_PT_SAR(%r1) 2211da177e4SLinus Torvalds 2221da177e4SLinus Torvalds loadgp 2231da177e4SLinus Torvalds 224413059f2SGrant Grundler#ifdef CONFIG_64BIT 2251da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 2261da177e4SLinus Torvalds copy %r19,%r2 /* W bit back to r2 */ 2271da177e4SLinus Torvalds#else 2281da177e4SLinus Torvalds /* no need to save these on stack in wide mode because the first 8 2291da177e4SLinus Torvalds * args are passed in registers */ 2301da177e4SLinus Torvalds stw %r22, -52(%r30) /* 5th argument */ 2311da177e4SLinus Torvalds stw %r21, -56(%r30) /* 6th argument */ 2321da177e4SLinus Torvalds#endif 2331da177e4SLinus Torvalds 2341da177e4SLinus Torvalds /* Are we being ptraced? */ 2351da177e4SLinus Torvalds mfctl %cr30, %r1 2362214c0e7SHelge Deller LDREG TASK_TI_FLAGS(%r1),%r1 23764482bd8SAl Viro ldi _TIF_SYSCALL_TRACE_MASK, %r19 23864482bd8SAl Viro and,COND(=) %r1, %r19, %r0 23964482bd8SAl Viro b,n .Ltracesys 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds /* Note! We cannot use the syscall table that is mapped 2421da177e4SLinus Torvalds nearby since the gateway page is mapped execute-only. */ 2431da177e4SLinus Torvalds 244413059f2SGrant Grundler#ifdef CONFIG_64BIT 2451da177e4SLinus Torvalds ldil L%sys_call_table, %r1 2461da177e4SLinus Torvalds or,= %r2,%r2,%r2 2471da177e4SLinus Torvalds addil L%(sys_call_table64-sys_call_table), %r1 2481da177e4SLinus Torvalds ldo R%sys_call_table(%r1), %r19 2491da177e4SLinus Torvalds or,= %r2,%r2,%r2 2501da177e4SLinus Torvalds ldo R%sys_call_table64(%r1), %r19 2511da177e4SLinus Torvalds#else 252b6fc0cccSHelge Deller load32 sys_call_table, %r19 2531da177e4SLinus Torvalds#endif 2543bb457afSKyle McMartin comiclr,>> __NR_Linux_syscalls, %r20, %r0 2551da177e4SLinus Torvalds b,n .Lsyscall_nosys 2561da177e4SLinus Torvalds 2571da177e4SLinus Torvalds LDREGX %r20(%r19), %r19 2581da177e4SLinus Torvalds 2591da177e4SLinus Torvalds /* If this is a sys_rt_sigreturn call, and the signal was received 2601da177e4SLinus Torvalds * when not in_syscall, then we want to return via syscall_exit_rfi, 2611da177e4SLinus Torvalds * not syscall_exit. Signal no. in r20, in_syscall in r25 (see 2621da177e4SLinus Torvalds * trampoline code in signal.c). 2631da177e4SLinus Torvalds */ 2641da177e4SLinus Torvalds ldi __NR_rt_sigreturn,%r2 2651da177e4SLinus Torvalds comb,= %r2,%r20,.Lrt_sigreturn 2661da177e4SLinus Torvalds.Lin_syscall: 2671da177e4SLinus Torvalds ldil L%syscall_exit,%r2 2681da177e4SLinus Torvalds be 0(%sr7,%r19) 2691da177e4SLinus Torvalds ldo R%syscall_exit(%r2),%r2 2701da177e4SLinus Torvalds.Lrt_sigreturn: 2711da177e4SLinus Torvalds comib,<> 0,%r25,.Lin_syscall 2721da177e4SLinus Torvalds ldil L%syscall_exit_rfi,%r2 2731da177e4SLinus Torvalds be 0(%sr7,%r19) 2741da177e4SLinus Torvalds ldo R%syscall_exit_rfi(%r2),%r2 2751da177e4SLinus Torvalds 2761da177e4SLinus Torvalds /* Note! Because we are not running where we were linked, any 2771da177e4SLinus Torvalds calls to functions external to this file must be indirect. To 2781da177e4SLinus Torvalds be safe, we apply the opposite rule to functions within this 2791da177e4SLinus Torvalds file, with local labels given to them to ensure correctness. */ 2801da177e4SLinus Torvalds 2811da177e4SLinus Torvalds.Lsyscall_nosys: 2821da177e4SLinus Torvaldssyscall_nosys: 2831da177e4SLinus Torvalds ldil L%syscall_exit,%r1 2841da177e4SLinus Torvalds be R%syscall_exit(%sr7,%r1) 2851da177e4SLinus Torvalds ldo -ENOSYS(%r0),%r28 /* set errno */ 2861da177e4SLinus Torvalds 2871da177e4SLinus Torvalds 2881da177e4SLinus Torvalds/* Warning! This trace code is a virtual duplicate of the code above so be 2891da177e4SLinus Torvalds * sure to maintain both! */ 2901da177e4SLinus Torvalds.Ltracesys: 2911da177e4SLinus Torvaldstracesys: 2921da177e4SLinus Torvalds /* Need to save more registers so the debugger can see where we 2931da177e4SLinus Torvalds * are. This saves only the lower 8 bits of PSW, so that the C 2941da177e4SLinus Torvalds * bit is still clear on syscalls, and the D bit is set if this 2951da177e4SLinus Torvalds * full register save path has been executed. We check the D 2961da177e4SLinus Torvalds * bit on syscall_return_rfi to determine which registers to 2971da177e4SLinus Torvalds * restore. An interrupt results in a full PSW saved with the 2981da177e4SLinus Torvalds * C bit set, a non-straced syscall entry results in C and D clear 2991da177e4SLinus Torvalds * in the saved PSW. 3001da177e4SLinus Torvalds */ 3012214c0e7SHelge Deller mfctl %cr30,%r1 /* get task ptr */ 3021da177e4SLinus Torvalds ssm 0,%r2 3031da177e4SLinus Torvalds STREG %r2,TASK_PT_PSW(%r1) /* Lower 8 bits only!! */ 3041da177e4SLinus Torvalds mfsp %sr0,%r2 3051da177e4SLinus Torvalds STREG %r2,TASK_PT_SR0(%r1) 3061da177e4SLinus Torvalds mfsp %sr1,%r2 3071da177e4SLinus Torvalds STREG %r2,TASK_PT_SR1(%r1) 3081da177e4SLinus Torvalds mfsp %sr2,%r2 3091da177e4SLinus Torvalds STREG %r2,TASK_PT_SR2(%r1) 3101da177e4SLinus Torvalds mfsp %sr3,%r2 3111da177e4SLinus Torvalds STREG %r2,TASK_PT_SR3(%r1) 3121da177e4SLinus Torvalds STREG %r2,TASK_PT_SR4(%r1) 3131da177e4SLinus Torvalds STREG %r2,TASK_PT_SR5(%r1) 3141da177e4SLinus Torvalds STREG %r2,TASK_PT_SR6(%r1) 3151da177e4SLinus Torvalds STREG %r2,TASK_PT_SR7(%r1) 3161da177e4SLinus Torvalds STREG %r2,TASK_PT_IASQ0(%r1) 3171da177e4SLinus Torvalds STREG %r2,TASK_PT_IASQ1(%r1) 3181da177e4SLinus Torvalds LDREG TASK_PT_GR31(%r1),%r2 3191da177e4SLinus Torvalds STREG %r2,TASK_PT_IAOQ0(%r1) 3201da177e4SLinus Torvalds ldo 4(%r2),%r2 3211da177e4SLinus Torvalds STREG %r2,TASK_PT_IAOQ1(%r1) 3221da177e4SLinus Torvalds ldo TASK_REGS(%r1),%r2 3231da177e4SLinus Torvalds /* reg_save %r2 */ 3241da177e4SLinus Torvalds STREG %r3,PT_GR3(%r2) 3251da177e4SLinus Torvalds STREG %r4,PT_GR4(%r2) 3261da177e4SLinus Torvalds STREG %r5,PT_GR5(%r2) 3271da177e4SLinus Torvalds STREG %r6,PT_GR6(%r2) 3281da177e4SLinus Torvalds STREG %r7,PT_GR7(%r2) 3291da177e4SLinus Torvalds STREG %r8,PT_GR8(%r2) 3301da177e4SLinus Torvalds STREG %r9,PT_GR9(%r2) 3311da177e4SLinus Torvalds STREG %r10,PT_GR10(%r2) 3321da177e4SLinus Torvalds STREG %r11,PT_GR11(%r2) 3331da177e4SLinus Torvalds STREG %r12,PT_GR12(%r2) 3341da177e4SLinus Torvalds STREG %r13,PT_GR13(%r2) 3351da177e4SLinus Torvalds STREG %r14,PT_GR14(%r2) 3361da177e4SLinus Torvalds STREG %r15,PT_GR15(%r2) 3371da177e4SLinus Torvalds STREG %r16,PT_GR16(%r2) 3381da177e4SLinus Torvalds STREG %r17,PT_GR17(%r2) 3391da177e4SLinus Torvalds STREG %r18,PT_GR18(%r2) 3401da177e4SLinus Torvalds /* Finished saving things for the debugger */ 3411da177e4SLinus Torvalds 3422798af1aSKyle McMartin copy %r2,%r26 3432798af1aSKyle McMartin ldil L%do_syscall_trace_enter,%r1 3441da177e4SLinus Torvalds ldil L%tracesys_next,%r2 3452798af1aSKyle McMartin be R%do_syscall_trace_enter(%sr7,%r1) 3461da177e4SLinus Torvalds ldo R%tracesys_next(%r2),%r2 3471da177e4SLinus Torvalds 3481da177e4SLinus Torvaldstracesys_next: 3492798af1aSKyle McMartin /* do_syscall_trace_enter either returned the syscallno, or -1L, 3502798af1aSKyle McMartin * so we skip restoring the PT_GR20 below, since we pulled it from 3512798af1aSKyle McMartin * task->thread.regs.gr[20] above. 3522798af1aSKyle McMartin */ 3532798af1aSKyle McMartin copy %ret0,%r20 3541da177e4SLinus Torvalds 3552214c0e7SHelge Deller mfctl %cr30,%r1 /* get task ptr */ 356910cd32eSHelge Deller LDREG TASK_PT_GR28(%r1), %r28 /* Restore return value */ 3571da177e4SLinus Torvalds LDREG TASK_PT_GR26(%r1), %r26 /* Restore the users args */ 3581da177e4SLinus Torvalds LDREG TASK_PT_GR25(%r1), %r25 3591da177e4SLinus Torvalds LDREG TASK_PT_GR24(%r1), %r24 3601da177e4SLinus Torvalds LDREG TASK_PT_GR23(%r1), %r23 3611da177e4SLinus Torvalds LDREG TASK_PT_GR22(%r1), %r22 3621da177e4SLinus Torvalds LDREG TASK_PT_GR21(%r1), %r21 36352ab532eSAl Viro#ifdef CONFIG_64BIT 3641da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 36552ab532eSAl Viro#else 36652ab532eSAl Viro stw %r22, -52(%r30) /* 5th argument */ 36752ab532eSAl Viro stw %r21, -56(%r30) /* 6th argument */ 3681da177e4SLinus Torvalds#endif 3691da177e4SLinus Torvalds 370910cd32eSHelge Deller cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */ 371f0b22d1bSDmitry V. Levin comiclr,>> __NR_Linux_syscalls, %r20, %r0 37298e8b6c9SHelge Deller b,n .Ltracesys_nosys 3731da177e4SLinus Torvalds 374b6fc0cccSHelge Deller /* Note! We cannot use the syscall table that is mapped 375b6fc0cccSHelge Deller nearby since the gateway page is mapped execute-only. */ 376b6fc0cccSHelge Deller 377b6fc0cccSHelge Deller#ifdef CONFIG_64BIT 378b6fc0cccSHelge Deller LDREG TASK_PT_GR30(%r1), %r19 /* get users sp back */ 379b6fc0cccSHelge Deller extrd,u %r19,63,1,%r2 /* W hidden in bottom bit */ 380b6fc0cccSHelge Deller 381b6fc0cccSHelge Deller ldil L%sys_call_table, %r1 382b6fc0cccSHelge Deller or,= %r2,%r2,%r2 383b6fc0cccSHelge Deller addil L%(sys_call_table64-sys_call_table), %r1 384b6fc0cccSHelge Deller ldo R%sys_call_table(%r1), %r19 385b6fc0cccSHelge Deller or,= %r2,%r2,%r2 386b6fc0cccSHelge Deller ldo R%sys_call_table64(%r1), %r19 387b6fc0cccSHelge Deller#else 388b6fc0cccSHelge Deller load32 sys_call_table, %r19 389b6fc0cccSHelge Deller#endif 390b6fc0cccSHelge Deller 3911da177e4SLinus Torvalds LDREGX %r20(%r19), %r19 3921da177e4SLinus Torvalds 3931da177e4SLinus Torvalds /* If this is a sys_rt_sigreturn call, and the signal was received 3941da177e4SLinus Torvalds * when not in_syscall, then we want to return via syscall_exit_rfi, 3951da177e4SLinus Torvalds * not syscall_exit. Signal no. in r20, in_syscall in r25 (see 3961da177e4SLinus Torvalds * trampoline code in signal.c). 3971da177e4SLinus Torvalds */ 3981da177e4SLinus Torvalds ldi __NR_rt_sigreturn,%r2 3991da177e4SLinus Torvalds comb,= %r2,%r20,.Ltrace_rt_sigreturn 4001da177e4SLinus Torvalds.Ltrace_in_syscall: 4011da177e4SLinus Torvalds ldil L%tracesys_exit,%r2 4021da177e4SLinus Torvalds be 0(%sr7,%r19) 4031da177e4SLinus Torvalds ldo R%tracesys_exit(%r2),%r2 4041da177e4SLinus Torvalds 40598e8b6c9SHelge Deller.Ltracesys_nosys: 40698e8b6c9SHelge Deller ldo -ENOSYS(%r0),%r28 /* set errno */ 40798e8b6c9SHelge Deller 4081da177e4SLinus Torvalds /* Do *not* call this function on the gateway page, because it 4091da177e4SLinus Torvalds makes a direct call to syscall_trace. */ 4101da177e4SLinus Torvalds 4111da177e4SLinus Torvaldstracesys_exit: 4122214c0e7SHelge Deller mfctl %cr30,%r1 /* get task ptr */ 413413059f2SGrant Grundler#ifdef CONFIG_64BIT 4141da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 4151da177e4SLinus Torvalds#endif 4162798af1aSKyle McMartin ldo TASK_REGS(%r1),%r26 417337685e5SHelge Deller BL do_syscall_trace_exit,%r2 4181da177e4SLinus Torvalds STREG %r28,TASK_PT_GR28(%r1) /* save return value now */ 4192214c0e7SHelge Deller mfctl %cr30,%r1 /* get task ptr */ 4201da177e4SLinus Torvalds LDREG TASK_PT_GR28(%r1), %r28 /* Restore return val. */ 4211da177e4SLinus Torvalds 4221da177e4SLinus Torvalds ldil L%syscall_exit,%r1 4231da177e4SLinus Torvalds be,n R%syscall_exit(%sr7,%r1) 4241da177e4SLinus Torvalds 4251da177e4SLinus Torvalds.Ltrace_rt_sigreturn: 4261da177e4SLinus Torvalds comib,<> 0,%r25,.Ltrace_in_syscall 4271da177e4SLinus Torvalds ldil L%tracesys_sigexit,%r2 4281da177e4SLinus Torvalds be 0(%sr7,%r19) 4291da177e4SLinus Torvalds ldo R%tracesys_sigexit(%r2),%r2 4301da177e4SLinus Torvalds 4311da177e4SLinus Torvaldstracesys_sigexit: 4322214c0e7SHelge Deller mfctl %cr30,%r1 /* get task ptr */ 433413059f2SGrant Grundler#ifdef CONFIG_64BIT 4341da177e4SLinus Torvalds ldo -16(%r30),%r29 /* Reference param save area */ 4351da177e4SLinus Torvalds#endif 436337685e5SHelge Deller BL do_syscall_trace_exit,%r2 4372798af1aSKyle McMartin ldo TASK_REGS(%r1),%r26 4381da177e4SLinus Torvalds 4391da177e4SLinus Torvalds ldil L%syscall_exit_rfi,%r1 4401da177e4SLinus Torvalds be,n R%syscall_exit_rfi(%sr7,%r1) 4411da177e4SLinus Torvalds 4421da177e4SLinus Torvalds 4431da177e4SLinus Torvalds /********************************************************* 444c84c3a69SHelge Deller 32/64-bit Light-Weight-Syscall ABI 4451da177e4SLinus Torvalds 446c84c3a69SHelge Deller * - Indicates a hint for userspace inline asm 447c84c3a69SHelge Deller implementations. 4481da177e4SLinus Torvalds 449c84c3a69SHelge Deller Syscall number (caller-saves) 450c84c3a69SHelge Deller - %r20 451c84c3a69SHelge Deller * In asm clobber. 4521da177e4SLinus Torvalds 453c84c3a69SHelge Deller Argument registers (caller-saves) 454c84c3a69SHelge Deller - %r26, %r25, %r24, %r23, %r22 455c84c3a69SHelge Deller * In asm input. 456c84c3a69SHelge Deller 457c84c3a69SHelge Deller Return registers (caller-saves) 458c84c3a69SHelge Deller - %r28 (return), %r21 (errno) 459c84c3a69SHelge Deller * In asm output. 460c84c3a69SHelge Deller 461c84c3a69SHelge Deller Caller-saves registers 462c84c3a69SHelge Deller - %r1, %r27, %r29 463c84c3a69SHelge Deller - %r2 (return pointer) 464c84c3a69SHelge Deller - %r31 (ble link register) 465c84c3a69SHelge Deller * In asm clobber. 466c84c3a69SHelge Deller 467c84c3a69SHelge Deller Callee-saves registers 468c84c3a69SHelge Deller - %r3-%r18 469c84c3a69SHelge Deller - %r30 (stack pointer) 470c84c3a69SHelge Deller * Not in asm clobber. 471c84c3a69SHelge Deller 472c84c3a69SHelge Deller If userspace is 32-bit: 473c84c3a69SHelge Deller Callee-saves registers 474c84c3a69SHelge Deller - %r19 (32-bit PIC register) 475c84c3a69SHelge Deller 476c84c3a69SHelge Deller Differences from 32-bit calling convention: 477c84c3a69SHelge Deller - Syscall number in %r20 478c84c3a69SHelge Deller - Additional argument register %r22 (arg4) 479c84c3a69SHelge Deller - Callee-saves %r19. 480c84c3a69SHelge Deller 481c84c3a69SHelge Deller If userspace is 64-bit: 482c84c3a69SHelge Deller Callee-saves registers 483c84c3a69SHelge Deller - %r27 (64-bit PIC register) 484c84c3a69SHelge Deller 485c84c3a69SHelge Deller Differences from 64-bit calling convention: 486c84c3a69SHelge Deller - Syscall number in %r20 487c84c3a69SHelge Deller - Additional argument register %r22 (arg4) 488c84c3a69SHelge Deller - Callee-saves %r27. 4891da177e4SLinus Torvalds 4901da177e4SLinus Torvalds Error codes returned by entry path: 4911da177e4SLinus Torvalds 4921da177e4SLinus Torvalds ENOSYS - r20 was an invalid LWS number. 4931da177e4SLinus Torvalds 4941da177e4SLinus Torvalds *********************************************************/ 4951da177e4SLinus Torvaldslws_start: 4961da177e4SLinus Torvalds 497413059f2SGrant Grundler#ifdef CONFIG_64BIT 4981da177e4SLinus Torvalds ssm PSW_SM_W, %r1 4991da177e4SLinus Torvalds extrd,u %r1,PSW_W_BIT,1,%r1 5001da177e4SLinus Torvalds /* sp must be aligned on 4, so deposit the W bit setting into 5011da177e4SLinus Torvalds * the bottom of sp temporarily */ 5028f66fce0SJohn David Anglin or,od %r1,%r30,%r30 5031da177e4SLinus Torvalds 50454c770daSHelge Deller /* Clip LWS number to a 32-bit value for 32-bit processes */ 5051da177e4SLinus Torvalds depdi 0, 31, 32, %r20 5061da177e4SLinus Torvalds#endif 5071da177e4SLinus Torvalds 5081da177e4SLinus Torvalds /* Is the lws entry number valid? */ 509f4c0346cSJohn David Anglin comiclr,>> __NR_lws_entries, %r20, %r0 5101da177e4SLinus Torvalds b,n lws_exit_nosys 5111da177e4SLinus Torvalds 5121da177e4SLinus Torvalds /* Load table start */ 5131da177e4SLinus Torvalds ldil L%lws_table, %r1 5141da177e4SLinus Torvalds ldo R%lws_table(%r1), %r28 /* Scratch use of r28 */ 5151da177e4SLinus Torvalds LDREGX %r20(%sr2,r28), %r21 /* Scratch use of r21 */ 5161da177e4SLinus Torvalds 5171da177e4SLinus Torvalds /* Jump to lws, lws table pointers already relocated */ 5181da177e4SLinus Torvalds be,n 0(%sr2,%r21) 5191da177e4SLinus Torvalds 520d0585d74SJohn David Anglinlws_exit_noerror: 521d0585d74SJohn David Anglin lws_pagefault_enable %r1,%r21 522*a0f4b787SHelge Deller ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, %r21 523*a0f4b787SHelge Deller stw,ma %r21, 0(%sr2,%r20) 524d0585d74SJohn David Anglin ssm PSW_SM_I, %r0 525d0585d74SJohn David Anglin b lws_exit 526d0585d74SJohn David Anglin copy %r0, %r21 527d0585d74SJohn David Anglin 528d0585d74SJohn David Anglinlws_wouldblock: 529d0585d74SJohn David Anglin ssm PSW_SM_I, %r0 530d0585d74SJohn David Anglin ldo 2(%r0), %r28 531d0585d74SJohn David Anglin b lws_exit 532d0585d74SJohn David Anglin ldo -EAGAIN(%r0), %r21 533d0585d74SJohn David Anglin 534d0585d74SJohn David Anglinlws_pagefault: 535d0585d74SJohn David Anglin lws_pagefault_enable %r1,%r21 536*a0f4b787SHelge Deller ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, %r21 537*a0f4b787SHelge Deller stw,ma %r21, 0(%sr2,%r20) 538d0585d74SJohn David Anglin ssm PSW_SM_I, %r0 539d0585d74SJohn David Anglin ldo 3(%r0),%r28 540d0585d74SJohn David Anglin b lws_exit 541d0585d74SJohn David Anglin ldo -EAGAIN(%r0),%r21 542d0585d74SJohn David Anglin 543d0585d74SJohn David Anglinlws_fault: 544d0585d74SJohn David Anglin ldo 1(%r0),%r28 545d0585d74SJohn David Anglin b lws_exit 546d0585d74SJohn David Anglin ldo -EFAULT(%r0),%r21 547d0585d74SJohn David Anglin 5481da177e4SLinus Torvaldslws_exit_nosys: 549d0585d74SJohn David Anglin ldo -ENOSYS(%r0),%r21 5501da177e4SLinus Torvalds /* Fall through: Return to userspace */ 5511da177e4SLinus Torvalds 5521da177e4SLinus Torvaldslws_exit: 553413059f2SGrant Grundler#ifdef CONFIG_64BIT 5541da177e4SLinus Torvalds /* decide whether to reset the wide mode bit 5551da177e4SLinus Torvalds * 5561da177e4SLinus Torvalds * For a syscall, the W bit is stored in the lowest bit 5571da177e4SLinus Torvalds * of sp. Extract it and reset W if it is zero */ 5581da177e4SLinus Torvalds extrd,u,*<> %r30,63,1,%r1 5591da177e4SLinus Torvalds rsm PSW_SM_W, %r0 5601da177e4SLinus Torvalds /* now reset the lowest bit of sp if it was set */ 5611da177e4SLinus Torvalds xor %r30,%r1,%r30 5621da177e4SLinus Torvalds#endif 563f4c0346cSJohn David Anglin be,n 0(%sr7, %r31) 5641da177e4SLinus Torvalds 5651da177e4SLinus Torvalds 5661da177e4SLinus Torvalds 5671da177e4SLinus Torvalds /*************************************************** 56889206491SGuy Martin Implementing 32bit CAS as an atomic operation: 5691da177e4SLinus Torvalds 5701da177e4SLinus Torvalds %r26 - Address to examine 5711da177e4SLinus Torvalds %r25 - Old value to check (old) 5721da177e4SLinus Torvalds %r24 - New value to set (new) 5731da177e4SLinus Torvalds %r28 - Return prev through this register. 5741da177e4SLinus Torvalds %r21 - Kernel error code 5751da177e4SLinus Torvalds 576d0585d74SJohn David Anglin %r21 returns the following error codes: 5771da177e4SLinus Torvalds EAGAIN - CAS is busy, ldcw failed, try again. 5781da177e4SLinus Torvalds EFAULT - Read or write failed. 5791da177e4SLinus Torvalds 580d0585d74SJohn David Anglin If EAGAIN is returned, %r28 indicates the busy reason: 581d0585d74SJohn David Anglin r28 == 1 - CAS is busy. lock contended. 582d0585d74SJohn David Anglin r28 == 2 - CAS is busy. ldcw failed. 583d0585d74SJohn David Anglin r28 == 3 - CAS is busy. page fault. 5841da177e4SLinus Torvalds 5851da177e4SLinus Torvalds Scratch: r20, r28, r1 5861da177e4SLinus Torvalds 5871da177e4SLinus Torvalds ****************************************************/ 5881da177e4SLinus Torvalds 5891da177e4SLinus Torvalds /* ELF64 Process entry path */ 5901da177e4SLinus Torvaldslws_compare_and_swap64: 591413059f2SGrant Grundler#ifdef CONFIG_64BIT 5921da177e4SLinus Torvalds b,n lws_compare_and_swap 5931da177e4SLinus Torvalds#else 5941da177e4SLinus Torvalds /* If we are not a 64-bit kernel, then we don't 595c84c3a69SHelge Deller * have 64-bit input registers, and calling 596c84c3a69SHelge Deller * the 64-bit LWS CAS returns ENOSYS. 5971da177e4SLinus Torvalds */ 5981da177e4SLinus Torvalds b,n lws_exit_nosys 5991da177e4SLinus Torvalds#endif 6001da177e4SLinus Torvalds 601d0585d74SJohn David Anglin /* ELF32/ELF64 Process entry path */ 6021da177e4SLinus Torvaldslws_compare_and_swap32: 603413059f2SGrant Grundler#ifdef CONFIG_64BIT 604d0585d74SJohn David Anglin /* Wide mode user process? */ 605d0585d74SJohn David Anglin bb,<,n %sp, 31, lws_compare_and_swap 606d0585d74SJohn David Anglin 607d0585d74SJohn David Anglin /* Clip all the input registers for 32-bit processes */ 6081da177e4SLinus Torvalds depdi 0, 31, 32, %r26 6091da177e4SLinus Torvalds depdi 0, 31, 32, %r25 6101da177e4SLinus Torvalds depdi 0, 31, 32, %r24 6111da177e4SLinus Torvalds#endif 6121da177e4SLinus Torvalds 6131da177e4SLinus Torvaldslws_compare_and_swap: 614d0585d74SJohn David Anglin /* Trigger memory reference interruptions without writing to memory */ 615d0585d74SJohn David Anglin1: ldw 0(%r26), %r28 616d0585d74SJohn David Anglin2: stbys,e %r0, 0(%r26) 617d0585d74SJohn David Anglin 618d0585d74SJohn David Anglin /* Calculate 8-bit hash index from virtual address */ 619d0585d74SJohn David Anglin extru_safe %r26, 27, 8, %r20 620d0585d74SJohn David Anglin 6211da177e4SLinus Torvalds /* Load start of lock table */ 622d0585d74SJohn David Anglin ldil L%lws_lock_start, %r28 623d0585d74SJohn David Anglin ldo R%lws_lock_start(%r28), %r28 6241da177e4SLinus Torvalds 625d0585d74SJohn David Anglin /* Find lock to use, the hash index is one of 0 to 626d0585d74SJohn David Anglin 255, multiplied by 16 (keep it 16-byte aligned) 6271da177e4SLinus Torvalds and add to the lock table offset. */ 6281da177e4SLinus Torvalds shlw %r20, 4, %r20 6291da177e4SLinus Torvalds add %r20, %r28, %r20 6301da177e4SLinus Torvalds 631d0585d74SJohn David Anglin rsm PSW_SM_I, %r0 /* Disable interrupts */ 6321da177e4SLinus Torvalds 633d0585d74SJohn David Anglin /* Try to acquire the lock */ 634d0585d74SJohn David Anglin LDCW 0(%sr2,%r20), %r28 635*a0f4b787SHelge Deller spinlock_check %r28, %r21 636d0585d74SJohn David Anglin comclr,<> %r0, %r28, %r0 637d0585d74SJohn David Anglin b,n lws_wouldblock 638d0585d74SJohn David Anglin 639d0585d74SJohn David Anglin /* Disable page faults to prevent sleeping in critical region */ 640d0585d74SJohn David Anglin lws_pagefault_disable %r21,%r28 6411da177e4SLinus Torvalds 6421da177e4SLinus Torvalds /* 6431da177e4SLinus Torvalds prev = *addr; 6441da177e4SLinus Torvalds if ( prev == old ) 6451da177e4SLinus Torvalds *addr = new; 6461da177e4SLinus Torvalds return prev; 6471da177e4SLinus Torvalds */ 6481da177e4SLinus Torvalds 6491da177e4SLinus Torvalds /* NOTES: 650d0585d74SJohn David Anglin This all works because intr_do_signal 6511da177e4SLinus Torvalds and schedule both check the return iasq 6521da177e4SLinus Torvalds and see that we are on the kernel page 6531da177e4SLinus Torvalds so this process is never scheduled off 6541da177e4SLinus Torvalds or is ever sent any signal of any sort, 655d0585d74SJohn David Anglin thus it is wholly atomic from usrspace's 6561da177e4SLinus Torvalds perspective 6571da177e4SLinus Torvalds */ 6581da177e4SLinus Torvalds /* The load and store could fail */ 659d0585d74SJohn David Anglin3: ldw 0(%r26), %r28 6601da177e4SLinus Torvalds sub,<> %r28, %r25, %r0 661d0585d74SJohn David Anglin4: stw %r24, 0(%r26) 662d0585d74SJohn David Anglin b,n lws_exit_noerror 6631da177e4SLinus Torvalds 664d0585d74SJohn David Anglin /* A fault occurred on load or stbys,e store */ 665d0585d74SJohn David Anglin5: b,n lws_fault 666d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 5b-linux_gateway_page) 667d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 5b-linux_gateway_page) 6681da177e4SLinus Torvalds 669d0585d74SJohn David Anglin /* A page fault occurred in critical region */ 670d0585d74SJohn David Anglin6: b,n lws_pagefault 671d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 6b-linux_gateway_page) 672d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 6b-linux_gateway_page) 6731da177e4SLinus Torvalds 6741da177e4SLinus Torvalds 67589206491SGuy Martin /*************************************************** 67689206491SGuy Martin New CAS implementation which uses pointers and variable size 67789206491SGuy Martin information. The value pointed by old and new MUST NOT change 678d0585d74SJohn David Anglin while performing CAS. The lock only protects the value at %r26. 67989206491SGuy Martin 68089206491SGuy Martin %r26 - Address to examine 68189206491SGuy Martin %r25 - Pointer to the value to check (old) 68289206491SGuy Martin %r24 - Pointer to the value to set (new) 68389206491SGuy Martin %r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 68489206491SGuy Martin %r28 - Return non-zero on failure 68589206491SGuy Martin %r21 - Kernel error code 68689206491SGuy Martin 687d0585d74SJohn David Anglin %r21 returns the following error codes: 68889206491SGuy Martin EAGAIN - CAS is busy, ldcw failed, try again. 68989206491SGuy Martin EFAULT - Read or write failed. 69089206491SGuy Martin 691d0585d74SJohn David Anglin If EAGAIN is returned, %r28 indicates the busy reason: 692d0585d74SJohn David Anglin r28 == 1 - CAS is busy. lock contended. 693d0585d74SJohn David Anglin r28 == 2 - CAS is busy. ldcw failed. 694d0585d74SJohn David Anglin r28 == 3 - CAS is busy. page fault. 695d0585d74SJohn David Anglin 69689206491SGuy Martin Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only) 69789206491SGuy Martin 69889206491SGuy Martin ****************************************************/ 69989206491SGuy Martin 70089206491SGuy Martinlws_compare_and_swap_2: 70189206491SGuy Martin#ifdef CONFIG_64BIT 702d0585d74SJohn David Anglin /* Wide mode user process? */ 703d0585d74SJohn David Anglin bb,<,n %sp, 31, cas2_begin 704d0585d74SJohn David Anglin 705d0585d74SJohn David Anglin /* Clip the input registers for 32-bit processes. We don't 706d0585d74SJohn David Anglin need to clip %r23 as we only use it for word operations */ 70789206491SGuy Martin depdi 0, 31, 32, %r26 70889206491SGuy Martin depdi 0, 31, 32, %r25 70989206491SGuy Martin depdi 0, 31, 32, %r24 71089206491SGuy Martin#endif 71189206491SGuy Martin 712d0585d74SJohn David Anglincas2_begin: 71389206491SGuy Martin /* Check the validity of the size pointer */ 71405f016d2SJohn David Anglin subi,>>= 3, %r23, %r0 71589206491SGuy Martin b,n lws_exit_nosys 71689206491SGuy Martin 71789206491SGuy Martin /* Jump to the functions which will load the old and new values into 71889206491SGuy Martin registers depending on the their size */ 71989206491SGuy Martin shlw %r23, 2, %r29 72089206491SGuy Martin blr %r29, %r0 72189206491SGuy Martin nop 72289206491SGuy Martin 723d0585d74SJohn David Anglin /* 8-bit load */ 724d0585d74SJohn David Anglin1: ldb 0(%r25), %r25 72589206491SGuy Martin b cas2_lock_start 726d0585d74SJohn David Anglin2: ldb 0(%r24), %r24 72789206491SGuy Martin nop 72889206491SGuy Martin nop 72989206491SGuy Martin nop 73089206491SGuy Martin nop 73189206491SGuy Martin nop 73289206491SGuy Martin 733d0585d74SJohn David Anglin /* 16-bit load */ 734d0585d74SJohn David Anglin3: ldh 0(%r25), %r25 73589206491SGuy Martin b cas2_lock_start 736d0585d74SJohn David Anglin4: ldh 0(%r24), %r24 73789206491SGuy Martin nop 73889206491SGuy Martin nop 73989206491SGuy Martin nop 74089206491SGuy Martin nop 74189206491SGuy Martin nop 74289206491SGuy Martin 743d0585d74SJohn David Anglin /* 32-bit load */ 744d0585d74SJohn David Anglin5: ldw 0(%r25), %r25 74589206491SGuy Martin b cas2_lock_start 746d0585d74SJohn David Anglin6: ldw 0(%r24), %r24 74789206491SGuy Martin nop 74889206491SGuy Martin nop 74989206491SGuy Martin nop 75089206491SGuy Martin nop 75189206491SGuy Martin nop 75289206491SGuy Martin 753d0585d74SJohn David Anglin /* 64-bit load */ 75489206491SGuy Martin#ifdef CONFIG_64BIT 755d0585d74SJohn David Anglin7: ldd 0(%r25), %r25 756d0585d74SJohn David Anglin8: ldd 0(%r24), %r24 75789206491SGuy Martin#else 758374b3bf8SJohn David Anglin /* Load old value into r22/r23 - high/low */ 759d0585d74SJohn David Anglin7: ldw 0(%r25), %r22 760d0585d74SJohn David Anglin8: ldw 4(%r25), %r23 76189206491SGuy Martin /* Load new value into fr4 for atomic store later */ 762d0585d74SJohn David Anglin9: flddx 0(%r24), %fr4 76389206491SGuy Martin#endif 76489206491SGuy Martin 76589206491SGuy Martincas2_lock_start: 766d0585d74SJohn David Anglin /* Trigger memory reference interruptions without writing to memory */ 767d0585d74SJohn David Anglin copy %r26, %r28 768d0585d74SJohn David Anglin depi_safe 0, 31, 2, %r28 769d0585d74SJohn David Anglin10: ldw 0(%r28), %r1 770d0585d74SJohn David Anglin11: stbys,e %r0, 0(%r28) 771d0585d74SJohn David Anglin 772d0585d74SJohn David Anglin /* Calculate 8-bit hash index from virtual address */ 773d0585d74SJohn David Anglin extru_safe %r26, 27, 8, %r20 774d0585d74SJohn David Anglin 77589206491SGuy Martin /* Load start of lock table */ 776d0585d74SJohn David Anglin ldil L%lws_lock_start, %r28 777d0585d74SJohn David Anglin ldo R%lws_lock_start(%r28), %r28 77889206491SGuy Martin 779d0585d74SJohn David Anglin /* Find lock to use, the hash index is one of 0 to 780d0585d74SJohn David Anglin 255, multiplied by 16 (keep it 16-byte aligned) 78189206491SGuy Martin and add to the lock table offset. */ 78289206491SGuy Martin shlw %r20, 4, %r20 78389206491SGuy Martin add %r20, %r28, %r20 78489206491SGuy Martin 785d0585d74SJohn David Anglin rsm PSW_SM_I, %r0 /* Disable interrupts */ 786d0585d74SJohn David Anglin 787d0585d74SJohn David Anglin /* Try to acquire the lock */ 788d0585d74SJohn David Anglin LDCW 0(%sr2,%r20), %r28 789*a0f4b787SHelge Deller spinlock_check %r28, %r21 790d0585d74SJohn David Anglin comclr,<> %r0, %r28, %r0 791d0585d74SJohn David Anglin b,n lws_wouldblock 792d0585d74SJohn David Anglin 793d0585d74SJohn David Anglin /* Disable page faults to prevent sleeping in critical region */ 794d0585d74SJohn David Anglin lws_pagefault_disable %r21,%r28 79589206491SGuy Martin 79689206491SGuy Martin /* 79789206491SGuy Martin prev = *addr; 79889206491SGuy Martin if ( prev == old ) 79989206491SGuy Martin *addr = new; 80089206491SGuy Martin return prev; 80189206491SGuy Martin */ 80289206491SGuy Martin 80389206491SGuy Martin /* NOTES: 804d0585d74SJohn David Anglin This all works because intr_do_signal 80589206491SGuy Martin and schedule both check the return iasq 80689206491SGuy Martin and see that we are on the kernel page 80789206491SGuy Martin so this process is never scheduled off 80889206491SGuy Martin or is ever sent any signal of any sort, 809d0585d74SJohn David Anglin thus it is wholly atomic from usrspace's 81089206491SGuy Martin perspective 81189206491SGuy Martin */ 812d0585d74SJohn David Anglin 81389206491SGuy Martin /* Jump to the correct function */ 81489206491SGuy Martin blr %r29, %r0 81589206491SGuy Martin /* Set %r28 as non-zero for now */ 81689206491SGuy Martin ldo 1(%r0),%r28 81789206491SGuy Martin 818d0585d74SJohn David Anglin /* 8-bit CAS */ 819d0585d74SJohn David Anglin12: ldb 0(%r26), %r29 82089206491SGuy Martin sub,= %r29, %r25, %r0 821d0585d74SJohn David Anglin b,n lws_exit_noerror 822d0585d74SJohn David Anglin13: stb %r24, 0(%r26) 823d0585d74SJohn David Anglin b lws_exit_noerror 82489206491SGuy Martin copy %r0, %r28 82589206491SGuy Martin nop 82689206491SGuy Martin nop 82789206491SGuy Martin 828d0585d74SJohn David Anglin /* 16-bit CAS */ 829d0585d74SJohn David Anglin14: ldh 0(%r26), %r29 83089206491SGuy Martin sub,= %r29, %r25, %r0 831d0585d74SJohn David Anglin b,n lws_exit_noerror 832d0585d74SJohn David Anglin15: sth %r24, 0(%r26) 833d0585d74SJohn David Anglin b lws_exit_noerror 83489206491SGuy Martin copy %r0, %r28 83589206491SGuy Martin nop 83689206491SGuy Martin nop 83789206491SGuy Martin 838d0585d74SJohn David Anglin /* 32-bit CAS */ 839d0585d74SJohn David Anglin16: ldw 0(%r26), %r29 84089206491SGuy Martin sub,= %r29, %r25, %r0 841d0585d74SJohn David Anglin b,n lws_exit_noerror 842d0585d74SJohn David Anglin17: stw %r24, 0(%r26) 843d0585d74SJohn David Anglin b lws_exit_noerror 84489206491SGuy Martin copy %r0, %r28 84589206491SGuy Martin nop 84689206491SGuy Martin nop 84789206491SGuy Martin 848d0585d74SJohn David Anglin /* 64-bit CAS */ 84989206491SGuy Martin#ifdef CONFIG_64BIT 850d0585d74SJohn David Anglin18: ldd 0(%r26), %r29 8511b59ddfcSJohn David Anglin sub,*= %r29, %r25, %r0 852d0585d74SJohn David Anglin b,n lws_exit_noerror 853d0585d74SJohn David Anglin19: std %r24, 0(%r26) 85489206491SGuy Martin copy %r0, %r28 85589206491SGuy Martin#else 85689206491SGuy Martin /* Compare first word */ 857d0585d74SJohn David Anglin18: ldw 0(%r26), %r29 85889206491SGuy Martin sub,= %r29, %r22, %r0 859d0585d74SJohn David Anglin b,n lws_exit_noerror 86089206491SGuy Martin /* Compare second word */ 861d0585d74SJohn David Anglin19: ldw 4(%r26), %r29 86289206491SGuy Martin sub,= %r29, %r23, %r0 863d0585d74SJohn David Anglin b,n lws_exit_noerror 86489206491SGuy Martin /* Perform the store */ 865d0585d74SJohn David Anglin20: fstdx %fr4, 0(%r26) 86689206491SGuy Martin copy %r0, %r28 86789206491SGuy Martin#endif 868d0585d74SJohn David Anglin b lws_exit_noerror 869d0585d74SJohn David Anglin copy %r0, %r28 87089206491SGuy Martin 871d0585d74SJohn David Anglin /* A fault occurred on load or stbys,e store */ 872d0585d74SJohn David Anglin30: b,n lws_fault 873d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) 874d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) 875d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) 876d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) 877d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) 878d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) 879d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) 880d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) 88189206491SGuy Martin#ifndef CONFIG_64BIT 882d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) 883d0585d74SJohn David Anglin#endif 884d0585d74SJohn David Anglin 885d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) 886d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) 887d0585d74SJohn David Anglin 888d0585d74SJohn David Anglin /* A page fault occurred in critical region */ 889d0585d74SJohn David Anglin31: b,n lws_pagefault 890d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) 891d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) 892d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) 893d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) 894d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) 895d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) 896d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) 897d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) 898d0585d74SJohn David Anglin#ifndef CONFIG_64BIT 899d0585d74SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) 90089206491SGuy Martin#endif 90189206491SGuy Martin 90272c3dd82SJohn David Anglin 90372c3dd82SJohn David Anglin /*************************************************** 90472c3dd82SJohn David Anglin LWS atomic exchange. 90572c3dd82SJohn David Anglin 90672c3dd82SJohn David Anglin %r26 - Exchange address 90772c3dd82SJohn David Anglin %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 90872c3dd82SJohn David Anglin %r24 - Address of new value 90972c3dd82SJohn David Anglin %r23 - Address of old value 91072c3dd82SJohn David Anglin %r28 - Return non-zero on failure 91172c3dd82SJohn David Anglin %r21 - Kernel error code 91272c3dd82SJohn David Anglin 91372c3dd82SJohn David Anglin %r21 returns the following error codes: 91472c3dd82SJohn David Anglin EAGAIN - CAS is busy, ldcw failed, try again. 91572c3dd82SJohn David Anglin EFAULT - Read or write failed. 91672c3dd82SJohn David Anglin 91772c3dd82SJohn David Anglin If EAGAIN is returned, %r28 indicates the busy reason: 91872c3dd82SJohn David Anglin r28 == 1 - CAS is busy. lock contended. 91972c3dd82SJohn David Anglin r28 == 2 - CAS is busy. ldcw failed. 92072c3dd82SJohn David Anglin r28 == 3 - CAS is busy. page fault. 92172c3dd82SJohn David Anglin 92272c3dd82SJohn David Anglin Scratch: r20, r1 92372c3dd82SJohn David Anglin 92472c3dd82SJohn David Anglin ****************************************************/ 92572c3dd82SJohn David Anglin 92672c3dd82SJohn David Anglinlws_atomic_xchg: 92772c3dd82SJohn David Anglin#ifdef CONFIG_64BIT 92872c3dd82SJohn David Anglin /* Wide mode user process? */ 92972c3dd82SJohn David Anglin bb,<,n %sp, 31, atomic_xchg_begin 93072c3dd82SJohn David Anglin 93172c3dd82SJohn David Anglin /* Clip the input registers for 32-bit processes. We don't 93272c3dd82SJohn David Anglin need to clip %r23 as we only use it for word operations */ 93372c3dd82SJohn David Anglin depdi 0, 31, 32, %r26 93472c3dd82SJohn David Anglin depdi 0, 31, 32, %r25 93572c3dd82SJohn David Anglin depdi 0, 31, 32, %r24 93672c3dd82SJohn David Anglin depdi 0, 31, 32, %r23 93772c3dd82SJohn David Anglin#endif 93872c3dd82SJohn David Anglin 93972c3dd82SJohn David Anglinatomic_xchg_begin: 94072c3dd82SJohn David Anglin /* Check the validity of the size pointer */ 94172c3dd82SJohn David Anglin subi,>>= 3, %r25, %r0 94272c3dd82SJohn David Anglin b,n lws_exit_nosys 94372c3dd82SJohn David Anglin 94472c3dd82SJohn David Anglin /* Jump to the functions which will load the old and new values into 94572c3dd82SJohn David Anglin registers depending on the their size */ 94672c3dd82SJohn David Anglin shlw %r25, 2, %r1 94772c3dd82SJohn David Anglin blr %r1, %r0 94872c3dd82SJohn David Anglin nop 94972c3dd82SJohn David Anglin 95072c3dd82SJohn David Anglin /* Perform exception checks */ 95172c3dd82SJohn David Anglin 95272c3dd82SJohn David Anglin /* 8-bit exchange */ 95372c3dd82SJohn David Anglin1: ldb 0(%r24), %r20 95472c3dd82SJohn David Anglin copy %r23, %r20 95572c3dd82SJohn David Anglin depi_safe 0, 31, 2, %r20 95672c3dd82SJohn David Anglin b atomic_xchg_start 95772c3dd82SJohn David Anglin2: stbys,e %r0, 0(%r20) 95872c3dd82SJohn David Anglin nop 95972c3dd82SJohn David Anglin nop 96072c3dd82SJohn David Anglin nop 96172c3dd82SJohn David Anglin 96272c3dd82SJohn David Anglin /* 16-bit exchange */ 96372c3dd82SJohn David Anglin3: ldh 0(%r24), %r20 96472c3dd82SJohn David Anglin copy %r23, %r20 96572c3dd82SJohn David Anglin depi_safe 0, 31, 2, %r20 96672c3dd82SJohn David Anglin b atomic_xchg_start 96772c3dd82SJohn David Anglin4: stbys,e %r0, 0(%r20) 96872c3dd82SJohn David Anglin nop 96972c3dd82SJohn David Anglin nop 97072c3dd82SJohn David Anglin nop 97172c3dd82SJohn David Anglin 97272c3dd82SJohn David Anglin /* 32-bit exchange */ 97372c3dd82SJohn David Anglin5: ldw 0(%r24), %r20 97472c3dd82SJohn David Anglin b atomic_xchg_start 97572c3dd82SJohn David Anglin6: stbys,e %r0, 0(%r23) 97672c3dd82SJohn David Anglin nop 97772c3dd82SJohn David Anglin nop 97872c3dd82SJohn David Anglin nop 97972c3dd82SJohn David Anglin nop 98072c3dd82SJohn David Anglin nop 98172c3dd82SJohn David Anglin 98272c3dd82SJohn David Anglin /* 64-bit exchange */ 98372c3dd82SJohn David Anglin#ifdef CONFIG_64BIT 98472c3dd82SJohn David Anglin7: ldd 0(%r24), %r20 98572c3dd82SJohn David Anglin8: stdby,e %r0, 0(%r23) 98672c3dd82SJohn David Anglin#else 98772c3dd82SJohn David Anglin7: ldw 0(%r24), %r20 98872c3dd82SJohn David Anglin8: ldw 4(%r24), %r20 98972c3dd82SJohn David Anglin copy %r23, %r20 99072c3dd82SJohn David Anglin depi_safe 0, 31, 2, %r20 99172c3dd82SJohn David Anglin9: stbys,e %r0, 0(%r20) 99272c3dd82SJohn David Anglin10: stbys,e %r0, 4(%r20) 99372c3dd82SJohn David Anglin#endif 99472c3dd82SJohn David Anglin 99572c3dd82SJohn David Anglinatomic_xchg_start: 99672c3dd82SJohn David Anglin /* Trigger memory reference interruptions without writing to memory */ 99772c3dd82SJohn David Anglin copy %r26, %r28 99872c3dd82SJohn David Anglin depi_safe 0, 31, 2, %r28 99972c3dd82SJohn David Anglin11: ldw 0(%r28), %r1 100072c3dd82SJohn David Anglin12: stbys,e %r0, 0(%r28) 100172c3dd82SJohn David Anglin 100272c3dd82SJohn David Anglin /* Calculate 8-bit hash index from virtual address */ 100372c3dd82SJohn David Anglin extru_safe %r26, 27, 8, %r20 100472c3dd82SJohn David Anglin 100572c3dd82SJohn David Anglin /* Load start of lock table */ 100672c3dd82SJohn David Anglin ldil L%lws_lock_start, %r28 100772c3dd82SJohn David Anglin ldo R%lws_lock_start(%r28), %r28 100872c3dd82SJohn David Anglin 100972c3dd82SJohn David Anglin /* Find lock to use, the hash index is one of 0 to 101072c3dd82SJohn David Anglin 255, multiplied by 16 (keep it 16-byte aligned) 101172c3dd82SJohn David Anglin and add to the lock table offset. */ 101272c3dd82SJohn David Anglin shlw %r20, 4, %r20 101372c3dd82SJohn David Anglin add %r20, %r28, %r20 101472c3dd82SJohn David Anglin 101572c3dd82SJohn David Anglin rsm PSW_SM_I, %r0 /* Disable interrupts */ 101672c3dd82SJohn David Anglin 101772c3dd82SJohn David Anglin /* Try to acquire the lock */ 101872c3dd82SJohn David Anglin LDCW 0(%sr2,%r20), %r28 1019*a0f4b787SHelge Deller spinlock_check %r28, %r21 102072c3dd82SJohn David Anglin comclr,<> %r0, %r28, %r0 102172c3dd82SJohn David Anglin b,n lws_wouldblock 102272c3dd82SJohn David Anglin 102372c3dd82SJohn David Anglin /* Disable page faults to prevent sleeping in critical region */ 102472c3dd82SJohn David Anglin lws_pagefault_disable %r21,%r28 102572c3dd82SJohn David Anglin 102672c3dd82SJohn David Anglin /* NOTES: 102772c3dd82SJohn David Anglin This all works because intr_do_signal 102872c3dd82SJohn David Anglin and schedule both check the return iasq 102972c3dd82SJohn David Anglin and see that we are on the kernel page 103072c3dd82SJohn David Anglin so this process is never scheduled off 103172c3dd82SJohn David Anglin or is ever sent any signal of any sort, 103272c3dd82SJohn David Anglin thus it is wholly atomic from userspace's 103372c3dd82SJohn David Anglin perspective 103472c3dd82SJohn David Anglin */ 103572c3dd82SJohn David Anglin 103672c3dd82SJohn David Anglin /* Jump to the correct function */ 103772c3dd82SJohn David Anglin blr %r1, %r0 103872c3dd82SJohn David Anglin /* Set %r28 as non-zero for now */ 103972c3dd82SJohn David Anglin ldo 1(%r0),%r28 104072c3dd82SJohn David Anglin 104172c3dd82SJohn David Anglin /* 8-bit exchange */ 104272c3dd82SJohn David Anglin14: ldb 0(%r26), %r1 104372c3dd82SJohn David Anglin15: stb %r1, 0(%r23) 104472c3dd82SJohn David Anglin15: ldb 0(%r24), %r1 104572c3dd82SJohn David Anglin17: stb %r1, 0(%r26) 104672c3dd82SJohn David Anglin b lws_exit_noerror 104772c3dd82SJohn David Anglin copy %r0, %r28 104872c3dd82SJohn David Anglin nop 104972c3dd82SJohn David Anglin nop 105072c3dd82SJohn David Anglin 105172c3dd82SJohn David Anglin /* 16-bit exchange */ 105272c3dd82SJohn David Anglin18: ldh 0(%r26), %r1 105372c3dd82SJohn David Anglin19: sth %r1, 0(%r23) 105472c3dd82SJohn David Anglin20: ldh 0(%r24), %r1 105572c3dd82SJohn David Anglin21: sth %r1, 0(%r26) 105672c3dd82SJohn David Anglin b lws_exit_noerror 105772c3dd82SJohn David Anglin copy %r0, %r28 105872c3dd82SJohn David Anglin nop 105972c3dd82SJohn David Anglin nop 106072c3dd82SJohn David Anglin 106172c3dd82SJohn David Anglin /* 32-bit exchange */ 106272c3dd82SJohn David Anglin22: ldw 0(%r26), %r1 106372c3dd82SJohn David Anglin23: stw %r1, 0(%r23) 106472c3dd82SJohn David Anglin24: ldw 0(%r24), %r1 106572c3dd82SJohn David Anglin25: stw %r1, 0(%r26) 106672c3dd82SJohn David Anglin b lws_exit_noerror 106772c3dd82SJohn David Anglin copy %r0, %r28 106872c3dd82SJohn David Anglin nop 106972c3dd82SJohn David Anglin nop 107072c3dd82SJohn David Anglin 107172c3dd82SJohn David Anglin /* 64-bit exchange */ 107272c3dd82SJohn David Anglin#ifdef CONFIG_64BIT 107372c3dd82SJohn David Anglin26: ldd 0(%r26), %r1 107472c3dd82SJohn David Anglin27: std %r1, 0(%r23) 107572c3dd82SJohn David Anglin28: ldd 0(%r24), %r1 107672c3dd82SJohn David Anglin29: std %r1, 0(%r26) 107772c3dd82SJohn David Anglin#else 107872c3dd82SJohn David Anglin26: flddx 0(%r26), %fr4 107972c3dd82SJohn David Anglin27: fstdx %fr4, 0(%r23) 108072c3dd82SJohn David Anglin28: flddx 0(%r24), %fr4 108172c3dd82SJohn David Anglin29: fstdx %fr4, 0(%r26) 108272c3dd82SJohn David Anglin#endif 108372c3dd82SJohn David Anglin b lws_exit_noerror 108472c3dd82SJohn David Anglin copy %r0, %r28 108572c3dd82SJohn David Anglin 108672c3dd82SJohn David Anglin /* A fault occurred on load or stbys,e store */ 108772c3dd82SJohn David Anglin30: b,n lws_fault 108872c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) 108972c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) 109072c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) 109172c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) 109272c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) 109372c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) 109472c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) 109572c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) 109672c3dd82SJohn David Anglin#ifndef CONFIG_64BIT 109772c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) 109872c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) 109972c3dd82SJohn David Anglin#endif 110072c3dd82SJohn David Anglin 110172c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) 110272c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 30b-linux_gateway_page) 110372c3dd82SJohn David Anglin 110472c3dd82SJohn David Anglin /* A page fault occurred in critical region */ 110572c3dd82SJohn David Anglin31: b,n lws_pagefault 110672c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) 110772c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) 110872c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) 110972c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) 111072c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) 111172c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) 111272c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) 111372c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 31b-linux_gateway_page) 111472c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(22b-linux_gateway_page, 31b-linux_gateway_page) 111572c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(23b-linux_gateway_page, 31b-linux_gateway_page) 111672c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(24b-linux_gateway_page, 31b-linux_gateway_page) 111772c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(25b-linux_gateway_page, 31b-linux_gateway_page) 111872c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(26b-linux_gateway_page, 31b-linux_gateway_page) 111972c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(27b-linux_gateway_page, 31b-linux_gateway_page) 112072c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(28b-linux_gateway_page, 31b-linux_gateway_page) 112172c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(29b-linux_gateway_page, 31b-linux_gateway_page) 112272c3dd82SJohn David Anglin 112372c3dd82SJohn David Anglin /*************************************************** 112472c3dd82SJohn David Anglin LWS atomic store. 112572c3dd82SJohn David Anglin 112672c3dd82SJohn David Anglin %r26 - Address to store 112772c3dd82SJohn David Anglin %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 112872c3dd82SJohn David Anglin %r24 - Address of value to store 112972c3dd82SJohn David Anglin %r28 - Return non-zero on failure 113072c3dd82SJohn David Anglin %r21 - Kernel error code 113172c3dd82SJohn David Anglin 113272c3dd82SJohn David Anglin %r21 returns the following error codes: 113372c3dd82SJohn David Anglin EAGAIN - CAS is busy, ldcw failed, try again. 113472c3dd82SJohn David Anglin EFAULT - Read or write failed. 113572c3dd82SJohn David Anglin 113672c3dd82SJohn David Anglin If EAGAIN is returned, %r28 indicates the busy reason: 113772c3dd82SJohn David Anglin r28 == 1 - CAS is busy. lock contended. 113872c3dd82SJohn David Anglin r28 == 2 - CAS is busy. ldcw failed. 113972c3dd82SJohn David Anglin r28 == 3 - CAS is busy. page fault. 114072c3dd82SJohn David Anglin 114172c3dd82SJohn David Anglin Scratch: r20, r1 114272c3dd82SJohn David Anglin 114372c3dd82SJohn David Anglin ****************************************************/ 114472c3dd82SJohn David Anglin 114572c3dd82SJohn David Anglinlws_atomic_store: 114672c3dd82SJohn David Anglin#ifdef CONFIG_64BIT 114772c3dd82SJohn David Anglin /* Wide mode user process? */ 114872c3dd82SJohn David Anglin bb,<,n %sp, 31, atomic_store_begin 114972c3dd82SJohn David Anglin 115072c3dd82SJohn David Anglin /* Clip the input registers for 32-bit processes. We don't 115172c3dd82SJohn David Anglin need to clip %r23 as we only use it for word operations */ 115272c3dd82SJohn David Anglin depdi 0, 31, 32, %r26 115372c3dd82SJohn David Anglin depdi 0, 31, 32, %r25 115472c3dd82SJohn David Anglin depdi 0, 31, 32, %r24 115572c3dd82SJohn David Anglin#endif 115672c3dd82SJohn David Anglin 115772c3dd82SJohn David Anglinatomic_store_begin: 115872c3dd82SJohn David Anglin /* Check the validity of the size pointer */ 115972c3dd82SJohn David Anglin subi,>>= 3, %r25, %r0 116072c3dd82SJohn David Anglin b,n lws_exit_nosys 116172c3dd82SJohn David Anglin 116272c3dd82SJohn David Anglin shlw %r25, 1, %r1 116372c3dd82SJohn David Anglin blr %r1, %r0 116472c3dd82SJohn David Anglin nop 116572c3dd82SJohn David Anglin 116672c3dd82SJohn David Anglin /* Perform exception checks */ 116772c3dd82SJohn David Anglin 116872c3dd82SJohn David Anglin /* 8-bit store */ 116972c3dd82SJohn David Anglin1: ldb 0(%r24), %r20 117072c3dd82SJohn David Anglin b,n atomic_store_start 117172c3dd82SJohn David Anglin nop 117272c3dd82SJohn David Anglin nop 117372c3dd82SJohn David Anglin 117472c3dd82SJohn David Anglin /* 16-bit store */ 117572c3dd82SJohn David Anglin2: ldh 0(%r24), %r20 117672c3dd82SJohn David Anglin b,n atomic_store_start 117772c3dd82SJohn David Anglin nop 117872c3dd82SJohn David Anglin nop 117972c3dd82SJohn David Anglin 118072c3dd82SJohn David Anglin /* 32-bit store */ 118172c3dd82SJohn David Anglin3: ldw 0(%r24), %r20 118272c3dd82SJohn David Anglin b,n atomic_store_start 118372c3dd82SJohn David Anglin nop 118472c3dd82SJohn David Anglin nop 118572c3dd82SJohn David Anglin 118672c3dd82SJohn David Anglin /* 64-bit store */ 118772c3dd82SJohn David Anglin#ifdef CONFIG_64BIT 118872c3dd82SJohn David Anglin4: ldd 0(%r24), %r20 118972c3dd82SJohn David Anglin#else 119072c3dd82SJohn David Anglin4: ldw 0(%r24), %r20 119172c3dd82SJohn David Anglin5: ldw 4(%r24), %r20 119272c3dd82SJohn David Anglin#endif 119372c3dd82SJohn David Anglin 119472c3dd82SJohn David Anglinatomic_store_start: 119572c3dd82SJohn David Anglin /* Trigger memory reference interruptions without writing to memory */ 119672c3dd82SJohn David Anglin copy %r26, %r28 119772c3dd82SJohn David Anglin depi_safe 0, 31, 2, %r28 119872c3dd82SJohn David Anglin6: ldw 0(%r28), %r1 119972c3dd82SJohn David Anglin7: stbys,e %r0, 0(%r28) 120072c3dd82SJohn David Anglin 120172c3dd82SJohn David Anglin /* Calculate 8-bit hash index from virtual address */ 120272c3dd82SJohn David Anglin extru_safe %r26, 27, 8, %r20 120372c3dd82SJohn David Anglin 120472c3dd82SJohn David Anglin /* Load start of lock table */ 120572c3dd82SJohn David Anglin ldil L%lws_lock_start, %r28 120672c3dd82SJohn David Anglin ldo R%lws_lock_start(%r28), %r28 120772c3dd82SJohn David Anglin 120872c3dd82SJohn David Anglin /* Find lock to use, the hash index is one of 0 to 120972c3dd82SJohn David Anglin 255, multiplied by 16 (keep it 16-byte aligned) 121072c3dd82SJohn David Anglin and add to the lock table offset. */ 121172c3dd82SJohn David Anglin shlw %r20, 4, %r20 121272c3dd82SJohn David Anglin add %r20, %r28, %r20 121372c3dd82SJohn David Anglin 121472c3dd82SJohn David Anglin rsm PSW_SM_I, %r0 /* Disable interrupts */ 121572c3dd82SJohn David Anglin 121672c3dd82SJohn David Anglin /* Try to acquire the lock */ 121772c3dd82SJohn David Anglin LDCW 0(%sr2,%r20), %r28 1218*a0f4b787SHelge Deller spinlock_check %r28, %r21 121972c3dd82SJohn David Anglin comclr,<> %r0, %r28, %r0 122072c3dd82SJohn David Anglin b,n lws_wouldblock 122172c3dd82SJohn David Anglin 122272c3dd82SJohn David Anglin /* Disable page faults to prevent sleeping in critical region */ 122372c3dd82SJohn David Anglin lws_pagefault_disable %r21,%r28 122472c3dd82SJohn David Anglin 122572c3dd82SJohn David Anglin /* NOTES: 122672c3dd82SJohn David Anglin This all works because intr_do_signal 122772c3dd82SJohn David Anglin and schedule both check the return iasq 122872c3dd82SJohn David Anglin and see that we are on the kernel page 122972c3dd82SJohn David Anglin so this process is never scheduled off 123072c3dd82SJohn David Anglin or is ever sent any signal of any sort, 123172c3dd82SJohn David Anglin thus it is wholly atomic from userspace's 123272c3dd82SJohn David Anglin perspective 123372c3dd82SJohn David Anglin */ 123472c3dd82SJohn David Anglin 123572c3dd82SJohn David Anglin /* Jump to the correct function */ 123672c3dd82SJohn David Anglin blr %r1, %r0 123772c3dd82SJohn David Anglin /* Set %r28 as non-zero for now */ 123872c3dd82SJohn David Anglin ldo 1(%r0),%r28 123972c3dd82SJohn David Anglin 124072c3dd82SJohn David Anglin /* 8-bit store */ 124172c3dd82SJohn David Anglin9: ldb 0(%r24), %r1 124272c3dd82SJohn David Anglin10: stb %r1, 0(%r26) 124372c3dd82SJohn David Anglin b lws_exit_noerror 124472c3dd82SJohn David Anglin copy %r0, %r28 124572c3dd82SJohn David Anglin 124672c3dd82SJohn David Anglin /* 16-bit store */ 124772c3dd82SJohn David Anglin11: ldh 0(%r24), %r1 124872c3dd82SJohn David Anglin12: sth %r1, 0(%r26) 124972c3dd82SJohn David Anglin b lws_exit_noerror 125072c3dd82SJohn David Anglin copy %r0, %r28 125172c3dd82SJohn David Anglin 125272c3dd82SJohn David Anglin /* 32-bit store */ 125372c3dd82SJohn David Anglin13: ldw 0(%r24), %r1 125472c3dd82SJohn David Anglin14: stw %r1, 0(%r26) 125572c3dd82SJohn David Anglin b lws_exit_noerror 125672c3dd82SJohn David Anglin copy %r0, %r28 125772c3dd82SJohn David Anglin 125872c3dd82SJohn David Anglin /* 64-bit store */ 125972c3dd82SJohn David Anglin#ifdef CONFIG_64BIT 126072c3dd82SJohn David Anglin15: ldd 0(%r24), %r1 126172c3dd82SJohn David Anglin16: std %r1, 0(%r26) 126272c3dd82SJohn David Anglin#else 126372c3dd82SJohn David Anglin15: flddx 0(%r24), %fr4 126472c3dd82SJohn David Anglin16: fstdx %fr4, 0(%r26) 126572c3dd82SJohn David Anglin#endif 126672c3dd82SJohn David Anglin b lws_exit_noerror 126772c3dd82SJohn David Anglin copy %r0, %r28 126872c3dd82SJohn David Anglin 126972c3dd82SJohn David Anglin /* A fault occurred on load or stbys,e store */ 127072c3dd82SJohn David Anglin30: b,n lws_fault 127172c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) 127272c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) 127372c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) 127472c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) 127572c3dd82SJohn David Anglin#ifndef CONFIG_64BIT 127672c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) 127772c3dd82SJohn David Anglin#endif 127872c3dd82SJohn David Anglin 127972c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) 128072c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) 128172c3dd82SJohn David Anglin 128272c3dd82SJohn David Anglin /* A page fault occurred in critical region */ 128372c3dd82SJohn David Anglin31: b,n lws_pagefault 128472c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 31b-linux_gateway_page) 128572c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 31b-linux_gateway_page) 128672c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 31b-linux_gateway_page) 128772c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) 128872c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) 128972c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) 129072c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) 129172c3dd82SJohn David Anglin ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) 129272c3dd82SJohn David Anglin 12931da177e4SLinus Torvalds /* Make sure nothing else is placed on this page */ 12941c593571SSam Ravnborg .align PAGE_SIZE 12958e9e9844SHelge DellerEND(linux_gateway_page) 12968e9e9844SHelge DellerENTRY(end_linux_gateway_page) 12971da177e4SLinus Torvalds 12981da177e4SLinus Torvalds /* Relocate symbols assuming linux_gateway_page is mapped 12991da177e4SLinus Torvalds to virtual address 0x0 */ 13008e9e9844SHelge Deller 13010b3d643fSHelge Deller#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page) 13021da177e4SLinus Torvalds 13031bcdd854SHelge Deller .section .rodata,"a" 13041bcdd854SHelge Deller 13056a45716aSHelge Deller .align 8 13061da177e4SLinus Torvalds /* Light-weight-syscall table */ 13071da177e4SLinus Torvalds /* Start of lws table. */ 13088e9e9844SHelge DellerENTRY(lws_table) 130989206491SGuy Martin LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic 32bit CAS */ 131089206491SGuy Martin LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic 32bit CAS */ 1311d0585d74SJohn David Anglin LWS_ENTRY(compare_and_swap_2) /* 2 - Atomic 64bit CAS */ 131272c3dd82SJohn David Anglin LWS_ENTRY(atomic_xchg) /* 3 - Atomic Exchange */ 131372c3dd82SJohn David Anglin LWS_ENTRY(atomic_store) /* 4 - Atomic Store */ 13148e9e9844SHelge DellerEND(lws_table) 13151da177e4SLinus Torvalds /* End of lws table */ 13161da177e4SLinus Torvalds 1317df86ddbbSMasahiro Yamada#ifdef CONFIG_64BIT 1318df86ddbbSMasahiro Yamada#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) 1319df86ddbbSMasahiro Yamada#else 1320df86ddbbSMasahiro Yamada#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) 1321df86ddbbSMasahiro Yamada#endif 1322106c9092SFiroz Khan#define __SYSCALL(nr, entry) ASM_ULONG_INSN entry 13236a45716aSHelge Deller .align 8 13248e9e9844SHelge DellerENTRY(sys_call_table) 1325fc79168aSHelge Deller .export sys_call_table,data 1326df86ddbbSMasahiro Yamada#include <asm/syscall_table_32.h> /* 32-bit syscalls */ 13278e9e9844SHelge DellerEND(sys_call_table) 13281da177e4SLinus Torvalds 1329413059f2SGrant Grundler#ifdef CONFIG_64BIT 13306a45716aSHelge Deller .align 8 13318e9e9844SHelge DellerENTRY(sys_call_table64) 1332df86ddbbSMasahiro Yamada#include <asm/syscall_table_64.h> /* 64-bit syscalls */ 13338e9e9844SHelge DellerEND(sys_call_table64) 13341da177e4SLinus Torvalds#endif 13351da177e4SLinus Torvalds 13361da177e4SLinus Torvalds /* 13371da177e4SLinus Torvalds All light-weight-syscall atomic operations 13381da177e4SLinus Torvalds will use this set of locks 1339c84c3a69SHelge Deller 1340c84c3a69SHelge Deller NOTE: The lws_lock_start symbol must be 1341c84c3a69SHelge Deller at least 16-byte aligned for safe use 1342c84c3a69SHelge Deller with ldcw. 13431da177e4SLinus Torvalds */ 1344dfcf753bSKyle McMartin .section .data 13456a45716aSHelge Deller .align L1_CACHE_BYTES 13468e9e9844SHelge DellerENTRY(lws_lock_start) 13471da177e4SLinus Torvalds /* lws locks */ 134853a42b63SJohn David Anglin .rept 256 13491da177e4SLinus Torvalds /* Keep locks aligned at 16-bytes */ 1350*a0f4b787SHelge Deller .word __ARCH_SPIN_LOCK_UNLOCKED_VAL 13511da177e4SLinus Torvalds .word 0 13521da177e4SLinus Torvalds .word 0 13531da177e4SLinus Torvalds .word 0 13541da177e4SLinus Torvalds .endr 13558e9e9844SHelge DellerEND(lws_lock_start) 13561da177e4SLinus Torvalds .previous 13571da177e4SLinus Torvalds 13581da177e4SLinus Torvalds.end 1359