1/* 2 * Linux/PA-RISC Project (http://www.parisc-linux.org/) 3 * 4 * System call entry code / Linux gateway page 5 * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org> 6 * Licensed under the GNU GPL. 7 * thanks to Philipp Rumpf, Mike Shaver and various others 8 * sorry about the wall, puffin.. 9 */ 10 11/* 12How does the Linux gateway page on PA-RISC work? 13------------------------------------------------ 14The Linux gateway page on PA-RISC is "special". 15It actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc 16terminology it's Execute, promote to PL0) in the page map. So anything 17executing on this page executes with kernel level privilege (there's more to it 18than that: to have this happen, you also have to use a branch with a ,gate 19completer to activate the privilege promotion). The upshot is that everything 20that runs on the gateway page runs at kernel privilege but with the current 21user process address space (although you have access to kernel space via %sr2). 22For the 0x100 syscall entry, we redo the space registers to point to the kernel 23address space (preserving the user address space in %sr3), move to wide mode if 24required, save the user registers and branch into the kernel syscall entry 25point. For all the other functions, we execute at kernel privilege but don't 26flip address spaces. The basic upshot of this is that these code snippets are 27executed atomically (because the kernel can't be pre-empted) and they may 28perform architecturally forbidden (to PL3) operations (like setting control 29registers). 30*/ 31 32 33#include <asm/asm-offsets.h> 34#include <asm/unistd.h> 35#include <asm/errno.h> 36#include <asm/page.h> 37#include <asm/psw.h> 38#include <asm/thread_info.h> 39#include <asm/assembly.h> 40#include <asm/processor.h> 41#include <asm/cache.h> 42 43#include <linux/linkage.h> 44 45 /* We fill the empty parts of the gateway page with 46 * something that will kill the kernel or a 47 * userspace application. 48 */ 49#define KILL_INSN break 0,0 50 51 .level PA_ASM_LEVEL 52 53 .macro lws_pagefault_disable reg1,reg2 54 mfctl %cr30, \reg2 55 ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 56 ldw 0(%sr2,\reg2), \reg1 57 ldo 1(\reg1), \reg1 58 stw \reg1, 0(%sr2,\reg2) 59 .endm 60 61 .macro lws_pagefault_enable reg1,reg2 62 mfctl %cr30, \reg2 63 ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 64 ldw 0(%sr2,\reg2), \reg1 65 ldo -1(\reg1), \reg1 66 stw \reg1, 0(%sr2,\reg2) 67 .endm 68 69 .text 70 71 .import syscall_exit,code 72 .import syscall_exit_rfi,code 73 74 /* Linux gateway page is aliased to virtual page 0 in the kernel 75 * address space. Since it is a gateway page it cannot be 76 * dereferenced, so null pointers will still fault. We start 77 * the actual entry point at 0x100. We put break instructions 78 * at the beginning of the page to trap null indirect function 79 * pointers. 80 */ 81 82 .align PAGE_SIZE 83ENTRY(linux_gateway_page) 84 85 /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */ 86 .rept 44 87 KILL_INSN 88 .endr 89 90 /* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */ 91 /* Light-weight-syscall entry must always be located at 0xb0 */ 92 /* WARNING: Keep this number updated with table size changes */ 93#define __NR_lws_entries (5) 94 95lws_entry: 96 gate lws_start, %r0 /* increase privilege */ 97 depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ 98 99 /* Fill from 0xb8 to 0xe0 */ 100 .rept 10 101 KILL_INSN 102 .endr 103 104 /* This function MUST be located at 0xe0 for glibc's threading 105 mechanism to work. DO NOT MOVE THIS CODE EVER! */ 106set_thread_pointer: 107 gate .+8, %r0 /* increase privilege */ 108 depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ 109 be 0(%sr7,%r31) /* return to user space */ 110 mtctl %r26, %cr27 /* move arg0 to the control register */ 111 112 /* Increase the chance of trapping if random jumps occur to this 113 address, fill from 0xf0 to 0x100 */ 114 .rept 4 115 KILL_INSN 116 .endr 117 118/* This address must remain fixed at 0x100 for glibc's syscalls to work */ 119 .align LINUX_GATEWAY_ADDR 120linux_gateway_entry: 121 gate .+8, %r0 /* become privileged */ 122 mtsp %r0,%sr4 /* get kernel space into sr4 */ 123 mtsp %r0,%sr5 /* get kernel space into sr5 */ 124 mtsp %r0,%sr6 /* get kernel space into sr6 */ 125 126#ifdef CONFIG_64BIT 127 /* Store W bit on entry to the syscall in case it's a wide userland 128 * process. */ 129 ssm PSW_SM_W, %r1 130 extrd,u %r1,PSW_W_BIT,1,%r1 131 /* sp must be aligned on 4, so deposit the W bit setting into 132 * the bottom of sp temporarily */ 133 or,ev %r1,%r30,%r30 134 b,n 1f 135 /* The top halves of argument registers must be cleared on syscall 136 * entry from narrow executable. 137 */ 138 depdi 0, 31, 32, %r26 139 depdi 0, 31, 32, %r25 140 depdi 0, 31, 32, %r24 141 depdi 0, 31, 32, %r23 142 depdi 0, 31, 32, %r22 143 depdi 0, 31, 32, %r21 1441: 145#endif 146 147 /* We use a rsm/ssm pair to prevent sr3 from being clobbered 148 * by external interrupts. 149 */ 150 mfsp %sr7,%r1 /* save user sr7 */ 151 rsm PSW_SM_I, %r0 /* disable interrupts */ 152 mtsp %r1,%sr3 /* and store it in sr3 */ 153 154 mfctl %cr30,%r1 155 xor %r1,%r30,%r30 /* ye olde xor trick */ 156 xor %r1,%r30,%r1 157 xor %r1,%r30,%r30 158 159 LDREG TASK_STACK(%r30),%r30 /* set up kernel stack */ 160 ldo FRAME_SIZE(%r30),%r30 161 /* N.B.: It is critical that we don't set sr7 to 0 until r30 162 * contains a valid kernel stack pointer. It is also 163 * critical that we don't start using the kernel stack 164 * until after sr7 has been set to 0. 165 */ 166 167 mtsp %r0,%sr7 /* get kernel space into sr7 */ 168 ssm PSW_SM_I, %r0 /* enable interrupts */ 169 STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ 170 mfctl %cr30,%r1 /* get task ptr in %r1 */ 171 172 /* Save some registers for sigcontext and potential task 173 switch (see entry.S for the details of which ones are 174 saved/restored). TASK_PT_PSW is zeroed so we can see whether 175 a process is on a syscall or not. For an interrupt the real 176 PSW value is stored. This is needed for gdb and sys_ptrace. */ 177 STREG %r0, TASK_PT_PSW(%r1) 178 STREG %r2, TASK_PT_GR2(%r1) /* preserve rp */ 179 STREG %r19, TASK_PT_GR19(%r1) 180 181 LDREGM -FRAME_SIZE(%r30), %r2 /* get users sp back */ 182#ifdef CONFIG_64BIT 183 extrd,u %r2,63,1,%r19 /* W hidden in bottom bit */ 184#if 0 185 xor %r19,%r2,%r2 /* clear bottom bit */ 186 depd,z %r19,1,1,%r19 187 std %r19,TASK_PT_PSW(%r1) 188#endif 189#endif 190 STREG %r2, TASK_PT_GR30(%r1) /* ... and save it */ 191 192 STREG %r20, TASK_PT_GR20(%r1) /* Syscall number */ 193 STREG %r21, TASK_PT_GR21(%r1) 194 STREG %r22, TASK_PT_GR22(%r1) 195 STREG %r23, TASK_PT_GR23(%r1) /* 4th argument */ 196 STREG %r24, TASK_PT_GR24(%r1) /* 3rd argument */ 197 STREG %r25, TASK_PT_GR25(%r1) /* 2nd argument */ 198 STREG %r26, TASK_PT_GR26(%r1) /* 1st argument */ 199 STREG %r27, TASK_PT_GR27(%r1) /* user dp */ 200 STREG %r28, TASK_PT_GR28(%r1) /* return value 0 */ 201 STREG %r0, TASK_PT_ORIG_R28(%r1) /* don't prohibit restarts */ 202 STREG %r29, TASK_PT_GR29(%r1) /* return value 1 */ 203 STREG %r31, TASK_PT_GR31(%r1) /* preserve syscall return ptr */ 204 205 ldo TASK_PT_FR0(%r1), %r27 /* save fpregs from the kernel */ 206 save_fp %r27 /* or potential task switch */ 207 208 mfctl %cr11, %r27 /* i.e. SAR */ 209 STREG %r27, TASK_PT_SAR(%r1) 210 211 loadgp 212 213#ifdef CONFIG_64BIT 214 ldo -16(%r30),%r29 /* Reference param save area */ 215 copy %r19,%r2 /* W bit back to r2 */ 216#else 217 /* no need to save these on stack in wide mode because the first 8 218 * args are passed in registers */ 219 stw %r22, -52(%r30) /* 5th argument */ 220 stw %r21, -56(%r30) /* 6th argument */ 221#endif 222 223 /* Are we being ptraced? */ 224 mfctl %cr30, %r1 225 LDREG TASK_TI_FLAGS(%r1),%r1 226 ldi _TIF_SYSCALL_TRACE_MASK, %r19 227 and,COND(=) %r1, %r19, %r0 228 b,n .Ltracesys 229 230 /* Note! We cannot use the syscall table that is mapped 231 nearby since the gateway page is mapped execute-only. */ 232 233#ifdef CONFIG_64BIT 234 ldil L%sys_call_table, %r1 235 or,= %r2,%r2,%r2 236 addil L%(sys_call_table64-sys_call_table), %r1 237 ldo R%sys_call_table(%r1), %r19 238 or,= %r2,%r2,%r2 239 ldo R%sys_call_table64(%r1), %r19 240#else 241 load32 sys_call_table, %r19 242#endif 243 comiclr,>> __NR_Linux_syscalls, %r20, %r0 244 b,n .Lsyscall_nosys 245 246 LDREGX %r20(%r19), %r19 247 248 /* If this is a sys_rt_sigreturn call, and the signal was received 249 * when not in_syscall, then we want to return via syscall_exit_rfi, 250 * not syscall_exit. Signal no. in r20, in_syscall in r25 (see 251 * trampoline code in signal.c). 252 */ 253 ldi __NR_rt_sigreturn,%r2 254 comb,= %r2,%r20,.Lrt_sigreturn 255.Lin_syscall: 256 ldil L%syscall_exit,%r2 257 be 0(%sr7,%r19) 258 ldo R%syscall_exit(%r2),%r2 259.Lrt_sigreturn: 260 comib,<> 0,%r25,.Lin_syscall 261 ldil L%syscall_exit_rfi,%r2 262 be 0(%sr7,%r19) 263 ldo R%syscall_exit_rfi(%r2),%r2 264 265 /* Note! Because we are not running where we were linked, any 266 calls to functions external to this file must be indirect. To 267 be safe, we apply the opposite rule to functions within this 268 file, with local labels given to them to ensure correctness. */ 269 270.Lsyscall_nosys: 271syscall_nosys: 272 ldil L%syscall_exit,%r1 273 be R%syscall_exit(%sr7,%r1) 274 ldo -ENOSYS(%r0),%r28 /* set errno */ 275 276 277/* Warning! This trace code is a virtual duplicate of the code above so be 278 * sure to maintain both! */ 279.Ltracesys: 280tracesys: 281 /* Need to save more registers so the debugger can see where we 282 * are. This saves only the lower 8 bits of PSW, so that the C 283 * bit is still clear on syscalls, and the D bit is set if this 284 * full register save path has been executed. We check the D 285 * bit on syscall_return_rfi to determine which registers to 286 * restore. An interrupt results in a full PSW saved with the 287 * C bit set, a non-straced syscall entry results in C and D clear 288 * in the saved PSW. 289 */ 290 mfctl %cr30,%r1 /* get task ptr */ 291 ssm 0,%r2 292 STREG %r2,TASK_PT_PSW(%r1) /* Lower 8 bits only!! */ 293 mfsp %sr0,%r2 294 STREG %r2,TASK_PT_SR0(%r1) 295 mfsp %sr1,%r2 296 STREG %r2,TASK_PT_SR1(%r1) 297 mfsp %sr2,%r2 298 STREG %r2,TASK_PT_SR2(%r1) 299 mfsp %sr3,%r2 300 STREG %r2,TASK_PT_SR3(%r1) 301 STREG %r2,TASK_PT_SR4(%r1) 302 STREG %r2,TASK_PT_SR5(%r1) 303 STREG %r2,TASK_PT_SR6(%r1) 304 STREG %r2,TASK_PT_SR7(%r1) 305 STREG %r2,TASK_PT_IASQ0(%r1) 306 STREG %r2,TASK_PT_IASQ1(%r1) 307 LDREG TASK_PT_GR31(%r1),%r2 308 STREG %r2,TASK_PT_IAOQ0(%r1) 309 ldo 4(%r2),%r2 310 STREG %r2,TASK_PT_IAOQ1(%r1) 311 ldo TASK_REGS(%r1),%r2 312 /* reg_save %r2 */ 313 STREG %r3,PT_GR3(%r2) 314 STREG %r4,PT_GR4(%r2) 315 STREG %r5,PT_GR5(%r2) 316 STREG %r6,PT_GR6(%r2) 317 STREG %r7,PT_GR7(%r2) 318 STREG %r8,PT_GR8(%r2) 319 STREG %r9,PT_GR9(%r2) 320 STREG %r10,PT_GR10(%r2) 321 STREG %r11,PT_GR11(%r2) 322 STREG %r12,PT_GR12(%r2) 323 STREG %r13,PT_GR13(%r2) 324 STREG %r14,PT_GR14(%r2) 325 STREG %r15,PT_GR15(%r2) 326 STREG %r16,PT_GR16(%r2) 327 STREG %r17,PT_GR17(%r2) 328 STREG %r18,PT_GR18(%r2) 329 /* Finished saving things for the debugger */ 330 331 copy %r2,%r26 332 ldil L%do_syscall_trace_enter,%r1 333 ldil L%tracesys_next,%r2 334 be R%do_syscall_trace_enter(%sr7,%r1) 335 ldo R%tracesys_next(%r2),%r2 336 337tracesys_next: 338 /* do_syscall_trace_enter either returned the syscallno, or -1L, 339 * so we skip restoring the PT_GR20 below, since we pulled it from 340 * task->thread.regs.gr[20] above. 341 */ 342 copy %ret0,%r20 343 344 mfctl %cr30,%r1 /* get task ptr */ 345 LDREG TASK_PT_GR28(%r1), %r28 /* Restore return value */ 346 LDREG TASK_PT_GR26(%r1), %r26 /* Restore the users args */ 347 LDREG TASK_PT_GR25(%r1), %r25 348 LDREG TASK_PT_GR24(%r1), %r24 349 LDREG TASK_PT_GR23(%r1), %r23 350 LDREG TASK_PT_GR22(%r1), %r22 351 LDREG TASK_PT_GR21(%r1), %r21 352#ifdef CONFIG_64BIT 353 ldo -16(%r30),%r29 /* Reference param save area */ 354#else 355 stw %r22, -52(%r30) /* 5th argument */ 356 stw %r21, -56(%r30) /* 6th argument */ 357#endif 358 359 cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */ 360 comiclr,>> __NR_Linux_syscalls, %r20, %r0 361 b,n .Ltracesys_nosys 362 363 /* Note! We cannot use the syscall table that is mapped 364 nearby since the gateway page is mapped execute-only. */ 365 366#ifdef CONFIG_64BIT 367 LDREG TASK_PT_GR30(%r1), %r19 /* get users sp back */ 368 extrd,u %r19,63,1,%r2 /* W hidden in bottom bit */ 369 370 ldil L%sys_call_table, %r1 371 or,= %r2,%r2,%r2 372 addil L%(sys_call_table64-sys_call_table), %r1 373 ldo R%sys_call_table(%r1), %r19 374 or,= %r2,%r2,%r2 375 ldo R%sys_call_table64(%r1), %r19 376#else 377 load32 sys_call_table, %r19 378#endif 379 380 LDREGX %r20(%r19), %r19 381 382 /* If this is a sys_rt_sigreturn call, and the signal was received 383 * when not in_syscall, then we want to return via syscall_exit_rfi, 384 * not syscall_exit. Signal no. in r20, in_syscall in r25 (see 385 * trampoline code in signal.c). 386 */ 387 ldi __NR_rt_sigreturn,%r2 388 comb,= %r2,%r20,.Ltrace_rt_sigreturn 389.Ltrace_in_syscall: 390 ldil L%tracesys_exit,%r2 391 be 0(%sr7,%r19) 392 ldo R%tracesys_exit(%r2),%r2 393 394.Ltracesys_nosys: 395 ldo -ENOSYS(%r0),%r28 /* set errno */ 396 397 /* Do *not* call this function on the gateway page, because it 398 makes a direct call to syscall_trace. */ 399 400tracesys_exit: 401 mfctl %cr30,%r1 /* get task ptr */ 402#ifdef CONFIG_64BIT 403 ldo -16(%r30),%r29 /* Reference param save area */ 404#endif 405 ldo TASK_REGS(%r1),%r26 406 BL do_syscall_trace_exit,%r2 407 STREG %r28,TASK_PT_GR28(%r1) /* save return value now */ 408 mfctl %cr30,%r1 /* get task ptr */ 409 LDREG TASK_PT_GR28(%r1), %r28 /* Restore return val. */ 410 411 ldil L%syscall_exit,%r1 412 be,n R%syscall_exit(%sr7,%r1) 413 414.Ltrace_rt_sigreturn: 415 comib,<> 0,%r25,.Ltrace_in_syscall 416 ldil L%tracesys_sigexit,%r2 417 be 0(%sr7,%r19) 418 ldo R%tracesys_sigexit(%r2),%r2 419 420tracesys_sigexit: 421 mfctl %cr30,%r1 /* get task ptr */ 422#ifdef CONFIG_64BIT 423 ldo -16(%r30),%r29 /* Reference param save area */ 424#endif 425 BL do_syscall_trace_exit,%r2 426 ldo TASK_REGS(%r1),%r26 427 428 ldil L%syscall_exit_rfi,%r1 429 be,n R%syscall_exit_rfi(%sr7,%r1) 430 431 432 /********************************************************* 433 32/64-bit Light-Weight-Syscall ABI 434 435 * - Indicates a hint for userspace inline asm 436 implementations. 437 438 Syscall number (caller-saves) 439 - %r20 440 * In asm clobber. 441 442 Argument registers (caller-saves) 443 - %r26, %r25, %r24, %r23, %r22 444 * In asm input. 445 446 Return registers (caller-saves) 447 - %r28 (return), %r21 (errno) 448 * In asm output. 449 450 Caller-saves registers 451 - %r1, %r27, %r29 452 - %r2 (return pointer) 453 - %r31 (ble link register) 454 * In asm clobber. 455 456 Callee-saves registers 457 - %r3-%r18 458 - %r30 (stack pointer) 459 * Not in asm clobber. 460 461 If userspace is 32-bit: 462 Callee-saves registers 463 - %r19 (32-bit PIC register) 464 465 Differences from 32-bit calling convention: 466 - Syscall number in %r20 467 - Additional argument register %r22 (arg4) 468 - Callee-saves %r19. 469 470 If userspace is 64-bit: 471 Callee-saves registers 472 - %r27 (64-bit PIC register) 473 474 Differences from 64-bit calling convention: 475 - Syscall number in %r20 476 - Additional argument register %r22 (arg4) 477 - Callee-saves %r27. 478 479 Error codes returned by entry path: 480 481 ENOSYS - r20 was an invalid LWS number. 482 483 *********************************************************/ 484lws_start: 485 486#ifdef CONFIG_64BIT 487 ssm PSW_SM_W, %r1 488 extrd,u %r1,PSW_W_BIT,1,%r1 489 /* sp must be aligned on 4, so deposit the W bit setting into 490 * the bottom of sp temporarily */ 491 or,od %r1,%r30,%r30 492 493 /* Clip LWS number to a 32-bit value for 32-bit processes */ 494 depdi 0, 31, 32, %r20 495#endif 496 497 /* Is the lws entry number valid? */ 498 comiclr,>> __NR_lws_entries, %r20, %r0 499 b,n lws_exit_nosys 500 501 /* Load table start */ 502 ldil L%lws_table, %r1 503 ldo R%lws_table(%r1), %r28 /* Scratch use of r28 */ 504 LDREGX %r20(%sr2,r28), %r21 /* Scratch use of r21 */ 505 506 /* Jump to lws, lws table pointers already relocated */ 507 be,n 0(%sr2,%r21) 508 509lws_exit_noerror: 510 lws_pagefault_enable %r1,%r21 511 stw,ma %r20, 0(%sr2,%r20) 512 ssm PSW_SM_I, %r0 513 b lws_exit 514 copy %r0, %r21 515 516lws_wouldblock: 517 ssm PSW_SM_I, %r0 518 ldo 2(%r0), %r28 519 b lws_exit 520 ldo -EAGAIN(%r0), %r21 521 522lws_pagefault: 523 lws_pagefault_enable %r1,%r21 524 stw,ma %r20, 0(%sr2,%r20) 525 ssm PSW_SM_I, %r0 526 ldo 3(%r0),%r28 527 b lws_exit 528 ldo -EAGAIN(%r0),%r21 529 530lws_fault: 531 ldo 1(%r0),%r28 532 b lws_exit 533 ldo -EFAULT(%r0),%r21 534 535lws_exit_nosys: 536 ldo -ENOSYS(%r0),%r21 537 /* Fall through: Return to userspace */ 538 539lws_exit: 540#ifdef CONFIG_64BIT 541 /* decide whether to reset the wide mode bit 542 * 543 * For a syscall, the W bit is stored in the lowest bit 544 * of sp. Extract it and reset W if it is zero */ 545 extrd,u,*<> %r30,63,1,%r1 546 rsm PSW_SM_W, %r0 547 /* now reset the lowest bit of sp if it was set */ 548 xor %r30,%r1,%r30 549#endif 550 be,n 0(%sr7, %r31) 551 552 553 554 /*************************************************** 555 Implementing 32bit CAS as an atomic operation: 556 557 %r26 - Address to examine 558 %r25 - Old value to check (old) 559 %r24 - New value to set (new) 560 %r28 - Return prev through this register. 561 %r21 - Kernel error code 562 563 %r21 returns the following error codes: 564 EAGAIN - CAS is busy, ldcw failed, try again. 565 EFAULT - Read or write failed. 566 567 If EAGAIN is returned, %r28 indicates the busy reason: 568 r28 == 1 - CAS is busy. lock contended. 569 r28 == 2 - CAS is busy. ldcw failed. 570 r28 == 3 - CAS is busy. page fault. 571 572 Scratch: r20, r28, r1 573 574 ****************************************************/ 575 576 /* ELF64 Process entry path */ 577lws_compare_and_swap64: 578#ifdef CONFIG_64BIT 579 b,n lws_compare_and_swap 580#else 581 /* If we are not a 64-bit kernel, then we don't 582 * have 64-bit input registers, and calling 583 * the 64-bit LWS CAS returns ENOSYS. 584 */ 585 b,n lws_exit_nosys 586#endif 587 588 /* ELF32/ELF64 Process entry path */ 589lws_compare_and_swap32: 590#ifdef CONFIG_64BIT 591 /* Wide mode user process? */ 592 bb,<,n %sp, 31, lws_compare_and_swap 593 594 /* Clip all the input registers for 32-bit processes */ 595 depdi 0, 31, 32, %r26 596 depdi 0, 31, 32, %r25 597 depdi 0, 31, 32, %r24 598#endif 599 600lws_compare_and_swap: 601 /* Trigger memory reference interruptions without writing to memory */ 6021: ldw 0(%r26), %r28 6032: stbys,e %r0, 0(%r26) 604 605 /* Calculate 8-bit hash index from virtual address */ 606 extru_safe %r26, 27, 8, %r20 607 608 /* Load start of lock table */ 609 ldil L%lws_lock_start, %r28 610 ldo R%lws_lock_start(%r28), %r28 611 612 /* Find lock to use, the hash index is one of 0 to 613 255, multiplied by 16 (keep it 16-byte aligned) 614 and add to the lock table offset. */ 615 shlw %r20, 4, %r20 616 add %r20, %r28, %r20 617 618 rsm PSW_SM_I, %r0 /* Disable interrupts */ 619 620 /* Try to acquire the lock */ 621 LDCW 0(%sr2,%r20), %r28 622 comclr,<> %r0, %r28, %r0 623 b,n lws_wouldblock 624 625 /* Disable page faults to prevent sleeping in critical region */ 626 lws_pagefault_disable %r21,%r28 627 628 /* 629 prev = *addr; 630 if ( prev == old ) 631 *addr = new; 632 return prev; 633 */ 634 635 /* NOTES: 636 This all works because intr_do_signal 637 and schedule both check the return iasq 638 and see that we are on the kernel page 639 so this process is never scheduled off 640 or is ever sent any signal of any sort, 641 thus it is wholly atomic from usrspace's 642 perspective 643 */ 644 /* The load and store could fail */ 6453: ldw 0(%r26), %r28 646 sub,<> %r28, %r25, %r0 6474: stw %r24, 0(%r26) 648 b,n lws_exit_noerror 649 650 /* A fault occurred on load or stbys,e store */ 6515: b,n lws_fault 652 ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 5b-linux_gateway_page) 653 ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 5b-linux_gateway_page) 654 655 /* A page fault occurred in critical region */ 6566: b,n lws_pagefault 657 ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 6b-linux_gateway_page) 658 ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 6b-linux_gateway_page) 659 660 661 /*************************************************** 662 New CAS implementation which uses pointers and variable size 663 information. The value pointed by old and new MUST NOT change 664 while performing CAS. The lock only protects the value at %r26. 665 666 %r26 - Address to examine 667 %r25 - Pointer to the value to check (old) 668 %r24 - Pointer to the value to set (new) 669 %r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 670 %r28 - Return non-zero on failure 671 %r21 - Kernel error code 672 673 %r21 returns the following error codes: 674 EAGAIN - CAS is busy, ldcw failed, try again. 675 EFAULT - Read or write failed. 676 677 If EAGAIN is returned, %r28 indicates the busy reason: 678 r28 == 1 - CAS is busy. lock contended. 679 r28 == 2 - CAS is busy. ldcw failed. 680 r28 == 3 - CAS is busy. page fault. 681 682 Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only) 683 684 ****************************************************/ 685 686lws_compare_and_swap_2: 687#ifdef CONFIG_64BIT 688 /* Wide mode user process? */ 689 bb,<,n %sp, 31, cas2_begin 690 691 /* Clip the input registers for 32-bit processes. We don't 692 need to clip %r23 as we only use it for word operations */ 693 depdi 0, 31, 32, %r26 694 depdi 0, 31, 32, %r25 695 depdi 0, 31, 32, %r24 696#endif 697 698cas2_begin: 699 /* Check the validity of the size pointer */ 700 subi,>>= 3, %r23, %r0 701 b,n lws_exit_nosys 702 703 /* Jump to the functions which will load the old and new values into 704 registers depending on the their size */ 705 shlw %r23, 2, %r29 706 blr %r29, %r0 707 nop 708 709 /* 8-bit load */ 7101: ldb 0(%r25), %r25 711 b cas2_lock_start 7122: ldb 0(%r24), %r24 713 nop 714 nop 715 nop 716 nop 717 nop 718 719 /* 16-bit load */ 7203: ldh 0(%r25), %r25 721 b cas2_lock_start 7224: ldh 0(%r24), %r24 723 nop 724 nop 725 nop 726 nop 727 nop 728 729 /* 32-bit load */ 7305: ldw 0(%r25), %r25 731 b cas2_lock_start 7326: ldw 0(%r24), %r24 733 nop 734 nop 735 nop 736 nop 737 nop 738 739 /* 64-bit load */ 740#ifdef CONFIG_64BIT 7417: ldd 0(%r25), %r25 7428: ldd 0(%r24), %r24 743#else 744 /* Load old value into r22/r23 - high/low */ 7457: ldw 0(%r25), %r22 7468: ldw 4(%r25), %r23 747 /* Load new value into fr4 for atomic store later */ 7489: flddx 0(%r24), %fr4 749#endif 750 751cas2_lock_start: 752 /* Trigger memory reference interruptions without writing to memory */ 753 copy %r26, %r28 754 depi_safe 0, 31, 2, %r28 75510: ldw 0(%r28), %r1 75611: stbys,e %r0, 0(%r28) 757 758 /* Calculate 8-bit hash index from virtual address */ 759 extru_safe %r26, 27, 8, %r20 760 761 /* Load start of lock table */ 762 ldil L%lws_lock_start, %r28 763 ldo R%lws_lock_start(%r28), %r28 764 765 /* Find lock to use, the hash index is one of 0 to 766 255, multiplied by 16 (keep it 16-byte aligned) 767 and add to the lock table offset. */ 768 shlw %r20, 4, %r20 769 add %r20, %r28, %r20 770 771 rsm PSW_SM_I, %r0 /* Disable interrupts */ 772 773 /* Try to acquire the lock */ 774 LDCW 0(%sr2,%r20), %r28 775 comclr,<> %r0, %r28, %r0 776 b,n lws_wouldblock 777 778 /* Disable page faults to prevent sleeping in critical region */ 779 lws_pagefault_disable %r21,%r28 780 781 /* 782 prev = *addr; 783 if ( prev == old ) 784 *addr = new; 785 return prev; 786 */ 787 788 /* NOTES: 789 This all works because intr_do_signal 790 and schedule both check the return iasq 791 and see that we are on the kernel page 792 so this process is never scheduled off 793 or is ever sent any signal of any sort, 794 thus it is wholly atomic from usrspace's 795 perspective 796 */ 797 798 /* Jump to the correct function */ 799 blr %r29, %r0 800 /* Set %r28 as non-zero for now */ 801 ldo 1(%r0),%r28 802 803 /* 8-bit CAS */ 80412: ldb 0(%r26), %r29 805 sub,= %r29, %r25, %r0 806 b,n lws_exit_noerror 80713: stb %r24, 0(%r26) 808 b lws_exit_noerror 809 copy %r0, %r28 810 nop 811 nop 812 813 /* 16-bit CAS */ 81414: ldh 0(%r26), %r29 815 sub,= %r29, %r25, %r0 816 b,n lws_exit_noerror 81715: sth %r24, 0(%r26) 818 b lws_exit_noerror 819 copy %r0, %r28 820 nop 821 nop 822 823 /* 32-bit CAS */ 82416: ldw 0(%r26), %r29 825 sub,= %r29, %r25, %r0 826 b,n lws_exit_noerror 82717: stw %r24, 0(%r26) 828 b lws_exit_noerror 829 copy %r0, %r28 830 nop 831 nop 832 833 /* 64-bit CAS */ 834#ifdef CONFIG_64BIT 83518: ldd 0(%r26), %r29 836 sub,*= %r29, %r25, %r0 837 b,n lws_exit_noerror 83819: std %r24, 0(%r26) 839 copy %r0, %r28 840#else 841 /* Compare first word */ 84218: ldw 0(%r26), %r29 843 sub,= %r29, %r22, %r0 844 b,n lws_exit_noerror 845 /* Compare second word */ 84619: ldw 4(%r26), %r29 847 sub,= %r29, %r23, %r0 848 b,n lws_exit_noerror 849 /* Perform the store */ 85020: fstdx %fr4, 0(%r26) 851 copy %r0, %r28 852#endif 853 b lws_exit_noerror 854 copy %r0, %r28 855 856 /* A fault occurred on load or stbys,e store */ 85730: b,n lws_fault 858 ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) 859 ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) 860 ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) 861 ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) 862 ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) 863 ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) 864 ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) 865 ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) 866#ifndef CONFIG_64BIT 867 ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) 868#endif 869 870 ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) 871 ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) 872 873 /* A page fault occurred in critical region */ 87431: b,n lws_pagefault 875 ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) 876 ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) 877 ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) 878 ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) 879 ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) 880 ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) 881 ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) 882 ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) 883#ifndef CONFIG_64BIT 884 ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) 885#endif 886 887 888 /*************************************************** 889 LWS atomic exchange. 890 891 %r26 - Exchange address 892 %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 893 %r24 - Address of new value 894 %r23 - Address of old value 895 %r28 - Return non-zero on failure 896 %r21 - Kernel error code 897 898 %r21 returns the following error codes: 899 EAGAIN - CAS is busy, ldcw failed, try again. 900 EFAULT - Read or write failed. 901 902 If EAGAIN is returned, %r28 indicates the busy reason: 903 r28 == 1 - CAS is busy. lock contended. 904 r28 == 2 - CAS is busy. ldcw failed. 905 r28 == 3 - CAS is busy. page fault. 906 907 Scratch: r20, r1 908 909 ****************************************************/ 910 911lws_atomic_xchg: 912#ifdef CONFIG_64BIT 913 /* Wide mode user process? */ 914 bb,<,n %sp, 31, atomic_xchg_begin 915 916 /* Clip the input registers for 32-bit processes. We don't 917 need to clip %r23 as we only use it for word operations */ 918 depdi 0, 31, 32, %r26 919 depdi 0, 31, 32, %r25 920 depdi 0, 31, 32, %r24 921 depdi 0, 31, 32, %r23 922#endif 923 924atomic_xchg_begin: 925 /* Check the validity of the size pointer */ 926 subi,>>= 3, %r25, %r0 927 b,n lws_exit_nosys 928 929 /* Jump to the functions which will load the old and new values into 930 registers depending on the their size */ 931 shlw %r25, 2, %r1 932 blr %r1, %r0 933 nop 934 935 /* Perform exception checks */ 936 937 /* 8-bit exchange */ 9381: ldb 0(%r24), %r20 939 copy %r23, %r20 940 depi_safe 0, 31, 2, %r20 941 b atomic_xchg_start 9422: stbys,e %r0, 0(%r20) 943 nop 944 nop 945 nop 946 947 /* 16-bit exchange */ 9483: ldh 0(%r24), %r20 949 copy %r23, %r20 950 depi_safe 0, 31, 2, %r20 951 b atomic_xchg_start 9524: stbys,e %r0, 0(%r20) 953 nop 954 nop 955 nop 956 957 /* 32-bit exchange */ 9585: ldw 0(%r24), %r20 959 b atomic_xchg_start 9606: stbys,e %r0, 0(%r23) 961 nop 962 nop 963 nop 964 nop 965 nop 966 967 /* 64-bit exchange */ 968#ifdef CONFIG_64BIT 9697: ldd 0(%r24), %r20 9708: stdby,e %r0, 0(%r23) 971#else 9727: ldw 0(%r24), %r20 9738: ldw 4(%r24), %r20 974 copy %r23, %r20 975 depi_safe 0, 31, 2, %r20 9769: stbys,e %r0, 0(%r20) 97710: stbys,e %r0, 4(%r20) 978#endif 979 980atomic_xchg_start: 981 /* Trigger memory reference interruptions without writing to memory */ 982 copy %r26, %r28 983 depi_safe 0, 31, 2, %r28 98411: ldw 0(%r28), %r1 98512: stbys,e %r0, 0(%r28) 986 987 /* Calculate 8-bit hash index from virtual address */ 988 extru_safe %r26, 27, 8, %r20 989 990 /* Load start of lock table */ 991 ldil L%lws_lock_start, %r28 992 ldo R%lws_lock_start(%r28), %r28 993 994 /* Find lock to use, the hash index is one of 0 to 995 255, multiplied by 16 (keep it 16-byte aligned) 996 and add to the lock table offset. */ 997 shlw %r20, 4, %r20 998 add %r20, %r28, %r20 999 1000 rsm PSW_SM_I, %r0 /* Disable interrupts */ 1001 1002 /* Try to acquire the lock */ 1003 LDCW 0(%sr2,%r20), %r28 1004 comclr,<> %r0, %r28, %r0 1005 b,n lws_wouldblock 1006 1007 /* Disable page faults to prevent sleeping in critical region */ 1008 lws_pagefault_disable %r21,%r28 1009 1010 /* NOTES: 1011 This all works because intr_do_signal 1012 and schedule both check the return iasq 1013 and see that we are on the kernel page 1014 so this process is never scheduled off 1015 or is ever sent any signal of any sort, 1016 thus it is wholly atomic from userspace's 1017 perspective 1018 */ 1019 1020 /* Jump to the correct function */ 1021 blr %r1, %r0 1022 /* Set %r28 as non-zero for now */ 1023 ldo 1(%r0),%r28 1024 1025 /* 8-bit exchange */ 102614: ldb 0(%r26), %r1 102715: stb %r1, 0(%r23) 102815: ldb 0(%r24), %r1 102917: stb %r1, 0(%r26) 1030 b lws_exit_noerror 1031 copy %r0, %r28 1032 nop 1033 nop 1034 1035 /* 16-bit exchange */ 103618: ldh 0(%r26), %r1 103719: sth %r1, 0(%r23) 103820: ldh 0(%r24), %r1 103921: sth %r1, 0(%r26) 1040 b lws_exit_noerror 1041 copy %r0, %r28 1042 nop 1043 nop 1044 1045 /* 32-bit exchange */ 104622: ldw 0(%r26), %r1 104723: stw %r1, 0(%r23) 104824: ldw 0(%r24), %r1 104925: stw %r1, 0(%r26) 1050 b lws_exit_noerror 1051 copy %r0, %r28 1052 nop 1053 nop 1054 1055 /* 64-bit exchange */ 1056#ifdef CONFIG_64BIT 105726: ldd 0(%r26), %r1 105827: std %r1, 0(%r23) 105928: ldd 0(%r24), %r1 106029: std %r1, 0(%r26) 1061#else 106226: flddx 0(%r26), %fr4 106327: fstdx %fr4, 0(%r23) 106428: flddx 0(%r24), %fr4 106529: fstdx %fr4, 0(%r26) 1066#endif 1067 b lws_exit_noerror 1068 copy %r0, %r28 1069 1070 /* A fault occurred on load or stbys,e store */ 107130: b,n lws_fault 1072 ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) 1073 ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) 1074 ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) 1075 ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) 1076 ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) 1077 ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) 1078 ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) 1079 ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) 1080#ifndef CONFIG_64BIT 1081 ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) 1082 ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) 1083#endif 1084 1085 ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) 1086 ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 30b-linux_gateway_page) 1087 1088 /* A page fault occurred in critical region */ 108931: b,n lws_pagefault 1090 ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) 1091 ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) 1092 ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) 1093 ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) 1094 ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) 1095 ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) 1096 ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) 1097 ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 31b-linux_gateway_page) 1098 ASM_EXCEPTIONTABLE_ENTRY(22b-linux_gateway_page, 31b-linux_gateway_page) 1099 ASM_EXCEPTIONTABLE_ENTRY(23b-linux_gateway_page, 31b-linux_gateway_page) 1100 ASM_EXCEPTIONTABLE_ENTRY(24b-linux_gateway_page, 31b-linux_gateway_page) 1101 ASM_EXCEPTIONTABLE_ENTRY(25b-linux_gateway_page, 31b-linux_gateway_page) 1102 ASM_EXCEPTIONTABLE_ENTRY(26b-linux_gateway_page, 31b-linux_gateway_page) 1103 ASM_EXCEPTIONTABLE_ENTRY(27b-linux_gateway_page, 31b-linux_gateway_page) 1104 ASM_EXCEPTIONTABLE_ENTRY(28b-linux_gateway_page, 31b-linux_gateway_page) 1105 ASM_EXCEPTIONTABLE_ENTRY(29b-linux_gateway_page, 31b-linux_gateway_page) 1106 1107 /*************************************************** 1108 LWS atomic store. 1109 1110 %r26 - Address to store 1111 %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) 1112 %r24 - Address of value to store 1113 %r28 - Return non-zero on failure 1114 %r21 - Kernel error code 1115 1116 %r21 returns the following error codes: 1117 EAGAIN - CAS is busy, ldcw failed, try again. 1118 EFAULT - Read or write failed. 1119 1120 If EAGAIN is returned, %r28 indicates the busy reason: 1121 r28 == 1 - CAS is busy. lock contended. 1122 r28 == 2 - CAS is busy. ldcw failed. 1123 r28 == 3 - CAS is busy. page fault. 1124 1125 Scratch: r20, r1 1126 1127 ****************************************************/ 1128 1129lws_atomic_store: 1130#ifdef CONFIG_64BIT 1131 /* Wide mode user process? */ 1132 bb,<,n %sp, 31, atomic_store_begin 1133 1134 /* Clip the input registers for 32-bit processes. We don't 1135 need to clip %r23 as we only use it for word operations */ 1136 depdi 0, 31, 32, %r26 1137 depdi 0, 31, 32, %r25 1138 depdi 0, 31, 32, %r24 1139#endif 1140 1141atomic_store_begin: 1142 /* Check the validity of the size pointer */ 1143 subi,>>= 3, %r25, %r0 1144 b,n lws_exit_nosys 1145 1146 shlw %r25, 1, %r1 1147 blr %r1, %r0 1148 nop 1149 1150 /* Perform exception checks */ 1151 1152 /* 8-bit store */ 11531: ldb 0(%r24), %r20 1154 b,n atomic_store_start 1155 nop 1156 nop 1157 1158 /* 16-bit store */ 11592: ldh 0(%r24), %r20 1160 b,n atomic_store_start 1161 nop 1162 nop 1163 1164 /* 32-bit store */ 11653: ldw 0(%r24), %r20 1166 b,n atomic_store_start 1167 nop 1168 nop 1169 1170 /* 64-bit store */ 1171#ifdef CONFIG_64BIT 11724: ldd 0(%r24), %r20 1173#else 11744: ldw 0(%r24), %r20 11755: ldw 4(%r24), %r20 1176#endif 1177 1178atomic_store_start: 1179 /* Trigger memory reference interruptions without writing to memory */ 1180 copy %r26, %r28 1181 depi_safe 0, 31, 2, %r28 11826: ldw 0(%r28), %r1 11837: stbys,e %r0, 0(%r28) 1184 1185 /* Calculate 8-bit hash index from virtual address */ 1186 extru_safe %r26, 27, 8, %r20 1187 1188 /* Load start of lock table */ 1189 ldil L%lws_lock_start, %r28 1190 ldo R%lws_lock_start(%r28), %r28 1191 1192 /* Find lock to use, the hash index is one of 0 to 1193 255, multiplied by 16 (keep it 16-byte aligned) 1194 and add to the lock table offset. */ 1195 shlw %r20, 4, %r20 1196 add %r20, %r28, %r20 1197 1198 rsm PSW_SM_I, %r0 /* Disable interrupts */ 1199 1200 /* Try to acquire the lock */ 1201 LDCW 0(%sr2,%r20), %r28 1202 comclr,<> %r0, %r28, %r0 1203 b,n lws_wouldblock 1204 1205 /* Disable page faults to prevent sleeping in critical region */ 1206 lws_pagefault_disable %r21,%r28 1207 1208 /* NOTES: 1209 This all works because intr_do_signal 1210 and schedule both check the return iasq 1211 and see that we are on the kernel page 1212 so this process is never scheduled off 1213 or is ever sent any signal of any sort, 1214 thus it is wholly atomic from userspace's 1215 perspective 1216 */ 1217 1218 /* Jump to the correct function */ 1219 blr %r1, %r0 1220 /* Set %r28 as non-zero for now */ 1221 ldo 1(%r0),%r28 1222 1223 /* 8-bit store */ 12249: ldb 0(%r24), %r1 122510: stb %r1, 0(%r26) 1226 b lws_exit_noerror 1227 copy %r0, %r28 1228 1229 /* 16-bit store */ 123011: ldh 0(%r24), %r1 123112: sth %r1, 0(%r26) 1232 b lws_exit_noerror 1233 copy %r0, %r28 1234 1235 /* 32-bit store */ 123613: ldw 0(%r24), %r1 123714: stw %r1, 0(%r26) 1238 b lws_exit_noerror 1239 copy %r0, %r28 1240 1241 /* 64-bit store */ 1242#ifdef CONFIG_64BIT 124315: ldd 0(%r24), %r1 124416: std %r1, 0(%r26) 1245#else 124615: flddx 0(%r24), %fr4 124716: fstdx %fr4, 0(%r26) 1248#endif 1249 b lws_exit_noerror 1250 copy %r0, %r28 1251 1252 /* A fault occurred on load or stbys,e store */ 125330: b,n lws_fault 1254 ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) 1255 ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) 1256 ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) 1257 ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) 1258#ifndef CONFIG_64BIT 1259 ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) 1260#endif 1261 1262 ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) 1263 ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) 1264 1265 /* A page fault occurred in critical region */ 126631: b,n lws_pagefault 1267 ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 31b-linux_gateway_page) 1268 ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 31b-linux_gateway_page) 1269 ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 31b-linux_gateway_page) 1270 ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) 1271 ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) 1272 ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) 1273 ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) 1274 ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) 1275 1276 /* Make sure nothing else is placed on this page */ 1277 .align PAGE_SIZE 1278END(linux_gateway_page) 1279ENTRY(end_linux_gateway_page) 1280 1281 /* Relocate symbols assuming linux_gateway_page is mapped 1282 to virtual address 0x0 */ 1283 1284#define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page) 1285 1286 .section .rodata,"a" 1287 1288 .align 8 1289 /* Light-weight-syscall table */ 1290 /* Start of lws table. */ 1291ENTRY(lws_table) 1292 LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic 32bit CAS */ 1293 LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic 32bit CAS */ 1294 LWS_ENTRY(compare_and_swap_2) /* 2 - Atomic 64bit CAS */ 1295 LWS_ENTRY(atomic_xchg) /* 3 - Atomic Exchange */ 1296 LWS_ENTRY(atomic_store) /* 4 - Atomic Store */ 1297END(lws_table) 1298 /* End of lws table */ 1299 1300#ifdef CONFIG_64BIT 1301#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) 1302#else 1303#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) 1304#endif 1305#define __SYSCALL(nr, entry) ASM_ULONG_INSN entry 1306 .align 8 1307ENTRY(sys_call_table) 1308 .export sys_call_table,data 1309#include <asm/syscall_table_32.h> /* 32-bit syscalls */ 1310END(sys_call_table) 1311 1312#ifdef CONFIG_64BIT 1313 .align 8 1314ENTRY(sys_call_table64) 1315#include <asm/syscall_table_64.h> /* 64-bit syscalls */ 1316END(sys_call_table64) 1317#endif 1318 1319 /* 1320 All light-weight-syscall atomic operations 1321 will use this set of locks 1322 1323 NOTE: The lws_lock_start symbol must be 1324 at least 16-byte aligned for safe use 1325 with ldcw. 1326 */ 1327 .section .data 1328 .align L1_CACHE_BYTES 1329ENTRY(lws_lock_start) 1330 /* lws locks */ 1331 .rept 256 1332 /* Keep locks aligned at 16-bytes */ 1333 .word 1 1334 .word 0 1335 .word 0 1336 .word 0 1337 .endr 1338END(lws_lock_start) 1339 .previous 1340 1341.end 1342