1/* 2 * atomic64_t for 586+ 3 * 4 * Copyright © 2010 Luca Barbieri 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 */ 11 12#include <linux/linkage.h> 13#include <asm/alternative-asm.h> 14#include <asm/dwarf2.h> 15 16.macro SAVE reg 17 pushl_cfi %\reg 18 CFI_REL_OFFSET \reg, 0 19.endm 20 21.macro RESTORE reg 22 popl_cfi %\reg 23 CFI_RESTORE \reg 24.endm 25 26.macro read64 reg 27 movl %ebx, %eax 28 movl %ecx, %edx 29/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 30 LOCK_PREFIX 31 cmpxchg8b (\reg) 32.endm 33 34ENTRY(atomic64_read_cx8) 35 CFI_STARTPROC 36 37 read64 %ecx 38 ret 39 CFI_ENDPROC 40ENDPROC(atomic64_read_cx8) 41 42ENTRY(atomic64_set_cx8) 43 CFI_STARTPROC 44 451: 46/* we don't need LOCK_PREFIX since aligned 64-bit writes 47 * are atomic on 586 and newer */ 48 cmpxchg8b (%esi) 49 jne 1b 50 51 ret 52 CFI_ENDPROC 53ENDPROC(atomic64_set_cx8) 54 55ENTRY(atomic64_xchg_cx8) 56 CFI_STARTPROC 57 581: 59 LOCK_PREFIX 60 cmpxchg8b (%esi) 61 jne 1b 62 63 ret 64 CFI_ENDPROC 65ENDPROC(atomic64_xchg_cx8) 66 67.macro addsub_return func ins insc 68ENTRY(atomic64_\func\()_return_cx8) 69 CFI_STARTPROC 70 SAVE ebp 71 SAVE ebx 72 SAVE esi 73 SAVE edi 74 75 movl %eax, %esi 76 movl %edx, %edi 77 movl %ecx, %ebp 78 79 read64 %ecx 801: 81 movl %eax, %ebx 82 movl %edx, %ecx 83 \ins\()l %esi, %ebx 84 \insc\()l %edi, %ecx 85 LOCK_PREFIX 86 cmpxchg8b (%ebp) 87 jne 1b 88 8910: 90 movl %ebx, %eax 91 movl %ecx, %edx 92 RESTORE edi 93 RESTORE esi 94 RESTORE ebx 95 RESTORE ebp 96 ret 97 CFI_ENDPROC 98ENDPROC(atomic64_\func\()_return_cx8) 99.endm 100 101addsub_return add add adc 102addsub_return sub sub sbb 103 104.macro incdec_return func ins insc 105ENTRY(atomic64_\func\()_return_cx8) 106 CFI_STARTPROC 107 SAVE ebx 108 109 read64 %esi 1101: 111 movl %eax, %ebx 112 movl %edx, %ecx 113 \ins\()l $1, %ebx 114 \insc\()l $0, %ecx 115 LOCK_PREFIX 116 cmpxchg8b (%esi) 117 jne 1b 118 11910: 120 movl %ebx, %eax 121 movl %ecx, %edx 122 RESTORE ebx 123 ret 124 CFI_ENDPROC 125ENDPROC(atomic64_\func\()_return_cx8) 126.endm 127 128incdec_return inc add adc 129incdec_return dec sub sbb 130 131ENTRY(atomic64_dec_if_positive_cx8) 132 CFI_STARTPROC 133 SAVE ebx 134 135 read64 %esi 1361: 137 movl %eax, %ebx 138 movl %edx, %ecx 139 subl $1, %ebx 140 sbb $0, %ecx 141 js 2f 142 LOCK_PREFIX 143 cmpxchg8b (%esi) 144 jne 1b 145 1462: 147 movl %ebx, %eax 148 movl %ecx, %edx 149 RESTORE ebx 150 ret 151 CFI_ENDPROC 152ENDPROC(atomic64_dec_if_positive_cx8) 153 154ENTRY(atomic64_add_unless_cx8) 155 CFI_STARTPROC 156 SAVE ebp 157 SAVE ebx 158/* these just push these two parameters on the stack */ 159 SAVE edi 160 SAVE ecx 161 162 movl %eax, %ebp 163 movl %edx, %edi 164 165 read64 %esi 1661: 167 cmpl %eax, 0(%esp) 168 je 4f 1692: 170 movl %eax, %ebx 171 movl %edx, %ecx 172 addl %ebp, %ebx 173 adcl %edi, %ecx 174 LOCK_PREFIX 175 cmpxchg8b (%esi) 176 jne 1b 177 178 movl $1, %eax 1793: 180 addl $8, %esp 181 CFI_ADJUST_CFA_OFFSET -8 182 RESTORE ebx 183 RESTORE ebp 184 ret 1854: 186 cmpl %edx, 4(%esp) 187 jne 2b 188 xorl %eax, %eax 189 jmp 3b 190 CFI_ENDPROC 191ENDPROC(atomic64_add_unless_cx8) 192 193ENTRY(atomic64_inc_not_zero_cx8) 194 CFI_STARTPROC 195 SAVE ebx 196 197 read64 %esi 1981: 199 movl %eax, %ecx 200 orl %edx, %ecx 201 jz 3f 202 movl %eax, %ebx 203 xorl %ecx, %ecx 204 addl $1, %ebx 205 adcl %edx, %ecx 206 LOCK_PREFIX 207 cmpxchg8b (%esi) 208 jne 1b 209 210 movl $1, %eax 2113: 212 RESTORE ebx 213 ret 214 CFI_ENDPROC 215ENDPROC(atomic64_inc_not_zero_cx8) 216