1/* 2 * atomic64_t for 586+ 3 * 4 * Copyright © 2010 Luca Barbieri 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 */ 11 12#include <linux/linkage.h> 13#include <asm/alternative-asm.h> 14#include <asm/dwarf2.h> 15 16.macro read64 reg 17 movl %ebx, %eax 18 movl %ecx, %edx 19/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 20 LOCK_PREFIX 21 cmpxchg8b (\reg) 22.endm 23 24ENTRY(atomic64_read_cx8) 25 CFI_STARTPROC 26 27 read64 %ecx 28 ret 29 CFI_ENDPROC 30ENDPROC(atomic64_read_cx8) 31 32ENTRY(atomic64_set_cx8) 33 CFI_STARTPROC 34 351: 36/* we don't need LOCK_PREFIX since aligned 64-bit writes 37 * are atomic on 586 and newer */ 38 cmpxchg8b (%esi) 39 jne 1b 40 41 ret 42 CFI_ENDPROC 43ENDPROC(atomic64_set_cx8) 44 45ENTRY(atomic64_xchg_cx8) 46 CFI_STARTPROC 47 481: 49 LOCK_PREFIX 50 cmpxchg8b (%esi) 51 jne 1b 52 53 ret 54 CFI_ENDPROC 55ENDPROC(atomic64_xchg_cx8) 56 57.macro addsub_return func ins insc 58ENTRY(atomic64_\func\()_return_cx8) 59 CFI_STARTPROC 60 pushl_cfi_reg ebp 61 pushl_cfi_reg ebx 62 pushl_cfi_reg esi 63 pushl_cfi_reg edi 64 65 movl %eax, %esi 66 movl %edx, %edi 67 movl %ecx, %ebp 68 69 read64 %ecx 701: 71 movl %eax, %ebx 72 movl %edx, %ecx 73 \ins\()l %esi, %ebx 74 \insc\()l %edi, %ecx 75 LOCK_PREFIX 76 cmpxchg8b (%ebp) 77 jne 1b 78 7910: 80 movl %ebx, %eax 81 movl %ecx, %edx 82 popl_cfi_reg edi 83 popl_cfi_reg esi 84 popl_cfi_reg ebx 85 popl_cfi_reg ebp 86 ret 87 CFI_ENDPROC 88ENDPROC(atomic64_\func\()_return_cx8) 89.endm 90 91addsub_return add add adc 92addsub_return sub sub sbb 93 94.macro incdec_return func ins insc 95ENTRY(atomic64_\func\()_return_cx8) 96 CFI_STARTPROC 97 pushl_cfi_reg ebx 98 99 read64 %esi 1001: 101 movl %eax, %ebx 102 movl %edx, %ecx 103 \ins\()l $1, %ebx 104 \insc\()l $0, %ecx 105 LOCK_PREFIX 106 cmpxchg8b (%esi) 107 jne 1b 108 10910: 110 movl %ebx, %eax 111 movl %ecx, %edx 112 popl_cfi_reg ebx 113 ret 114 CFI_ENDPROC 115ENDPROC(atomic64_\func\()_return_cx8) 116.endm 117 118incdec_return inc add adc 119incdec_return dec sub sbb 120 121ENTRY(atomic64_dec_if_positive_cx8) 122 CFI_STARTPROC 123 pushl_cfi_reg ebx 124 125 read64 %esi 1261: 127 movl %eax, %ebx 128 movl %edx, %ecx 129 subl $1, %ebx 130 sbb $0, %ecx 131 js 2f 132 LOCK_PREFIX 133 cmpxchg8b (%esi) 134 jne 1b 135 1362: 137 movl %ebx, %eax 138 movl %ecx, %edx 139 popl_cfi_reg ebx 140 ret 141 CFI_ENDPROC 142ENDPROC(atomic64_dec_if_positive_cx8) 143 144ENTRY(atomic64_add_unless_cx8) 145 CFI_STARTPROC 146 pushl_cfi_reg ebp 147 pushl_cfi_reg ebx 148/* these just push these two parameters on the stack */ 149 pushl_cfi_reg edi 150 pushl_cfi_reg ecx 151 152 movl %eax, %ebp 153 movl %edx, %edi 154 155 read64 %esi 1561: 157 cmpl %eax, 0(%esp) 158 je 4f 1592: 160 movl %eax, %ebx 161 movl %edx, %ecx 162 addl %ebp, %ebx 163 adcl %edi, %ecx 164 LOCK_PREFIX 165 cmpxchg8b (%esi) 166 jne 1b 167 168 movl $1, %eax 1693: 170 addl $8, %esp 171 CFI_ADJUST_CFA_OFFSET -8 172 popl_cfi_reg ebx 173 popl_cfi_reg ebp 174 ret 1754: 176 cmpl %edx, 4(%esp) 177 jne 2b 178 xorl %eax, %eax 179 jmp 3b 180 CFI_ENDPROC 181ENDPROC(atomic64_add_unless_cx8) 182 183ENTRY(atomic64_inc_not_zero_cx8) 184 CFI_STARTPROC 185 pushl_cfi_reg ebx 186 187 read64 %esi 1881: 189 movl %eax, %ecx 190 orl %edx, %ecx 191 jz 3f 192 movl %eax, %ebx 193 xorl %ecx, %ecx 194 addl $1, %ebx 195 adcl %edx, %ecx 196 LOCK_PREFIX 197 cmpxchg8b (%esi) 198 jne 1b 199 200 movl $1, %eax 2013: 202 popl_cfi_reg ebx 203 ret 204 CFI_ENDPROC 205ENDPROC(atomic64_inc_not_zero_cx8) 206