12874c5fdSThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-or-later */ 2a7e926abSLuca Barbieri/* 3a7e926abSLuca Barbieri * atomic64_t for 586+ 4a7e926abSLuca Barbieri * 5a7e926abSLuca Barbieri * Copyright © 2010 Luca Barbieri 6a7e926abSLuca Barbieri */ 7a7e926abSLuca Barbieri 8a7e926abSLuca Barbieri#include <linux/linkage.h> 95e21a3ecSJuergen Gross#include <asm/alternative.h> 10a7e926abSLuca Barbieri 11a7e926abSLuca Barbieri.macro read64 reg 12a7e926abSLuca Barbieri movl %ebx, %eax 13a7e926abSLuca Barbieri movl %ecx, %edx 14a7e926abSLuca Barbieri/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 15a7e926abSLuca Barbieri LOCK_PREFIX 16a7e926abSLuca Barbieri cmpxchg8b (\reg) 17a7e926abSLuca Barbieri.endm 18a7e926abSLuca Barbieri 196d685e53SJiri SlabySYM_FUNC_START(atomic64_read_cx8) 20a7e926abSLuca Barbieri read64 %ecx 21*f94909ceSPeter Zijlstra RET 226d685e53SJiri SlabySYM_FUNC_END(atomic64_read_cx8) 23a7e926abSLuca Barbieri 246d685e53SJiri SlabySYM_FUNC_START(atomic64_set_cx8) 25a7e926abSLuca Barbieri1: 26a7e926abSLuca Barbieri/* we don't need LOCK_PREFIX since aligned 64-bit writes 27a7e926abSLuca Barbieri * are atomic on 586 and newer */ 28a7e926abSLuca Barbieri cmpxchg8b (%esi) 29a7e926abSLuca Barbieri jne 1b 30a7e926abSLuca Barbieri 31*f94909ceSPeter Zijlstra RET 326d685e53SJiri SlabySYM_FUNC_END(atomic64_set_cx8) 33a7e926abSLuca Barbieri 346d685e53SJiri SlabySYM_FUNC_START(atomic64_xchg_cx8) 35a7e926abSLuca Barbieri1: 36a7e926abSLuca Barbieri LOCK_PREFIX 37a7e926abSLuca Barbieri cmpxchg8b (%esi) 38a7e926abSLuca Barbieri jne 1b 39a7e926abSLuca Barbieri 40*f94909ceSPeter Zijlstra RET 416d685e53SJiri SlabySYM_FUNC_END(atomic64_xchg_cx8) 42a7e926abSLuca Barbieri 43a7e926abSLuca Barbieri.macro addsub_return func ins insc 446d685e53SJiri SlabySYM_FUNC_START(atomic64_\func\()_return_cx8) 45131484c8SIngo Molnar pushl %ebp 46131484c8SIngo Molnar pushl %ebx 47131484c8SIngo Molnar pushl %esi 48131484c8SIngo Molnar pushl %edi 49a7e926abSLuca Barbieri 50a7e926abSLuca Barbieri movl %eax, %esi 51a7e926abSLuca Barbieri movl %edx, %edi 52a7e926abSLuca Barbieri movl %ecx, %ebp 53a7e926abSLuca Barbieri 54cb8095bbSJan Beulich read64 %ecx 55a7e926abSLuca Barbieri1: 56a7e926abSLuca Barbieri movl %eax, %ebx 57a7e926abSLuca Barbieri movl %edx, %ecx 58a7e926abSLuca Barbieri \ins\()l %esi, %ebx 59a7e926abSLuca Barbieri \insc\()l %edi, %ecx 60a7e926abSLuca Barbieri LOCK_PREFIX 61a7e926abSLuca Barbieri cmpxchg8b (%ebp) 62a7e926abSLuca Barbieri jne 1b 63a7e926abSLuca Barbieri 64a7e926abSLuca Barbieri10: 65a7e926abSLuca Barbieri movl %ebx, %eax 66a7e926abSLuca Barbieri movl %ecx, %edx 67131484c8SIngo Molnar popl %edi 68131484c8SIngo Molnar popl %esi 69131484c8SIngo Molnar popl %ebx 70131484c8SIngo Molnar popl %ebp 71*f94909ceSPeter Zijlstra RET 726d685e53SJiri SlabySYM_FUNC_END(atomic64_\func\()_return_cx8) 73a7e926abSLuca Barbieri.endm 74a7e926abSLuca Barbieri 75a7e926abSLuca Barbieriaddsub_return add add adc 76a7e926abSLuca Barbieriaddsub_return sub sub sbb 77a7e926abSLuca Barbieri 78a7e926abSLuca Barbieri.macro incdec_return func ins insc 796d685e53SJiri SlabySYM_FUNC_START(atomic64_\func\()_return_cx8) 80131484c8SIngo Molnar pushl %ebx 81a7e926abSLuca Barbieri 82a7e926abSLuca Barbieri read64 %esi 83a7e926abSLuca Barbieri1: 84a7e926abSLuca Barbieri movl %eax, %ebx 85a7e926abSLuca Barbieri movl %edx, %ecx 86a7e926abSLuca Barbieri \ins\()l $1, %ebx 87a7e926abSLuca Barbieri \insc\()l $0, %ecx 88a7e926abSLuca Barbieri LOCK_PREFIX 89a7e926abSLuca Barbieri cmpxchg8b (%esi) 90a7e926abSLuca Barbieri jne 1b 91a7e926abSLuca Barbieri 92a7e926abSLuca Barbieri10: 93a7e926abSLuca Barbieri movl %ebx, %eax 94a7e926abSLuca Barbieri movl %ecx, %edx 95131484c8SIngo Molnar popl %ebx 96*f94909ceSPeter Zijlstra RET 976d685e53SJiri SlabySYM_FUNC_END(atomic64_\func\()_return_cx8) 98a7e926abSLuca Barbieri.endm 99a7e926abSLuca Barbieri 100a7e926abSLuca Barbieriincdec_return inc add adc 101a7e926abSLuca Barbieriincdec_return dec sub sbb 102a7e926abSLuca Barbieri 1036d685e53SJiri SlabySYM_FUNC_START(atomic64_dec_if_positive_cx8) 104131484c8SIngo Molnar pushl %ebx 105a7e926abSLuca Barbieri 106a7e926abSLuca Barbieri read64 %esi 107a7e926abSLuca Barbieri1: 108a7e926abSLuca Barbieri movl %eax, %ebx 109a7e926abSLuca Barbieri movl %edx, %ecx 110a7e926abSLuca Barbieri subl $1, %ebx 111a7e926abSLuca Barbieri sbb $0, %ecx 112a7e926abSLuca Barbieri js 2f 113a7e926abSLuca Barbieri LOCK_PREFIX 114a7e926abSLuca Barbieri cmpxchg8b (%esi) 115a7e926abSLuca Barbieri jne 1b 116a7e926abSLuca Barbieri 117a7e926abSLuca Barbieri2: 118a7e926abSLuca Barbieri movl %ebx, %eax 119a7e926abSLuca Barbieri movl %ecx, %edx 120131484c8SIngo Molnar popl %ebx 121*f94909ceSPeter Zijlstra RET 1226d685e53SJiri SlabySYM_FUNC_END(atomic64_dec_if_positive_cx8) 123a7e926abSLuca Barbieri 1246d685e53SJiri SlabySYM_FUNC_START(atomic64_add_unless_cx8) 125131484c8SIngo Molnar pushl %ebp 126131484c8SIngo Molnar pushl %ebx 127a7e926abSLuca Barbieri/* these just push these two parameters on the stack */ 128131484c8SIngo Molnar pushl %edi 129131484c8SIngo Molnar pushl %ecx 130a7e926abSLuca Barbieri 131cb8095bbSJan Beulich movl %eax, %ebp 132a7e926abSLuca Barbieri movl %edx, %edi 133a7e926abSLuca Barbieri 134cb8095bbSJan Beulich read64 %esi 135a7e926abSLuca Barbieri1: 136a7e926abSLuca Barbieri cmpl %eax, 0(%esp) 137a7e926abSLuca Barbieri je 4f 138a7e926abSLuca Barbieri2: 139a7e926abSLuca Barbieri movl %eax, %ebx 140a7e926abSLuca Barbieri movl %edx, %ecx 141cb8095bbSJan Beulich addl %ebp, %ebx 142a7e926abSLuca Barbieri adcl %edi, %ecx 143a7e926abSLuca Barbieri LOCK_PREFIX 144cb8095bbSJan Beulich cmpxchg8b (%esi) 145a7e926abSLuca Barbieri jne 1b 146a7e926abSLuca Barbieri 1476e6104feSLuca Barbieri movl $1, %eax 148a7e926abSLuca Barbieri3: 149a7e926abSLuca Barbieri addl $8, %esp 150131484c8SIngo Molnar popl %ebx 151131484c8SIngo Molnar popl %ebp 152*f94909ceSPeter Zijlstra RET 153a7e926abSLuca Barbieri4: 154a7e926abSLuca Barbieri cmpl %edx, 4(%esp) 155a7e926abSLuca Barbieri jne 2b 1566e6104feSLuca Barbieri xorl %eax, %eax 157a7e926abSLuca Barbieri jmp 3b 1586d685e53SJiri SlabySYM_FUNC_END(atomic64_add_unless_cx8) 159a7e926abSLuca Barbieri 1606d685e53SJiri SlabySYM_FUNC_START(atomic64_inc_not_zero_cx8) 161131484c8SIngo Molnar pushl %ebx 162a7e926abSLuca Barbieri 163a7e926abSLuca Barbieri read64 %esi 164a7e926abSLuca Barbieri1: 165cb8095bbSJan Beulich movl %eax, %ecx 166cb8095bbSJan Beulich orl %edx, %ecx 167cb8095bbSJan Beulich jz 3f 168a7e926abSLuca Barbieri movl %eax, %ebx 169cb8095bbSJan Beulich xorl %ecx, %ecx 170a7e926abSLuca Barbieri addl $1, %ebx 171cb8095bbSJan Beulich adcl %edx, %ecx 172a7e926abSLuca Barbieri LOCK_PREFIX 173a7e926abSLuca Barbieri cmpxchg8b (%esi) 174a7e926abSLuca Barbieri jne 1b 175a7e926abSLuca Barbieri 176f3e83131SLuca Barbieri movl $1, %eax 177a7e926abSLuca Barbieri3: 178131484c8SIngo Molnar popl %ebx 179*f94909ceSPeter Zijlstra RET 1806d685e53SJiri SlabySYM_FUNC_END(atomic64_inc_not_zero_cx8) 181