1/* SPDX-License-Identifier: GPL-2.0 */ 2#include <linux/linkage.h> 3#include <asm/export.h> 4 5#include <asm/asm.h> 6 7/* 8 * unsigned int __sw_hweight32(unsigned int w) 9 * %rdi: w 10 */ 11ENTRY(__sw_hweight32) 12 13#ifdef CONFIG_X86_64 14 movl %edi, %eax # w 15#endif 16 __ASM_SIZE(push,) %__ASM_REG(dx) 17 movl %eax, %edx # w -> t 18 shrl %edx # t >>= 1 19 andl $0x55555555, %edx # t &= 0x55555555 20 subl %edx, %eax # w -= t 21 22 movl %eax, %edx # w -> t 23 shrl $2, %eax # w_tmp >>= 2 24 andl $0x33333333, %edx # t &= 0x33333333 25 andl $0x33333333, %eax # w_tmp &= 0x33333333 26 addl %edx, %eax # w = w_tmp + t 27 28 movl %eax, %edx # w -> t 29 shrl $4, %edx # t >>= 4 30 addl %edx, %eax # w_tmp += t 31 andl $0x0f0f0f0f, %eax # w_tmp &= 0x0f0f0f0f 32 imull $0x01010101, %eax, %eax # w_tmp *= 0x01010101 33 shrl $24, %eax # w = w_tmp >> 24 34 __ASM_SIZE(pop,) %__ASM_REG(dx) 35 ret 36ENDPROC(__sw_hweight32) 37EXPORT_SYMBOL(__sw_hweight32) 38 39ENTRY(__sw_hweight64) 40#ifdef CONFIG_X86_64 41 pushq %rdi 42 pushq %rdx 43 44 movq %rdi, %rdx # w -> t 45 movabsq $0x5555555555555555, %rax 46 shrq %rdx # t >>= 1 47 andq %rdx, %rax # t &= 0x5555555555555555 48 movabsq $0x3333333333333333, %rdx 49 subq %rax, %rdi # w -= t 50 51 movq %rdi, %rax # w -> t 52 shrq $2, %rdi # w_tmp >>= 2 53 andq %rdx, %rax # t &= 0x3333333333333333 54 andq %rdi, %rdx # w_tmp &= 0x3333333333333333 55 addq %rdx, %rax # w = w_tmp + t 56 57 movq %rax, %rdx # w -> t 58 shrq $4, %rdx # t >>= 4 59 addq %rdx, %rax # w_tmp += t 60 movabsq $0x0f0f0f0f0f0f0f0f, %rdx 61 andq %rdx, %rax # w_tmp &= 0x0f0f0f0f0f0f0f0f 62 movabsq $0x0101010101010101, %rdx 63 imulq %rdx, %rax # w_tmp *= 0x0101010101010101 64 shrq $56, %rax # w = w_tmp >> 56 65 66 popq %rdx 67 popq %rdi 68 ret 69#else /* CONFIG_X86_32 */ 70 /* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */ 71 pushl %ecx 72 73 call __sw_hweight32 74 movl %eax, %ecx # stash away result 75 movl %edx, %eax # second part of input 76 call __sw_hweight32 77 addl %ecx, %eax # result 78 79 popl %ecx 80 ret 81#endif 82ENDPROC(__sw_hweight64) 83EXPORT_SYMBOL(__sw_hweight64) 84