1/* SPDX-License-Identifier: GPL-2.0 */
 2#include <linux/export.h>
 3#include <linux/linkage.h>
 
 4
 5#include <asm/asm.h>
 6
 7/*
 8 * unsigned int __sw_hweight32(unsigned int w)
 9 * %rdi: w
10 */
11SYM_FUNC_START(__sw_hweight32)
12
13#ifdef CONFIG_X86_64
14	movl %edi, %eax				# w
15#endif
16	__ASM_SIZE(push,) %__ASM_REG(dx)
17	movl %eax, %edx				# w -> t
18	shrl %edx				# t >>= 1
19	andl $0x55555555, %edx			# t &= 0x55555555
20	subl %edx, %eax				# w -= t
21
22	movl %eax, %edx				# w -> t
23	shrl $2, %eax				# w_tmp >>= 2
24	andl $0x33333333, %edx			# t	&= 0x33333333
25	andl $0x33333333, %eax			# w_tmp &= 0x33333333
26	addl %edx, %eax				# w = w_tmp + t
27
28	movl %eax, %edx				# w -> t
29	shrl $4, %edx				# t >>= 4
30	addl %edx, %eax				# w_tmp += t
31	andl  $0x0f0f0f0f, %eax			# w_tmp &= 0x0f0f0f0f
32	imull $0x01010101, %eax, %eax		# w_tmp *= 0x01010101
33	shrl $24, %eax				# w = w_tmp >> 24
34	__ASM_SIZE(pop,) %__ASM_REG(dx)
35	RET
36SYM_FUNC_END(__sw_hweight32)
37EXPORT_SYMBOL(__sw_hweight32)
38
39/*
40 * No 32-bit variant, because it's implemented as an inline wrapper
41 * on top of __arch_hweight32():
42 */
43#ifdef CONFIG_X86_64
44SYM_FUNC_START(__sw_hweight64)
45	pushq   %rdi
46	pushq   %rdx
47
48	movq    %rdi, %rdx                      # w -> t
49	movabsq $0x5555555555555555, %rax
50	shrq    %rdx                            # t >>= 1
51	andq    %rdx, %rax                      # t &= 0x5555555555555555
52	movabsq $0x3333333333333333, %rdx
53	subq    %rax, %rdi                      # w -= t
54
55	movq    %rdi, %rax                      # w -> t
56	shrq    $2, %rdi                        # w_tmp >>= 2
57	andq    %rdx, %rax                      # t     &= 0x3333333333333333
58	andq    %rdi, %rdx                      # w_tmp &= 0x3333333333333333
59	addq    %rdx, %rax                      # w = w_tmp + t
60
61	movq    %rax, %rdx                      # w -> t
62	shrq    $4, %rdx                        # t >>= 4
63	addq    %rdx, %rax                      # w_tmp += t
64	movabsq $0x0f0f0f0f0f0f0f0f, %rdx
65	andq    %rdx, %rax                      # w_tmp &= 0x0f0f0f0f0f0f0f0f
66	movabsq $0x0101010101010101, %rdx
67	imulq   %rdx, %rax                      # w_tmp *= 0x0101010101010101
68	shrq    $56, %rax                       # w = w_tmp >> 56
69
70	popq    %rdx
71	popq    %rdi
72	RET
73SYM_FUNC_END(__sw_hweight64)
74EXPORT_SYMBOL(__sw_hweight64)
 
 
 
 
 
 
 
 
 
 
75#endif

 1/* SPDX-License-Identifier: GPL-2.0 */
 
 2#include <linux/linkage.h>
 3#include <asm/export.h>
 4
 5#include <asm/asm.h>
 6
 7/*
 8 * unsigned int __sw_hweight32(unsigned int w)
 9 * %rdi: w
10 */
11ENTRY(__sw_hweight32)
12
13#ifdef CONFIG_X86_64
14	movl %edi, %eax				# w
15#endif
16	__ASM_SIZE(push,) %__ASM_REG(dx)
17	movl %eax, %edx				# w -> t
18	shrl %edx				# t >>= 1
19	andl $0x55555555, %edx			# t &= 0x55555555
20	subl %edx, %eax				# w -= t
21
22	movl %eax, %edx				# w -> t
23	shrl $2, %eax				# w_tmp >>= 2
24	andl $0x33333333, %edx			# t	&= 0x33333333
25	andl $0x33333333, %eax			# w_tmp &= 0x33333333
26	addl %edx, %eax				# w = w_tmp + t
27
28	movl %eax, %edx				# w -> t
29	shrl $4, %edx				# t >>= 4
30	addl %edx, %eax				# w_tmp += t
31	andl  $0x0f0f0f0f, %eax			# w_tmp &= 0x0f0f0f0f
32	imull $0x01010101, %eax, %eax		# w_tmp *= 0x01010101
33	shrl $24, %eax				# w = w_tmp >> 24
34	__ASM_SIZE(pop,) %__ASM_REG(dx)
35	ret
36ENDPROC(__sw_hweight32)
37EXPORT_SYMBOL(__sw_hweight32)
38
39ENTRY(__sw_hweight64)
 
 
 
40#ifdef CONFIG_X86_64
 
41	pushq   %rdi
42	pushq   %rdx
43
44	movq    %rdi, %rdx                      # w -> t
45	movabsq $0x5555555555555555, %rax
46	shrq    %rdx                            # t >>= 1
47	andq    %rdx, %rax                      # t &= 0x5555555555555555
48	movabsq $0x3333333333333333, %rdx
49	subq    %rax, %rdi                      # w -= t
50
51	movq    %rdi, %rax                      # w -> t
52	shrq    $2, %rdi                        # w_tmp >>= 2
53	andq    %rdx, %rax                      # t     &= 0x3333333333333333
54	andq    %rdi, %rdx                      # w_tmp &= 0x3333333333333333
55	addq    %rdx, %rax                      # w = w_tmp + t
56
57	movq    %rax, %rdx                      # w -> t
58	shrq    $4, %rdx                        # t >>= 4
59	addq    %rdx, %rax                      # w_tmp += t
60	movabsq $0x0f0f0f0f0f0f0f0f, %rdx
61	andq    %rdx, %rax                      # w_tmp &= 0x0f0f0f0f0f0f0f0f
62	movabsq $0x0101010101010101, %rdx
63	imulq   %rdx, %rax                      # w_tmp *= 0x0101010101010101
64	shrq    $56, %rax                       # w = w_tmp >> 56
65
66	popq    %rdx
67	popq    %rdi
68	ret
69#else /* CONFIG_X86_32 */
70	/* We're getting an u64 arg in (%eax,%edx): unsigned long hweight64(__u64 w) */
71	pushl   %ecx
72
73	call    __sw_hweight32
74	movl    %eax, %ecx                      # stash away result
75	movl    %edx, %eax                      # second part of input
76	call    __sw_hweight32
77	addl    %ecx, %eax                      # result
78
79	popl    %ecx
80	ret
81#endif
82ENDPROC(__sw_hweight64)
83EXPORT_SYMBOL(__sw_hweight64)