Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/* Copyright 2002 Andi Kleen */
3
4#include <linux/export.h>
5#include <linux/linkage.h>
6#include <linux/cfi_types.h>
7#include <asm/errno.h>
8#include <asm/cpufeatures.h>
9#include <asm/alternative.h>
10
11.section .noinstr.text, "ax"
12
13/*
14 * memcpy - Copy a memory block.
15 *
16 * Input:
17 * rdi destination
18 * rsi source
19 * rdx count
20 *
21 * Output:
22 * rax original destination
23 *
24 * The FSRM alternative should be done inline (avoiding the call and
25 * the disgusting return handling), but that would require some help
26 * from the compiler for better calling conventions.
27 *
28 * The 'rep movsb' itself is small enough to replace the call, but the
29 * two register moves blow up the code. And one of them is "needed"
30 * only for the return value that is the same as the source input,
31 * which the compiler could/should do much better anyway.
32 */
33SYM_TYPED_FUNC_START(__memcpy)
34 ALTERNATIVE "jmp memcpy_orig", "", X86_FEATURE_FSRM
35
36 movq %rdi, %rax
37 movq %rdx, %rcx
38 rep movsb
39 RET
40SYM_FUNC_END(__memcpy)
41EXPORT_SYMBOL(__memcpy)
42
43SYM_FUNC_ALIAS_MEMFUNC(memcpy, __memcpy)
44EXPORT_SYMBOL(memcpy)
45
46SYM_FUNC_START_LOCAL(memcpy_orig)
47 movq %rdi, %rax
48
49 cmpq $0x20, %rdx
50 jb .Lhandle_tail
51
52 /*
53 * We check whether memory false dependence could occur,
54 * then jump to corresponding copy mode.
55 */
56 cmp %dil, %sil
57 jl .Lcopy_backward
58 subq $0x20, %rdx
59.Lcopy_forward_loop:
60 subq $0x20, %rdx
61
62 /*
63 * Move in blocks of 4x8 bytes:
64 */
65 movq 0*8(%rsi), %r8
66 movq 1*8(%rsi), %r9
67 movq 2*8(%rsi), %r10
68 movq 3*8(%rsi), %r11
69 leaq 4*8(%rsi), %rsi
70
71 movq %r8, 0*8(%rdi)
72 movq %r9, 1*8(%rdi)
73 movq %r10, 2*8(%rdi)
74 movq %r11, 3*8(%rdi)
75 leaq 4*8(%rdi), %rdi
76 jae .Lcopy_forward_loop
77 addl $0x20, %edx
78 jmp .Lhandle_tail
79
80.Lcopy_backward:
81 /*
82 * Calculate copy position to tail.
83 */
84 addq %rdx, %rsi
85 addq %rdx, %rdi
86 subq $0x20, %rdx
87 /*
88 * At most 3 ALU operations in one cycle,
89 * so append NOPS in the same 16 bytes trunk.
90 */
91 .p2align 4
92.Lcopy_backward_loop:
93 subq $0x20, %rdx
94 movq -1*8(%rsi), %r8
95 movq -2*8(%rsi), %r9
96 movq -3*8(%rsi), %r10
97 movq -4*8(%rsi), %r11
98 leaq -4*8(%rsi), %rsi
99 movq %r8, -1*8(%rdi)
100 movq %r9, -2*8(%rdi)
101 movq %r10, -3*8(%rdi)
102 movq %r11, -4*8(%rdi)
103 leaq -4*8(%rdi), %rdi
104 jae .Lcopy_backward_loop
105
106 /*
107 * Calculate copy position to head.
108 */
109 addl $0x20, %edx
110 subq %rdx, %rsi
111 subq %rdx, %rdi
112.Lhandle_tail:
113 cmpl $16, %edx
114 jb .Lless_16bytes
115
116 /*
117 * Move data from 16 bytes to 31 bytes.
118 */
119 movq 0*8(%rsi), %r8
120 movq 1*8(%rsi), %r9
121 movq -2*8(%rsi, %rdx), %r10
122 movq -1*8(%rsi, %rdx), %r11
123 movq %r8, 0*8(%rdi)
124 movq %r9, 1*8(%rdi)
125 movq %r10, -2*8(%rdi, %rdx)
126 movq %r11, -1*8(%rdi, %rdx)
127 RET
128 .p2align 4
129.Lless_16bytes:
130 cmpl $8, %edx
131 jb .Lless_8bytes
132 /*
133 * Move data from 8 bytes to 15 bytes.
134 */
135 movq 0*8(%rsi), %r8
136 movq -1*8(%rsi, %rdx), %r9
137 movq %r8, 0*8(%rdi)
138 movq %r9, -1*8(%rdi, %rdx)
139 RET
140 .p2align 4
141.Lless_8bytes:
142 cmpl $4, %edx
143 jb .Lless_3bytes
144
145 /*
146 * Move data from 4 bytes to 7 bytes.
147 */
148 movl (%rsi), %ecx
149 movl -4(%rsi, %rdx), %r8d
150 movl %ecx, (%rdi)
151 movl %r8d, -4(%rdi, %rdx)
152 RET
153 .p2align 4
154.Lless_3bytes:
155 subl $1, %edx
156 jb .Lend
157 /*
158 * Move data from 1 bytes to 3 bytes.
159 */
160 movzbl (%rsi), %ecx
161 jz .Lstore_1byte
162 movzbq 1(%rsi), %r8
163 movzbq (%rsi, %rdx), %r9
164 movb %r8b, 1(%rdi)
165 movb %r9b, (%rdi, %rdx)
166.Lstore_1byte:
167 movb %cl, (%rdi)
168
169.Lend:
170 RET
171SYM_FUNC_END(memcpy_orig)
172
1/* Copyright 2002 Andi Kleen */
2
3#include <linux/linkage.h>
4
5#include <asm/cpufeature.h>
6#include <asm/dwarf2.h>
7#include <asm/alternative-asm.h>
8
9/*
10 * memcpy - Copy a memory block.
11 *
12 * Input:
13 * rdi destination
14 * rsi source
15 * rdx count
16 *
17 * Output:
18 * rax original destination
19 */
20
21/*
22 * memcpy_c() - fast string ops (REP MOVSQ) based variant.
23 *
24 * This gets patched over the unrolled variant (below) via the
25 * alternative instructions framework:
26 */
27 .section .altinstr_replacement, "ax", @progbits
28.Lmemcpy_c:
29 movq %rdi, %rax
30 movq %rdx, %rcx
31 shrq $3, %rcx
32 andl $7, %edx
33 rep movsq
34 movl %edx, %ecx
35 rep movsb
36 ret
37.Lmemcpy_e:
38 .previous
39
40/*
41 * memcpy_c_e() - enhanced fast string memcpy. This is faster and simpler than
42 * memcpy_c. Use memcpy_c_e when possible.
43 *
44 * This gets patched over the unrolled variant (below) via the
45 * alternative instructions framework:
46 */
47 .section .altinstr_replacement, "ax", @progbits
48.Lmemcpy_c_e:
49 movq %rdi, %rax
50 movq %rdx, %rcx
51 rep movsb
52 ret
53.Lmemcpy_e_e:
54 .previous
55
56ENTRY(__memcpy)
57ENTRY(memcpy)
58 CFI_STARTPROC
59 movq %rdi, %rax
60
61 cmpq $0x20, %rdx
62 jb .Lhandle_tail
63
64 /*
65 * We check whether memory false dependence could occur,
66 * then jump to corresponding copy mode.
67 */
68 cmp %dil, %sil
69 jl .Lcopy_backward
70 subq $0x20, %rdx
71.Lcopy_forward_loop:
72 subq $0x20, %rdx
73
74 /*
75 * Move in blocks of 4x8 bytes:
76 */
77 movq 0*8(%rsi), %r8
78 movq 1*8(%rsi), %r9
79 movq 2*8(%rsi), %r10
80 movq 3*8(%rsi), %r11
81 leaq 4*8(%rsi), %rsi
82
83 movq %r8, 0*8(%rdi)
84 movq %r9, 1*8(%rdi)
85 movq %r10, 2*8(%rdi)
86 movq %r11, 3*8(%rdi)
87 leaq 4*8(%rdi), %rdi
88 jae .Lcopy_forward_loop
89 addl $0x20, %edx
90 jmp .Lhandle_tail
91
92.Lcopy_backward:
93 /*
94 * Calculate copy position to tail.
95 */
96 addq %rdx, %rsi
97 addq %rdx, %rdi
98 subq $0x20, %rdx
99 /*
100 * At most 3 ALU operations in one cycle,
101 * so append NOPS in the same 16bytes trunk.
102 */
103 .p2align 4
104.Lcopy_backward_loop:
105 subq $0x20, %rdx
106 movq -1*8(%rsi), %r8
107 movq -2*8(%rsi), %r9
108 movq -3*8(%rsi), %r10
109 movq -4*8(%rsi), %r11
110 leaq -4*8(%rsi), %rsi
111 movq %r8, -1*8(%rdi)
112 movq %r9, -2*8(%rdi)
113 movq %r10, -3*8(%rdi)
114 movq %r11, -4*8(%rdi)
115 leaq -4*8(%rdi), %rdi
116 jae .Lcopy_backward_loop
117
118 /*
119 * Calculate copy position to head.
120 */
121 addl $0x20, %edx
122 subq %rdx, %rsi
123 subq %rdx, %rdi
124.Lhandle_tail:
125 cmpl $16, %edx
126 jb .Lless_16bytes
127
128 /*
129 * Move data from 16 bytes to 31 bytes.
130 */
131 movq 0*8(%rsi), %r8
132 movq 1*8(%rsi), %r9
133 movq -2*8(%rsi, %rdx), %r10
134 movq -1*8(%rsi, %rdx), %r11
135 movq %r8, 0*8(%rdi)
136 movq %r9, 1*8(%rdi)
137 movq %r10, -2*8(%rdi, %rdx)
138 movq %r11, -1*8(%rdi, %rdx)
139 retq
140 .p2align 4
141.Lless_16bytes:
142 cmpl $8, %edx
143 jb .Lless_8bytes
144 /*
145 * Move data from 8 bytes to 15 bytes.
146 */
147 movq 0*8(%rsi), %r8
148 movq -1*8(%rsi, %rdx), %r9
149 movq %r8, 0*8(%rdi)
150 movq %r9, -1*8(%rdi, %rdx)
151 retq
152 .p2align 4
153.Lless_8bytes:
154 cmpl $4, %edx
155 jb .Lless_3bytes
156
157 /*
158 * Move data from 4 bytes to 7 bytes.
159 */
160 movl (%rsi), %ecx
161 movl -4(%rsi, %rdx), %r8d
162 movl %ecx, (%rdi)
163 movl %r8d, -4(%rdi, %rdx)
164 retq
165 .p2align 4
166.Lless_3bytes:
167 subl $1, %edx
168 jb .Lend
169 /*
170 * Move data from 1 bytes to 3 bytes.
171 */
172 movzbl (%rsi), %ecx
173 jz .Lstore_1byte
174 movzbq 1(%rsi), %r8
175 movzbq (%rsi, %rdx), %r9
176 movb %r8b, 1(%rdi)
177 movb %r9b, (%rdi, %rdx)
178.Lstore_1byte:
179 movb %cl, (%rdi)
180
181.Lend:
182 retq
183 CFI_ENDPROC
184ENDPROC(memcpy)
185ENDPROC(__memcpy)
186
187 /*
188 * Some CPUs are adding enhanced REP MOVSB/STOSB feature
189 * If the feature is supported, memcpy_c_e() is the first choice.
190 * If enhanced rep movsb copy is not available, use fast string copy
191 * memcpy_c() when possible. This is faster and code is simpler than
192 * original memcpy().
193 * Otherwise, original memcpy() is used.
194 * In .altinstructions section, ERMS feature is placed after REG_GOOD
195 * feature to implement the right patch order.
196 *
197 * Replace only beginning, memcpy is used to apply alternatives,
198 * so it is silly to overwrite itself with nops - reboot is the
199 * only outcome...
200 */
201 .section .altinstructions, "a"
202 altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
203 .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c
204 altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
205 .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e
206 .previous