Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */
3
4#include <linux/export.h>
5#include <linux/linkage.h>
6#include <asm/cpufeatures.h>
7#include <asm/alternative.h>
8
9/*
10 * Some CPUs run faster using the string copy instructions (sane microcode).
11 * It is also a lot simpler. Use this when possible. But, don't use streaming
12 * copy unless the CPU indicates X86_FEATURE_REP_GOOD. Could vary the
13 * prefetch distance based on SMP/UP.
14 */
15 ALIGN
16SYM_FUNC_START(copy_page)
17 ALTERNATIVE "jmp copy_page_regs", "", X86_FEATURE_REP_GOOD
18 movl $4096/8, %ecx
19 rep movsq
20 RET
21SYM_FUNC_END(copy_page)
22EXPORT_SYMBOL(copy_page)
23
24SYM_FUNC_START_LOCAL(copy_page_regs)
25 subq $2*8, %rsp
26 movq %rbx, (%rsp)
27 movq %r12, 1*8(%rsp)
28
29 movl $(4096/64)-5, %ecx
30 .p2align 4
31.Loop64:
32 dec %rcx
33 movq 0x8*0(%rsi), %rax
34 movq 0x8*1(%rsi), %rbx
35 movq 0x8*2(%rsi), %rdx
36 movq 0x8*3(%rsi), %r8
37 movq 0x8*4(%rsi), %r9
38 movq 0x8*5(%rsi), %r10
39 movq 0x8*6(%rsi), %r11
40 movq 0x8*7(%rsi), %r12
41
42 prefetcht0 5*64(%rsi)
43
44 movq %rax, 0x8*0(%rdi)
45 movq %rbx, 0x8*1(%rdi)
46 movq %rdx, 0x8*2(%rdi)
47 movq %r8, 0x8*3(%rdi)
48 movq %r9, 0x8*4(%rdi)
49 movq %r10, 0x8*5(%rdi)
50 movq %r11, 0x8*6(%rdi)
51 movq %r12, 0x8*7(%rdi)
52
53 leaq 64 (%rsi), %rsi
54 leaq 64 (%rdi), %rdi
55
56 jnz .Loop64
57
58 movl $5, %ecx
59 .p2align 4
60.Loop2:
61 decl %ecx
62
63 movq 0x8*0(%rsi), %rax
64 movq 0x8*1(%rsi), %rbx
65 movq 0x8*2(%rsi), %rdx
66 movq 0x8*3(%rsi), %r8
67 movq 0x8*4(%rsi), %r9
68 movq 0x8*5(%rsi), %r10
69 movq 0x8*6(%rsi), %r11
70 movq 0x8*7(%rsi), %r12
71
72 movq %rax, 0x8*0(%rdi)
73 movq %rbx, 0x8*1(%rdi)
74 movq %rdx, 0x8*2(%rdi)
75 movq %r8, 0x8*3(%rdi)
76 movq %r9, 0x8*4(%rdi)
77 movq %r10, 0x8*5(%rdi)
78 movq %r11, 0x8*6(%rdi)
79 movq %r12, 0x8*7(%rdi)
80
81 leaq 64(%rdi), %rdi
82 leaq 64(%rsi), %rsi
83 jnz .Loop2
84
85 movq (%rsp), %rbx
86 movq 1*8(%rsp), %r12
87 addq $2*8, %rsp
88 RET
89SYM_FUNC_END(copy_page_regs)
1/* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */
2
3#include <linux/linkage.h>
4#include <asm/dwarf2.h>
5#include <asm/alternative-asm.h>
6
7 ALIGN
8copy_page_c:
9 CFI_STARTPROC
10 movl $4096/8,%ecx
11 rep movsq
12 ret
13 CFI_ENDPROC
14ENDPROC(copy_page_c)
15
16/* Don't use streaming store because it's better when the target
17 ends up in cache. */
18
19/* Could vary the prefetch distance based on SMP/UP */
20
21ENTRY(copy_page)
22 CFI_STARTPROC
23 subq $2*8,%rsp
24 CFI_ADJUST_CFA_OFFSET 2*8
25 movq %rbx,(%rsp)
26 CFI_REL_OFFSET rbx, 0
27 movq %r12,1*8(%rsp)
28 CFI_REL_OFFSET r12, 1*8
29
30 movl $(4096/64)-5,%ecx
31 .p2align 4
32.Loop64:
33 dec %rcx
34
35 movq (%rsi), %rax
36 movq 8 (%rsi), %rbx
37 movq 16 (%rsi), %rdx
38 movq 24 (%rsi), %r8
39 movq 32 (%rsi), %r9
40 movq 40 (%rsi), %r10
41 movq 48 (%rsi), %r11
42 movq 56 (%rsi), %r12
43
44 prefetcht0 5*64(%rsi)
45
46 movq %rax, (%rdi)
47 movq %rbx, 8 (%rdi)
48 movq %rdx, 16 (%rdi)
49 movq %r8, 24 (%rdi)
50 movq %r9, 32 (%rdi)
51 movq %r10, 40 (%rdi)
52 movq %r11, 48 (%rdi)
53 movq %r12, 56 (%rdi)
54
55 leaq 64 (%rsi), %rsi
56 leaq 64 (%rdi), %rdi
57
58 jnz .Loop64
59
60 movl $5,%ecx
61 .p2align 4
62.Loop2:
63 decl %ecx
64
65 movq (%rsi), %rax
66 movq 8 (%rsi), %rbx
67 movq 16 (%rsi), %rdx
68 movq 24 (%rsi), %r8
69 movq 32 (%rsi), %r9
70 movq 40 (%rsi), %r10
71 movq 48 (%rsi), %r11
72 movq 56 (%rsi), %r12
73
74 movq %rax, (%rdi)
75 movq %rbx, 8 (%rdi)
76 movq %rdx, 16 (%rdi)
77 movq %r8, 24 (%rdi)
78 movq %r9, 32 (%rdi)
79 movq %r10, 40 (%rdi)
80 movq %r11, 48 (%rdi)
81 movq %r12, 56 (%rdi)
82
83 leaq 64(%rdi),%rdi
84 leaq 64(%rsi),%rsi
85
86 jnz .Loop2
87
88 movq (%rsp),%rbx
89 CFI_RESTORE rbx
90 movq 1*8(%rsp),%r12
91 CFI_RESTORE r12
92 addq $2*8,%rsp
93 CFI_ADJUST_CFA_OFFSET -2*8
94 ret
95.Lcopy_page_end:
96 CFI_ENDPROC
97ENDPROC(copy_page)
98
99 /* Some CPUs run faster using the string copy instructions.
100 It is also a lot simpler. Use this when possible */
101
102#include <asm/cpufeature.h>
103
104 .section .altinstr_replacement,"ax"
1051: .byte 0xeb /* jmp <disp8> */
106 .byte (copy_page_c - copy_page) - (2f - 1b) /* offset */
1072:
108 .previous
109 .section .altinstructions,"a"
110 altinstruction_entry copy_page, 1b, X86_FEATURE_REP_GOOD, \
111 .Lcopy_page_end-copy_page, 2b-1b
112 .previous