Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2008 Mark Nelson, IBM Corp.
4 */
5#include <linux/export.h>
6#include <asm/page.h>
7#include <asm/processor.h>
8#include <asm/ppc_asm.h>
9#include <asm/asm-offsets.h>
10#include <asm/feature-fixups.h>
11
12_GLOBAL_TOC(copy_page)
13BEGIN_FTR_SECTION
14 lis r5,PAGE_SIZE@h
15FTR_SECTION_ELSE
16#ifdef CONFIG_PPC_BOOK3S_64
17 b copypage_power7
18#endif
19ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
20 ori r5,r5,PAGE_SIZE@l
21#ifdef CONFIG_PPC_KERNEL_PCREL
22 /*
23 * Hack for toolchain - prefixed instructions cause label difference to
24 * be non-constant even if 8 byte alignment is known, so they can not
25 * be put in FTR sections.
26 */
27 LOAD_REG_ADDR(r10, ppc64_caches)
28BEGIN_FTR_SECTION
29#else
30BEGIN_FTR_SECTION
31 LOAD_REG_ADDR(r10, ppc64_caches)
32#endif
33 lwz r11,DCACHEL1LOGBLOCKSIZE(r10) /* log2 of cache block size */
34 lwz r12,DCACHEL1BLOCKSIZE(r10) /* get cache block size */
35 li r9,0
36 srd r8,r5,r11
37
38 mtctr r8
39.Lsetup:
40 dcbt r9,r4
41 dcbz r9,r3
42 add r9,r9,r12
43 bdnz .Lsetup
44END_FTR_SECTION_IFSET(CPU_FTR_CP_USE_DCBTZ)
45 addi r3,r3,-8
46 srdi r8,r5,7 /* page is copied in 128 byte strides */
47 addi r8,r8,-1 /* one stride copied outside loop */
48
49 mtctr r8
50
51 ld r5,0(r4)
52 ld r6,8(r4)
53 ld r7,16(r4)
54 ldu r8,24(r4)
551: std r5,8(r3)
56 std r6,16(r3)
57 ld r9,8(r4)
58 ld r10,16(r4)
59 std r7,24(r3)
60 std r8,32(r3)
61 ld r11,24(r4)
62 ld r12,32(r4)
63 std r9,40(r3)
64 std r10,48(r3)
65 ld r5,40(r4)
66 ld r6,48(r4)
67 std r11,56(r3)
68 std r12,64(r3)
69 ld r7,56(r4)
70 ld r8,64(r4)
71 std r5,72(r3)
72 std r6,80(r3)
73 ld r9,72(r4)
74 ld r10,80(r4)
75 std r7,88(r3)
76 std r8,96(r3)
77 ld r11,88(r4)
78 ld r12,96(r4)
79 std r9,104(r3)
80 std r10,112(r3)
81 ld r5,104(r4)
82 ld r6,112(r4)
83 std r11,120(r3)
84 stdu r12,128(r3)
85 ld r7,120(r4)
86 ldu r8,128(r4)
87 bdnz 1b
88
89 std r5,8(r3)
90 std r6,16(r3)
91 ld r9,8(r4)
92 ld r10,16(r4)
93 std r7,24(r3)
94 std r8,32(r3)
95 ld r11,24(r4)
96 ld r12,32(r4)
97 std r9,40(r3)
98 std r10,48(r3)
99 ld r5,40(r4)
100 ld r6,48(r4)
101 std r11,56(r3)
102 std r12,64(r3)
103 ld r7,56(r4)
104 ld r8,64(r4)
105 std r5,72(r3)
106 std r6,80(r3)
107 ld r9,72(r4)
108 ld r10,80(r4)
109 std r7,88(r3)
110 std r8,96(r3)
111 ld r11,88(r4)
112 ld r12,96(r4)
113 std r9,104(r3)
114 std r10,112(r3)
115 std r11,120(r3)
116 std r12,128(r3)
117 blr
118EXPORT_SYMBOL(copy_page)
1/*
2 * Copyright (C) 2008 Mark Nelson, IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9#include <asm/page.h>
10#include <asm/processor.h>
11#include <asm/ppc_asm.h>
12#include <asm/asm-offsets.h>
13
14 .section ".toc","aw"
15PPC64_CACHES:
16 .tc ppc64_caches[TC],ppc64_caches
17 .section ".text"
18
19_GLOBAL_TOC(copy_page)
20BEGIN_FTR_SECTION
21 lis r5,PAGE_SIZE@h
22FTR_SECTION_ELSE
23 b copypage_power7
24ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
25 ori r5,r5,PAGE_SIZE@l
26BEGIN_FTR_SECTION
27 ld r10,PPC64_CACHES@toc(r2)
28 lwz r11,DCACHEL1LOGLINESIZE(r10) /* log2 of cache line size */
29 lwz r12,DCACHEL1LINESIZE(r10) /* get cache line size */
30 li r9,0
31 srd r8,r5,r11
32
33 mtctr r8
34.Lsetup:
35 dcbt r9,r4
36 dcbz r9,r3
37 add r9,r9,r12
38 bdnz .Lsetup
39END_FTR_SECTION_IFSET(CPU_FTR_CP_USE_DCBTZ)
40 addi r3,r3,-8
41 srdi r8,r5,7 /* page is copied in 128 byte strides */
42 addi r8,r8,-1 /* one stride copied outside loop */
43
44 mtctr r8
45
46 ld r5,0(r4)
47 ld r6,8(r4)
48 ld r7,16(r4)
49 ldu r8,24(r4)
501: std r5,8(r3)
51 std r6,16(r3)
52 ld r9,8(r4)
53 ld r10,16(r4)
54 std r7,24(r3)
55 std r8,32(r3)
56 ld r11,24(r4)
57 ld r12,32(r4)
58 std r9,40(r3)
59 std r10,48(r3)
60 ld r5,40(r4)
61 ld r6,48(r4)
62 std r11,56(r3)
63 std r12,64(r3)
64 ld r7,56(r4)
65 ld r8,64(r4)
66 std r5,72(r3)
67 std r6,80(r3)
68 ld r9,72(r4)
69 ld r10,80(r4)
70 std r7,88(r3)
71 std r8,96(r3)
72 ld r11,88(r4)
73 ld r12,96(r4)
74 std r9,104(r3)
75 std r10,112(r3)
76 ld r5,104(r4)
77 ld r6,112(r4)
78 std r11,120(r3)
79 stdu r12,128(r3)
80 ld r7,120(r4)
81 ldu r8,128(r4)
82 bdnz 1b
83
84 std r5,8(r3)
85 std r6,16(r3)
86 ld r9,8(r4)
87 ld r10,16(r4)
88 std r7,24(r3)
89 std r8,32(r3)
90 ld r11,24(r4)
91 ld r12,32(r4)
92 std r9,40(r3)
93 std r10,48(r3)
94 ld r5,40(r4)
95 ld r6,48(r4)
96 std r11,56(r3)
97 std r12,64(r3)
98 ld r7,56(r4)
99 ld r8,64(r4)
100 std r5,72(r3)
101 std r6,80(r3)
102 ld r9,72(r4)
103 ld r10,80(r4)
104 std r7,88(r3)
105 std r8,96(r3)
106 ld r11,88(r4)
107 ld r12,96(r4)
108 std r9,104(r3)
109 std r10,112(r3)
110 std r11,120(r3)
111 std r12,128(r3)
112 blr