Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *
4 * Copyright (C) IBM Corporation, 2012
5 *
6 * Author: Anton Blanchard <anton@au.ibm.com>
7 */
8
9#include <linux/export.h>
10#include <asm/ppc_asm.h>
11#include <asm/linkage.h>
12#include <asm/asm-offsets.h>
13
14/**
15 * __arch_clear_user: - Zero a block of memory in user space, with less checking.
16 * @to: Destination address, in user space.
17 * @n: Number of bytes to zero.
18 *
19 * Zero a block of memory in user space. Caller must check
20 * the specified block with access_ok() before calling this function.
21 *
22 * Returns number of bytes that could not be cleared.
23 * On success, this will be zero.
24 */
25
26 .macro err1
27100:
28 EX_TABLE(100b,.Ldo_err1)
29 .endm
30
31 .macro err2
32200:
33 EX_TABLE(200b,.Ldo_err2)
34 .endm
35
36 .macro err3
37300:
38 EX_TABLE(300b,.Ldo_err3)
39 .endm
40
41.Ldo_err1:
42 mr r3,r8
43
44.Ldo_err2:
45 mtctr r4
461:
47err3; stb r0,0(r3)
48 addi r3,r3,1
49 addi r4,r4,-1
50 bdnz 1b
51
52.Ldo_err3:
53 mr r3,r4
54 blr
55
56_GLOBAL_TOC(__arch_clear_user)
57 cmpdi r4,32
58 neg r6,r3
59 li r0,0
60 blt .Lshort_clear
61 mr r8,r3
62 mtocrf 0x01,r6
63 clrldi r6,r6,(64-3)
64
65 /* Get the destination 8 byte aligned */
66 bf cr7*4+3,1f
67err1; stb r0,0(r3)
68 addi r3,r3,1
69
701: bf cr7*4+2,2f
71err1; sth r0,0(r3)
72 addi r3,r3,2
73
742: bf cr7*4+1,3f
75err1; stw r0,0(r3)
76 addi r3,r3,4
77
783: sub r4,r4,r6
79
80 cmpdi r4,32
81 cmpdi cr1,r4,512
82 blt .Lshort_clear
83 bgt cr1,.Llong_clear
84
85.Lmedium_clear:
86 srdi r6,r4,5
87 mtctr r6
88
89 /* Do 32 byte chunks */
904:
91err2; std r0,0(r3)
92err2; std r0,8(r3)
93err2; std r0,16(r3)
94err2; std r0,24(r3)
95 addi r3,r3,32
96 addi r4,r4,-32
97 bdnz 4b
98
99.Lshort_clear:
100 /* up to 31 bytes to go */
101 cmpdi r4,16
102 blt 6f
103err2; std r0,0(r3)
104err2; std r0,8(r3)
105 addi r3,r3,16
106 addi r4,r4,-16
107
108 /* Up to 15 bytes to go */
1096: mr r8,r3
110 clrldi r4,r4,(64-4)
111 mtocrf 0x01,r4
112 bf cr7*4+0,7f
113err1; std r0,0(r3)
114 addi r3,r3,8
115
1167: bf cr7*4+1,8f
117err1; stw r0,0(r3)
118 addi r3,r3,4
119
1208: bf cr7*4+2,9f
121err1; sth r0,0(r3)
122 addi r3,r3,2
123
1249: bf cr7*4+3,10f
125err1; stb r0,0(r3)
126
12710: li r3,0
128 blr
129
130.Llong_clear:
131 LOAD_REG_ADDR(r5, ppc64_caches)
132
133 bf cr7*4+0,11f
134err2; std r0,0(r3)
135 addi r3,r3,8
136 addi r4,r4,-8
137
138 /* Destination is 16 byte aligned, need to get it cache block aligned */
13911: lwz r7,DCACHEL1LOGBLOCKSIZE(r5)
140 lwz r9,DCACHEL1BLOCKSIZE(r5)
141
142 /*
143 * With worst case alignment the long clear loop takes a minimum
144 * of 1 byte less than 2 cachelines.
145 */
146 sldi r10,r9,2
147 cmpd r4,r10
148 blt .Lmedium_clear
149
150 neg r6,r3
151 addi r10,r9,-1
152 and. r5,r6,r10
153 beq 13f
154
155 srdi r6,r5,4
156 mtctr r6
157 mr r8,r3
15812:
159err1; std r0,0(r3)
160err1; std r0,8(r3)
161 addi r3,r3,16
162 bdnz 12b
163
164 sub r4,r4,r5
165
16613: srd r6,r4,r7
167 mtctr r6
168 mr r8,r3
16914:
170err1; dcbz 0,r3
171 add r3,r3,r9
172 bdnz 14b
173
174 and r4,r4,r10
175
176 cmpdi r4,32
177 blt .Lshort_clear
178 b .Lmedium_clear
179EXPORT_SYMBOL(__arch_clear_user)
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *
4 * Copyright (C) IBM Corporation, 2012
5 *
6 * Author: Anton Blanchard <anton@au.ibm.com>
7 */
8
9#include <linux/export.h>
10#include <asm/ppc_asm.h>
11#include <asm/linkage.h>
12#include <asm/asm-offsets.h>
13
14/**
15 * __arch_clear_user: - Zero a block of memory in user space, with less checking.
16 * @to: Destination address, in user space.
17 * @n: Number of bytes to zero.
18 *
19 * Zero a block of memory in user space. Caller must check
20 * the specified block with access_ok() before calling this function.
21 *
22 * Returns number of bytes that could not be cleared.
23 * On success, this will be zero.
24 */
25
26 .macro err1
27100:
28 EX_TABLE(100b,.Ldo_err1)
29 .endm
30
31 .macro err2
32200:
33 EX_TABLE(200b,.Ldo_err2)
34 .endm
35
36 .macro err3
37300:
38 EX_TABLE(300b,.Ldo_err3)
39 .endm
40
41.Ldo_err1:
42 mr r3,r8
43
44.Ldo_err2:
45 mtctr r4
461:
47err3; stb r0,0(r3)
48 addi r3,r3,1
49 addi r4,r4,-1
50 bdnz 1b
51
52.Ldo_err3:
53 mr r3,r4
54 blr
55
56_GLOBAL_TOC(__arch_clear_user)
57 cmpdi r4,32
58 neg r6,r3
59 li r0,0
60 blt .Lshort_clear
61 mr r8,r3
62 mtocrf 0x01,r6
63 clrldi r6,r6,(64-3)
64
65 /* Get the destination 8 byte aligned */
66 bf cr7*4+3,1f
67err1; stb r0,0(r3)
68 addi r3,r3,1
69
701: bf cr7*4+2,2f
71err1; sth r0,0(r3)
72 addi r3,r3,2
73
742: bf cr7*4+1,3f
75err1; stw r0,0(r3)
76 addi r3,r3,4
77
783: sub r4,r4,r6
79
80 cmpdi r4,32
81 cmpdi cr1,r4,512
82 blt .Lshort_clear
83 bgt cr1,.Llong_clear
84
85.Lmedium_clear:
86 srdi r6,r4,5
87 mtctr r6
88
89 /* Do 32 byte chunks */
904:
91err2; std r0,0(r3)
92err2; std r0,8(r3)
93err2; std r0,16(r3)
94err2; std r0,24(r3)
95 addi r3,r3,32
96 addi r4,r4,-32
97 bdnz 4b
98
99.Lshort_clear:
100 /* up to 31 bytes to go */
101 cmpdi r4,16
102 blt 6f
103err2; std r0,0(r3)
104err2; std r0,8(r3)
105 addi r3,r3,16
106 addi r4,r4,-16
107
108 /* Up to 15 bytes to go */
1096: mr r8,r3
110 clrldi r4,r4,(64-4)
111 mtocrf 0x01,r4
112 bf cr7*4+0,7f
113err1; std r0,0(r3)
114 addi r3,r3,8
115
1167: bf cr7*4+1,8f
117err1; stw r0,0(r3)
118 addi r3,r3,4
119
1208: bf cr7*4+2,9f
121err1; sth r0,0(r3)
122 addi r3,r3,2
123
1249: bf cr7*4+3,10f
125err1; stb r0,0(r3)
126
12710: li r3,0
128 blr
129
130.Llong_clear:
131 LOAD_REG_ADDR(r5, ppc64_caches)
132
133 bf cr7*4+0,11f
134err2; std r0,0(r3)
135 addi r3,r3,8
136 addi r4,r4,-8
137
138 /* Destination is 16 byte aligned, need to get it cache block aligned */
13911: lwz r7,DCACHEL1LOGBLOCKSIZE(r5)
140 lwz r9,DCACHEL1BLOCKSIZE(r5)
141
142 /*
143 * With worst case alignment the long clear loop takes a minimum
144 * of 1 byte less than 2 cachelines.
145 */
146 sldi r10,r9,2
147 cmpd r4,r10
148 blt .Lmedium_clear
149
150 neg r6,r3
151 addi r10,r9,-1
152 and. r5,r6,r10
153 beq 13f
154
155 srdi r6,r5,4
156 mtctr r6
157 mr r8,r3
15812:
159err1; std r0,0(r3)
160err1; std r0,8(r3)
161 addi r3,r3,16
162 bdnz 12b
163
164 sub r4,r4,r5
165
16613: srd r6,r4,r7
167 mtctr r6
168 mr r8,r3
16914:
170err1; dcbz 0,r3
171 add r3,r3,r9
172 bdnz 14b
173
174 and r4,r4,r10
175
176 cmpdi r4,32
177 blt .Lshort_clear
178 b .Lmedium_clear
179EXPORT_SYMBOL(__arch_clear_user)