Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * String handling functions.
4 *
5 * Copyright IBM Corp. 2012
6 */
7
8#include <linux/linkage.h>
9#include <asm/export.h>
10#include <asm/nospec-insn.h>
11
12 GEN_BR_THUNK %r14
13
14/*
15 * void *memmove(void *dest, const void *src, size_t n)
16 */
17WEAK(memmove)
18ENTRY(__memmove)
19 ltgr %r4,%r4
20 lgr %r1,%r2
21 jz .Lmemmove_exit
22 aghi %r4,-1
23 clgr %r2,%r3
24 jnh .Lmemmove_forward
25 la %r5,1(%r4,%r3)
26 clgr %r2,%r5
27 jl .Lmemmove_reverse
28.Lmemmove_forward:
29 srlg %r0,%r4,8
30 ltgr %r0,%r0
31 jz .Lmemmove_forward_remainder
32.Lmemmove_forward_loop:
33 mvc 0(256,%r1),0(%r3)
34 la %r1,256(%r1)
35 la %r3,256(%r3)
36 brctg %r0,.Lmemmove_forward_loop
37.Lmemmove_forward_remainder:
38 larl %r5,.Lmemmove_mvc
39 ex %r4,0(%r5)
40.Lmemmove_exit:
41 BR_EX %r14
42.Lmemmove_reverse:
43 ic %r0,0(%r4,%r3)
44 stc %r0,0(%r4,%r1)
45 brctg %r4,.Lmemmove_reverse
46 ic %r0,0(%r4,%r3)
47 stc %r0,0(%r4,%r1)
48 BR_EX %r14
49.Lmemmove_mvc:
50 mvc 0(1,%r1),0(%r3)
51ENDPROC(__memmove)
52EXPORT_SYMBOL(memmove)
53
54/*
55 * memset implementation
56 *
57 * This code corresponds to the C construct below. We do distinguish
58 * between clearing (c == 0) and setting a memory array (c != 0) simply
59 * because nearly all memset invocations in the kernel clear memory and
60 * the xc instruction is preferred in such cases.
61 *
62 * void *memset(void *s, int c, size_t n)
63 * {
64 * if (likely(c == 0))
65 * return __builtin_memset(s, 0, n);
66 * return __builtin_memset(s, c, n);
67 * }
68 */
69WEAK(memset)
70ENTRY(__memset)
71 ltgr %r4,%r4
72 jz .Lmemset_exit
73 ltgr %r3,%r3
74 jnz .Lmemset_fill
75 aghi %r4,-1
76 srlg %r3,%r4,8
77 ltgr %r3,%r3
78 lgr %r1,%r2
79 jz .Lmemset_clear_remainder
80.Lmemset_clear_loop:
81 xc 0(256,%r1),0(%r1)
82 la %r1,256(%r1)
83 brctg %r3,.Lmemset_clear_loop
84.Lmemset_clear_remainder:
85 larl %r3,.Lmemset_xc
86 ex %r4,0(%r3)
87.Lmemset_exit:
88 BR_EX %r14
89.Lmemset_fill:
90 cghi %r4,1
91 lgr %r1,%r2
92 je .Lmemset_fill_exit
93 aghi %r4,-2
94 srlg %r5,%r4,8
95 ltgr %r5,%r5
96 jz .Lmemset_fill_remainder
97.Lmemset_fill_loop:
98 stc %r3,0(%r1)
99 mvc 1(255,%r1),0(%r1)
100 la %r1,256(%r1)
101 brctg %r5,.Lmemset_fill_loop
102.Lmemset_fill_remainder:
103 stc %r3,0(%r1)
104 larl %r5,.Lmemset_mvc
105 ex %r4,0(%r5)
106 BR_EX %r14
107.Lmemset_fill_exit:
108 stc %r3,0(%r1)
109 BR_EX %r14
110.Lmemset_xc:
111 xc 0(1,%r1),0(%r1)
112.Lmemset_mvc:
113 mvc 1(1,%r1),0(%r1)
114ENDPROC(__memset)
115EXPORT_SYMBOL(memset)
116
117/*
118 * memcpy implementation
119 *
120 * void *memcpy(void *dest, const void *src, size_t n)
121 */
122WEAK(memcpy)
123ENTRY(__memcpy)
124 ltgr %r4,%r4
125 jz .Lmemcpy_exit
126 aghi %r4,-1
127 srlg %r5,%r4,8
128 ltgr %r5,%r5
129 lgr %r1,%r2
130 jnz .Lmemcpy_loop
131.Lmemcpy_remainder:
132 larl %r5,.Lmemcpy_mvc
133 ex %r4,0(%r5)
134.Lmemcpy_exit:
135 BR_EX %r14
136.Lmemcpy_loop:
137 mvc 0(256,%r1),0(%r3)
138 la %r1,256(%r1)
139 la %r3,256(%r3)
140 brctg %r5,.Lmemcpy_loop
141 j .Lmemcpy_remainder
142.Lmemcpy_mvc:
143 mvc 0(1,%r1),0(%r3)
144ENDPROC(__memcpy)
145EXPORT_SYMBOL(memcpy)
146
147/*
148 * __memset16/32/64
149 *
150 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
151 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
152 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
153 */
154.macro __MEMSET bits,bytes,insn
155ENTRY(__memset\bits)
156 ltgr %r4,%r4
157 jz .L__memset_exit\bits
158 cghi %r4,\bytes
159 je .L__memset_store\bits
160 aghi %r4,-(\bytes+1)
161 srlg %r5,%r4,8
162 ltgr %r5,%r5
163 lgr %r1,%r2
164 jz .L__memset_remainder\bits
165.L__memset_loop\bits:
166 \insn %r3,0(%r1)
167 mvc \bytes(256-\bytes,%r1),0(%r1)
168 la %r1,256(%r1)
169 brctg %r5,.L__memset_loop\bits
170.L__memset_remainder\bits:
171 \insn %r3,0(%r1)
172 larl %r5,.L__memset_mvc\bits
173 ex %r4,0(%r5)
174 BR_EX %r14
175.L__memset_store\bits:
176 \insn %r3,0(%r2)
177.L__memset_exit\bits:
178 BR_EX %r14
179.L__memset_mvc\bits:
180 mvc \bytes(1,%r1),0(%r1)
181ENDPROC(__memset\bits)
182.endm
183
184__MEMSET 16,2,sth
185EXPORT_SYMBOL(__memset16)
186
187__MEMSET 32,4,st
188EXPORT_SYMBOL(__memset32)
189
190__MEMSET 64,8,stg
191EXPORT_SYMBOL(__memset64)
1/*
2 * String handling functions.
3 *
4 * Copyright IBM Corp. 2012
5 */
6
7#include <linux/linkage.h>
8#include <asm/export.h>
9
10/*
11 * void *memmove(void *dest, const void *src, size_t n)
12 */
13ENTRY(memmove)
14 ltgr %r4,%r4
15 lgr %r1,%r2
16 bzr %r14
17 clgr %r2,%r3
18 jnh .Lmemmove_forward
19 la %r5,0(%r4,%r3)
20 clgr %r2,%r5
21 jl .Lmemmove_reverse
22.Lmemmove_forward:
23 aghi %r4,-1
24 srlg %r0,%r4,8
25 ltgr %r0,%r0
26 jz .Lmemmove_rest
27.Lmemmove_loop:
28 mvc 0(256,%r1),0(%r3)
29 la %r1,256(%r1)
30 la %r3,256(%r3)
31 brctg %r0,.Lmemmove_loop
32.Lmemmove_rest:
33 larl %r5,.Lmemmove_mvc
34 ex %r4,0(%r5)
35 br %r14
36.Lmemmove_reverse:
37 aghi %r4,-1
38.Lmemmove_reverse_loop:
39 ic %r0,0(%r4,%r3)
40 stc %r0,0(%r4,%r1)
41 brctg %r4,.Lmemmove_reverse_loop
42 ic %r0,0(%r4,%r3)
43 stc %r0,0(%r4,%r1)
44 br %r14
45.Lmemmove_mvc:
46 mvc 0(1,%r1),0(%r3)
47EXPORT_SYMBOL(memmove)
48
49/*
50 * memset implementation
51 *
52 * This code corresponds to the C construct below. We do distinguish
53 * between clearing (c == 0) and setting a memory array (c != 0) simply
54 * because nearly all memset invocations in the kernel clear memory and
55 * the xc instruction is preferred in such cases.
56 *
57 * void *memset(void *s, int c, size_t n)
58 * {
59 * if (likely(c == 0))
60 * return __builtin_memset(s, 0, n);
61 * return __builtin_memset(s, c, n);
62 * }
63 */
64ENTRY(memset)
65 ltgr %r4,%r4
66 bzr %r14
67 ltgr %r3,%r3
68 jnz .Lmemset_fill
69 aghi %r4,-1
70 srlg %r3,%r4,8
71 ltgr %r3,%r3
72 lgr %r1,%r2
73 jz .Lmemset_clear_rest
74.Lmemset_clear_loop:
75 xc 0(256,%r1),0(%r1)
76 la %r1,256(%r1)
77 brctg %r3,.Lmemset_clear_loop
78.Lmemset_clear_rest:
79 larl %r3,.Lmemset_xc
80 ex %r4,0(%r3)
81 br %r14
82.Lmemset_fill:
83 stc %r3,0(%r2)
84 cghi %r4,1
85 lgr %r1,%r2
86 ber %r14
87 aghi %r4,-2
88 srlg %r3,%r4,8
89 ltgr %r3,%r3
90 jz .Lmemset_fill_rest
91.Lmemset_fill_loop:
92 mvc 1(256,%r1),0(%r1)
93 la %r1,256(%r1)
94 brctg %r3,.Lmemset_fill_loop
95.Lmemset_fill_rest:
96 larl %r3,.Lmemset_mvc
97 ex %r4,0(%r3)
98 br %r14
99.Lmemset_xc:
100 xc 0(1,%r1),0(%r1)
101.Lmemset_mvc:
102 mvc 1(1,%r1),0(%r1)
103EXPORT_SYMBOL(memset)
104
105/*
106 * memcpy implementation
107 *
108 * void *memcpy(void *dest, const void *src, size_t n)
109 */
110ENTRY(memcpy)
111 ltgr %r4,%r4
112 bzr %r14
113 aghi %r4,-1
114 srlg %r5,%r4,8
115 ltgr %r5,%r5
116 lgr %r1,%r2
117 jnz .Lmemcpy_loop
118.Lmemcpy_rest:
119 larl %r5,.Lmemcpy_mvc
120 ex %r4,0(%r5)
121 br %r14
122.Lmemcpy_loop:
123 mvc 0(256,%r1),0(%r3)
124 la %r1,256(%r1)
125 la %r3,256(%r3)
126 brctg %r5,.Lmemcpy_loop
127 j .Lmemcpy_rest
128.Lmemcpy_mvc:
129 mvc 0(1,%r1),0(%r3)
130EXPORT_SYMBOL(memcpy)