Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * String handling functions.
4 *
5 * Copyright IBM Corp. 2012
6 */
7
8#include <linux/export.h>
9#include <linux/linkage.h>
10#include <asm/nospec-insn.h>
11
12 GEN_BR_THUNK %r14
13
14/*
15 * void *memmove(void *dest, const void *src, size_t n)
16 */
17SYM_FUNC_START(__memmove)
18 ltgr %r4,%r4
19 lgr %r1,%r2
20 jz .Lmemmove_exit
21 aghi %r4,-1
22 clgr %r2,%r3
23 jnh .Lmemmove_forward
24 la %r5,1(%r4,%r3)
25 clgr %r2,%r5
26 jl .Lmemmove_reverse
27.Lmemmove_forward:
28 srlg %r0,%r4,8
29 ltgr %r0,%r0
30 jz .Lmemmove_forward_remainder
31.Lmemmove_forward_loop:
32 mvc 0(256,%r1),0(%r3)
33 la %r1,256(%r1)
34 la %r3,256(%r3)
35 brctg %r0,.Lmemmove_forward_loop
36.Lmemmove_forward_remainder:
37 larl %r5,.Lmemmove_mvc
38 ex %r4,0(%r5)
39.Lmemmove_exit:
40 BR_EX %r14
41.Lmemmove_reverse:
42 ic %r0,0(%r4,%r3)
43 stc %r0,0(%r4,%r1)
44 brctg %r4,.Lmemmove_reverse
45 ic %r0,0(%r4,%r3)
46 stc %r0,0(%r4,%r1)
47 BR_EX %r14
48.Lmemmove_mvc:
49 mvc 0(1,%r1),0(%r3)
50SYM_FUNC_END(__memmove)
51EXPORT_SYMBOL(__memmove)
52
53SYM_FUNC_ALIAS(memmove, __memmove)
54EXPORT_SYMBOL(memmove)
55
56/*
57 * memset implementation
58 *
59 * This code corresponds to the C construct below. We do distinguish
60 * between clearing (c == 0) and setting a memory array (c != 0) simply
61 * because nearly all memset invocations in the kernel clear memory and
62 * the xc instruction is preferred in such cases.
63 *
64 * void *memset(void *s, int c, size_t n)
65 * {
66 * if (likely(c == 0))
67 * return __builtin_memset(s, 0, n);
68 * return __builtin_memset(s, c, n);
69 * }
70 */
71SYM_FUNC_START(__memset)
72 ltgr %r4,%r4
73 jz .Lmemset_exit
74 ltgr %r3,%r3
75 jnz .Lmemset_fill
76 aghi %r4,-1
77 srlg %r3,%r4,8
78 ltgr %r3,%r3
79 lgr %r1,%r2
80 jz .Lmemset_clear_remainder
81.Lmemset_clear_loop:
82 xc 0(256,%r1),0(%r1)
83 la %r1,256(%r1)
84 brctg %r3,.Lmemset_clear_loop
85.Lmemset_clear_remainder:
86 larl %r3,.Lmemset_xc
87 ex %r4,0(%r3)
88.Lmemset_exit:
89 BR_EX %r14
90.Lmemset_fill:
91 cghi %r4,1
92 lgr %r1,%r2
93 je .Lmemset_fill_exit
94 aghi %r4,-2
95 srlg %r5,%r4,8
96 ltgr %r5,%r5
97 jz .Lmemset_fill_remainder
98.Lmemset_fill_loop:
99 stc %r3,0(%r1)
100 mvc 1(255,%r1),0(%r1)
101 la %r1,256(%r1)
102 brctg %r5,.Lmemset_fill_loop
103.Lmemset_fill_remainder:
104 stc %r3,0(%r1)
105 larl %r5,.Lmemset_mvc
106 ex %r4,0(%r5)
107 BR_EX %r14
108.Lmemset_fill_exit:
109 stc %r3,0(%r1)
110 BR_EX %r14
111.Lmemset_xc:
112 xc 0(1,%r1),0(%r1)
113.Lmemset_mvc:
114 mvc 1(1,%r1),0(%r1)
115SYM_FUNC_END(__memset)
116EXPORT_SYMBOL(__memset)
117
118SYM_FUNC_ALIAS(memset, __memset)
119EXPORT_SYMBOL(memset)
120
121/*
122 * memcpy implementation
123 *
124 * void *memcpy(void *dest, const void *src, size_t n)
125 */
126SYM_FUNC_START(__memcpy)
127 ltgr %r4,%r4
128 jz .Lmemcpy_exit
129 aghi %r4,-1
130 srlg %r5,%r4,8
131 ltgr %r5,%r5
132 lgr %r1,%r2
133 jnz .Lmemcpy_loop
134.Lmemcpy_remainder:
135 larl %r5,.Lmemcpy_mvc
136 ex %r4,0(%r5)
137.Lmemcpy_exit:
138 BR_EX %r14
139.Lmemcpy_loop:
140 mvc 0(256,%r1),0(%r3)
141 la %r1,256(%r1)
142 la %r3,256(%r3)
143 brctg %r5,.Lmemcpy_loop
144 j .Lmemcpy_remainder
145.Lmemcpy_mvc:
146 mvc 0(1,%r1),0(%r3)
147SYM_FUNC_END(__memcpy)
148EXPORT_SYMBOL(__memcpy)
149
150SYM_FUNC_ALIAS(memcpy, __memcpy)
151EXPORT_SYMBOL(memcpy)
152
153/*
154 * __memset16/32/64
155 *
156 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
157 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
158 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
159 */
160.macro __MEMSET bits,bytes,insn
161SYM_FUNC_START(__memset\bits)
162 ltgr %r4,%r4
163 jz .L__memset_exit\bits
164 cghi %r4,\bytes
165 je .L__memset_store\bits
166 aghi %r4,-(\bytes+1)
167 srlg %r5,%r4,8
168 ltgr %r5,%r5
169 lgr %r1,%r2
170 jz .L__memset_remainder\bits
171.L__memset_loop\bits:
172 \insn %r3,0(%r1)
173 mvc \bytes(256-\bytes,%r1),0(%r1)
174 la %r1,256(%r1)
175 brctg %r5,.L__memset_loop\bits
176.L__memset_remainder\bits:
177 \insn %r3,0(%r1)
178 larl %r5,.L__memset_mvc\bits
179 ex %r4,0(%r5)
180 BR_EX %r14
181.L__memset_store\bits:
182 \insn %r3,0(%r2)
183.L__memset_exit\bits:
184 BR_EX %r14
185.L__memset_mvc\bits:
186 mvc \bytes(1,%r1),0(%r1)
187SYM_FUNC_END(__memset\bits)
188.endm
189
190__MEMSET 16,2,sth
191EXPORT_SYMBOL(__memset16)
192
193__MEMSET 32,4,st
194EXPORT_SYMBOL(__memset32)
195
196__MEMSET 64,8,stg
197EXPORT_SYMBOL(__memset64)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * String handling functions.
4 *
5 * Copyright IBM Corp. 2012
6 */
7
8#include <linux/linkage.h>
9#include <asm/export.h>
10#include <asm/nospec-insn.h>
11
12 GEN_BR_THUNK %r14
13
14/*
15 * void *memmove(void *dest, const void *src, size_t n)
16 */
17ENTRY(memmove)
18 ltgr %r4,%r4
19 lgr %r1,%r2
20 bzr %r14
21 aghi %r4,-1
22 clgr %r2,%r3
23 jnh .Lmemmove_forward
24 la %r5,1(%r4,%r3)
25 clgr %r2,%r5
26 jl .Lmemmove_reverse
27.Lmemmove_forward:
28 srlg %r0,%r4,8
29 ltgr %r0,%r0
30 jz .Lmemmove_forward_remainder
31.Lmemmove_forward_loop:
32 mvc 0(256,%r1),0(%r3)
33 la %r1,256(%r1)
34 la %r3,256(%r3)
35 brctg %r0,.Lmemmove_forward_loop
36.Lmemmove_forward_remainder:
37 larl %r5,.Lmemmove_mvc
38 ex %r4,0(%r5)
39 BR_EX %r14
40.Lmemmove_reverse:
41 ic %r0,0(%r4,%r3)
42 stc %r0,0(%r4,%r1)
43 brctg %r4,.Lmemmove_reverse
44 ic %r0,0(%r4,%r3)
45 stc %r0,0(%r4,%r1)
46 BR_EX %r14
47.Lmemmove_mvc:
48 mvc 0(1,%r1),0(%r3)
49EXPORT_SYMBOL(memmove)
50
51/*
52 * memset implementation
53 *
54 * This code corresponds to the C construct below. We do distinguish
55 * between clearing (c == 0) and setting a memory array (c != 0) simply
56 * because nearly all memset invocations in the kernel clear memory and
57 * the xc instruction is preferred in such cases.
58 *
59 * void *memset(void *s, int c, size_t n)
60 * {
61 * if (likely(c == 0))
62 * return __builtin_memset(s, 0, n);
63 * return __builtin_memset(s, c, n);
64 * }
65 */
66ENTRY(memset)
67 ltgr %r4,%r4
68 bzr %r14
69 ltgr %r3,%r3
70 jnz .Lmemset_fill
71 aghi %r4,-1
72 srlg %r3,%r4,8
73 ltgr %r3,%r3
74 lgr %r1,%r2
75 jz .Lmemset_clear_remainder
76.Lmemset_clear_loop:
77 xc 0(256,%r1),0(%r1)
78 la %r1,256(%r1)
79 brctg %r3,.Lmemset_clear_loop
80.Lmemset_clear_remainder:
81 larl %r3,.Lmemset_xc
82 ex %r4,0(%r3)
83 BR_EX %r14
84.Lmemset_fill:
85 cghi %r4,1
86 lgr %r1,%r2
87 je .Lmemset_fill_exit
88 aghi %r4,-2
89 srlg %r5,%r4,8
90 ltgr %r5,%r5
91 jz .Lmemset_fill_remainder
92.Lmemset_fill_loop:
93 stc %r3,0(%r1)
94 mvc 1(255,%r1),0(%r1)
95 la %r1,256(%r1)
96 brctg %r5,.Lmemset_fill_loop
97.Lmemset_fill_remainder:
98 stc %r3,0(%r1)
99 larl %r5,.Lmemset_mvc
100 ex %r4,0(%r5)
101 BR_EX %r14
102.Lmemset_fill_exit:
103 stc %r3,0(%r1)
104 BR_EX %r14
105.Lmemset_xc:
106 xc 0(1,%r1),0(%r1)
107.Lmemset_mvc:
108 mvc 1(1,%r1),0(%r1)
109EXPORT_SYMBOL(memset)
110
111/*
112 * memcpy implementation
113 *
114 * void *memcpy(void *dest, const void *src, size_t n)
115 */
116ENTRY(memcpy)
117 ltgr %r4,%r4
118 bzr %r14
119 aghi %r4,-1
120 srlg %r5,%r4,8
121 ltgr %r5,%r5
122 lgr %r1,%r2
123 jnz .Lmemcpy_loop
124.Lmemcpy_remainder:
125 larl %r5,.Lmemcpy_mvc
126 ex %r4,0(%r5)
127 BR_EX %r14
128.Lmemcpy_loop:
129 mvc 0(256,%r1),0(%r3)
130 la %r1,256(%r1)
131 la %r3,256(%r3)
132 brctg %r5,.Lmemcpy_loop
133 j .Lmemcpy_remainder
134.Lmemcpy_mvc:
135 mvc 0(1,%r1),0(%r3)
136EXPORT_SYMBOL(memcpy)
137
138/*
139 * __memset16/32/64
140 *
141 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
142 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
143 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
144 */
145.macro __MEMSET bits,bytes,insn
146ENTRY(__memset\bits)
147 ltgr %r4,%r4
148 bzr %r14
149 cghi %r4,\bytes
150 je .L__memset_exit\bits
151 aghi %r4,-(\bytes+1)
152 srlg %r5,%r4,8
153 ltgr %r5,%r5
154 lgr %r1,%r2
155 jz .L__memset_remainder\bits
156.L__memset_loop\bits:
157 \insn %r3,0(%r1)
158 mvc \bytes(256-\bytes,%r1),0(%r1)
159 la %r1,256(%r1)
160 brctg %r5,.L__memset_loop\bits
161.L__memset_remainder\bits:
162 \insn %r3,0(%r1)
163 larl %r5,.L__memset_mvc\bits
164 ex %r4,0(%r5)
165 BR_EX %r14
166.L__memset_exit\bits:
167 \insn %r3,0(%r2)
168 BR_EX %r14
169.L__memset_mvc\bits:
170 mvc \bytes(1,%r1),0(%r1)
171.endm
172
173__MEMSET 16,2,sth
174EXPORT_SYMBOL(__memset16)
175
176__MEMSET 32,4,st
177EXPORT_SYMBOL(__memset32)
178
179__MEMSET 64,8,stg
180EXPORT_SYMBOL(__memset64)