1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * String handling functions.
  4 *
  5 * Copyright IBM Corp. 2012
  6 */
  7
  8#include <linux/linkage.h>
  9#include <asm/export.h>
 10#include <asm/nospec-insn.h>
 11
 12	GEN_BR_THUNK %r14
 13
 14/*
 15 * void *memmove(void *dest, const void *src, size_t n)
 16 */
 17ENTRY(memmove)
 18	ltgr	%r4,%r4
 19	lgr	%r1,%r2
 20	bzr	%r14
 21	aghi	%r4,-1
 22	clgr	%r2,%r3
 23	jnh	.Lmemmove_forward
 24	la	%r5,1(%r4,%r3)
 25	clgr	%r2,%r5
 26	jl	.Lmemmove_reverse
 27.Lmemmove_forward:
 28	srlg	%r0,%r4,8
 29	ltgr	%r0,%r0
 30	jz	.Lmemmove_forward_remainder
 31.Lmemmove_forward_loop:
 32	mvc	0(256,%r1),0(%r3)
 33	la	%r1,256(%r1)
 34	la	%r3,256(%r3)
 35	brctg	%r0,.Lmemmove_forward_loop
 36.Lmemmove_forward_remainder:
 37	larl	%r5,.Lmemmove_mvc
 38	ex	%r4,0(%r5)
 39	BR_EX	%r14
 40.Lmemmove_reverse:
 41	ic	%r0,0(%r4,%r3)
 42	stc	%r0,0(%r4,%r1)
 43	brctg	%r4,.Lmemmove_reverse
 44	ic	%r0,0(%r4,%r3)
 45	stc	%r0,0(%r4,%r1)
 46	BR_EX	%r14
 47.Lmemmove_mvc:
 48	mvc	0(1,%r1),0(%r3)
 49EXPORT_SYMBOL(memmove)
 50
 51/*
 52 * memset implementation
 53 *
 54 * This code corresponds to the C construct below. We do distinguish
 55 * between clearing (c == 0) and setting a memory array (c != 0) simply
 56 * because nearly all memset invocations in the kernel clear memory and
 57 * the xc instruction is preferred in such cases.
 58 *
 59 * void *memset(void *s, int c, size_t n)
 60 * {
 61 *	if (likely(c == 0))
 62 *		return __builtin_memset(s, 0, n);
 63 *	return __builtin_memset(s, c, n);
 64 * }
 65 */
 66ENTRY(memset)
 67	ltgr	%r4,%r4
 68	bzr	%r14
 69	ltgr	%r3,%r3
 70	jnz	.Lmemset_fill
 71	aghi	%r4,-1
 72	srlg	%r3,%r4,8
 73	ltgr	%r3,%r3
 74	lgr	%r1,%r2
 75	jz	.Lmemset_clear_remainder
 76.Lmemset_clear_loop:
 77	xc	0(256,%r1),0(%r1)
 78	la	%r1,256(%r1)
 79	brctg	%r3,.Lmemset_clear_loop
 80.Lmemset_clear_remainder:
 81	larl	%r3,.Lmemset_xc
 82	ex	%r4,0(%r3)
 83	BR_EX	%r14
 84.Lmemset_fill:
 85	cghi	%r4,1
 86	lgr	%r1,%r2
 87	je	.Lmemset_fill_exit
 88	aghi	%r4,-2
 89	srlg	%r5,%r4,8
 90	ltgr	%r5,%r5
 91	jz	.Lmemset_fill_remainder
 92.Lmemset_fill_loop:
 93	stc	%r3,0(%r1)
 94	mvc	1(255,%r1),0(%r1)
 95	la	%r1,256(%r1)
 96	brctg	%r5,.Lmemset_fill_loop
 97.Lmemset_fill_remainder:
 98	stc	%r3,0(%r1)
 99	larl	%r5,.Lmemset_mvc
100	ex	%r4,0(%r5)
101	BR_EX	%r14
102.Lmemset_fill_exit:
103	stc	%r3,0(%r1)
104	BR_EX	%r14
105.Lmemset_xc:
106	xc	0(1,%r1),0(%r1)
107.Lmemset_mvc:
108	mvc	1(1,%r1),0(%r1)
109EXPORT_SYMBOL(memset)
110
111/*
112 * memcpy implementation
113 *
114 * void *memcpy(void *dest, const void *src, size_t n)
115 */
116ENTRY(memcpy)
117	ltgr	%r4,%r4
118	bzr	%r14
119	aghi	%r4,-1
120	srlg	%r5,%r4,8
121	ltgr	%r5,%r5
122	lgr	%r1,%r2
123	jnz	.Lmemcpy_loop
124.Lmemcpy_remainder:
125	larl	%r5,.Lmemcpy_mvc
126	ex	%r4,0(%r5)
127	BR_EX	%r14
128.Lmemcpy_loop:
129	mvc	0(256,%r1),0(%r3)
130	la	%r1,256(%r1)
131	la	%r3,256(%r3)
132	brctg	%r5,.Lmemcpy_loop
133	j	.Lmemcpy_remainder
134.Lmemcpy_mvc:
135	mvc	0(1,%r1),0(%r3)
136EXPORT_SYMBOL(memcpy)
137
138/*
139 * __memset16/32/64
140 *
141 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
142 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
143 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
144 */
145.macro __MEMSET bits,bytes,insn
146ENTRY(__memset\bits)
147	ltgr	%r4,%r4
148	bzr	%r14
149	cghi	%r4,\bytes
150	je	.L__memset_exit\bits
151	aghi	%r4,-(\bytes+1)
152	srlg	%r5,%r4,8
153	ltgr	%r5,%r5
154	lgr	%r1,%r2
155	jz	.L__memset_remainder\bits
156.L__memset_loop\bits:
157	\insn	%r3,0(%r1)
158	mvc	\bytes(256-\bytes,%r1),0(%r1)
159	la	%r1,256(%r1)
160	brctg	%r5,.L__memset_loop\bits
161.L__memset_remainder\bits:
162	\insn	%r3,0(%r1)
163	larl	%r5,.L__memset_mvc\bits
164	ex	%r4,0(%r5)
165	BR_EX	%r14
166.L__memset_exit\bits:
167	\insn	%r3,0(%r2)
168	BR_EX	%r14
169.L__memset_mvc\bits:
170	mvc	\bytes(1,%r1),0(%r1)
171.endm
172
173__MEMSET 16,2,sth
174EXPORT_SYMBOL(__memset16)
175
176__MEMSET 32,4,st
177EXPORT_SYMBOL(__memset32)
178
179__MEMSET 64,8,stg
180EXPORT_SYMBOL(__memset64)