xref: /openbmc/linux/arch/s390/lib/mem.S (revision b8c723f1)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * String handling functions.
4 *
5 * Copyright IBM Corp. 2012
6 */
7
8#include <linux/export.h>
9#include <linux/linkage.h>
10#include <asm/nospec-insn.h>
11
12	GEN_BR_THUNK %r14
13
14/*
15 * void *memmove(void *dest, const void *src, size_t n)
16 */
17SYM_FUNC_START(__memmove)
18	ltgr	%r4,%r4
19	lgr	%r1,%r2
20	jz	.Lmemmove_exit
21	aghi	%r4,-1
22	clgr	%r2,%r3
23	jnh	.Lmemmove_forward
24	la	%r5,1(%r4,%r3)
25	clgr	%r2,%r5
26	jl	.Lmemmove_reverse
27.Lmemmove_forward:
28	srlg	%r0,%r4,8
29	ltgr	%r0,%r0
30	jz	.Lmemmove_forward_remainder
31.Lmemmove_forward_loop:
32	mvc	0(256,%r1),0(%r3)
33	la	%r1,256(%r1)
34	la	%r3,256(%r3)
35	brctg	%r0,.Lmemmove_forward_loop
36.Lmemmove_forward_remainder:
37	larl	%r5,.Lmemmove_mvc
38	ex	%r4,0(%r5)
39.Lmemmove_exit:
40	BR_EX	%r14
41.Lmemmove_reverse:
42	ic	%r0,0(%r4,%r3)
43	stc	%r0,0(%r4,%r1)
44	brctg	%r4,.Lmemmove_reverse
45	ic	%r0,0(%r4,%r3)
46	stc	%r0,0(%r4,%r1)
47	BR_EX	%r14
48.Lmemmove_mvc:
49	mvc	0(1,%r1),0(%r3)
50SYM_FUNC_END(__memmove)
51EXPORT_SYMBOL(__memmove)
52
53SYM_FUNC_ALIAS(memmove, __memmove)
54EXPORT_SYMBOL(memmove)
55
56/*
57 * memset implementation
58 *
59 * This code corresponds to the C construct below. We do distinguish
60 * between clearing (c == 0) and setting a memory array (c != 0) simply
61 * because nearly all memset invocations in the kernel clear memory and
62 * the xc instruction is preferred in such cases.
63 *
64 * void *memset(void *s, int c, size_t n)
65 * {
66 *	if (likely(c == 0))
67 *		return __builtin_memset(s, 0, n);
68 *	return __builtin_memset(s, c, n);
69 * }
70 */
71SYM_FUNC_START(__memset)
72	ltgr	%r4,%r4
73	jz	.Lmemset_exit
74	ltgr	%r3,%r3
75	jnz	.Lmemset_fill
76	aghi	%r4,-1
77	srlg	%r3,%r4,8
78	ltgr	%r3,%r3
79	lgr	%r1,%r2
80	jz	.Lmemset_clear_remainder
81.Lmemset_clear_loop:
82	xc	0(256,%r1),0(%r1)
83	la	%r1,256(%r1)
84	brctg	%r3,.Lmemset_clear_loop
85.Lmemset_clear_remainder:
86	larl	%r3,.Lmemset_xc
87	ex	%r4,0(%r3)
88.Lmemset_exit:
89	BR_EX	%r14
90.Lmemset_fill:
91	cghi	%r4,1
92	lgr	%r1,%r2
93	je	.Lmemset_fill_exit
94	aghi	%r4,-2
95	srlg	%r5,%r4,8
96	ltgr	%r5,%r5
97	jz	.Lmemset_fill_remainder
98.Lmemset_fill_loop:
99	stc	%r3,0(%r1)
100	mvc	1(255,%r1),0(%r1)
101	la	%r1,256(%r1)
102	brctg	%r5,.Lmemset_fill_loop
103.Lmemset_fill_remainder:
104	stc	%r3,0(%r1)
105	larl	%r5,.Lmemset_mvc
106	ex	%r4,0(%r5)
107	BR_EX	%r14
108.Lmemset_fill_exit:
109	stc	%r3,0(%r1)
110	BR_EX	%r14
111.Lmemset_xc:
112	xc	0(1,%r1),0(%r1)
113.Lmemset_mvc:
114	mvc	1(1,%r1),0(%r1)
115SYM_FUNC_END(__memset)
116EXPORT_SYMBOL(__memset)
117
118SYM_FUNC_ALIAS(memset, __memset)
119EXPORT_SYMBOL(memset)
120
121/*
122 * memcpy implementation
123 *
124 * void *memcpy(void *dest, const void *src, size_t n)
125 */
126SYM_FUNC_START(__memcpy)
127	ltgr	%r4,%r4
128	jz	.Lmemcpy_exit
129	aghi	%r4,-1
130	srlg	%r5,%r4,8
131	ltgr	%r5,%r5
132	lgr	%r1,%r2
133	jnz	.Lmemcpy_loop
134.Lmemcpy_remainder:
135	larl	%r5,.Lmemcpy_mvc
136	ex	%r4,0(%r5)
137.Lmemcpy_exit:
138	BR_EX	%r14
139.Lmemcpy_loop:
140	mvc	0(256,%r1),0(%r3)
141	la	%r1,256(%r1)
142	la	%r3,256(%r3)
143	brctg	%r5,.Lmemcpy_loop
144	j	.Lmemcpy_remainder
145.Lmemcpy_mvc:
146	mvc	0(1,%r1),0(%r3)
147SYM_FUNC_END(__memcpy)
148EXPORT_SYMBOL(__memcpy)
149
150SYM_FUNC_ALIAS(memcpy, __memcpy)
151EXPORT_SYMBOL(memcpy)
152
153/*
154 * __memset16/32/64
155 *
156 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
157 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
158 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
159 */
160.macro __MEMSET bits,bytes,insn
161SYM_FUNC_START(__memset\bits)
162	ltgr	%r4,%r4
163	jz	.L__memset_exit\bits
164	cghi	%r4,\bytes
165	je	.L__memset_store\bits
166	aghi	%r4,-(\bytes+1)
167	srlg	%r5,%r4,8
168	ltgr	%r5,%r5
169	lgr	%r1,%r2
170	jz	.L__memset_remainder\bits
171.L__memset_loop\bits:
172	\insn	%r3,0(%r1)
173	mvc	\bytes(256-\bytes,%r1),0(%r1)
174	la	%r1,256(%r1)
175	brctg	%r5,.L__memset_loop\bits
176.L__memset_remainder\bits:
177	\insn	%r3,0(%r1)
178	larl	%r5,.L__memset_mvc\bits
179	ex	%r4,0(%r5)
180	BR_EX	%r14
181.L__memset_store\bits:
182	\insn	%r3,0(%r2)
183.L__memset_exit\bits:
184	BR_EX	%r14
185.L__memset_mvc\bits:
186	mvc	\bytes(1,%r1),0(%r1)
187SYM_FUNC_END(__memset\bits)
188.endm
189
190__MEMSET 16,2,sth
191EXPORT_SYMBOL(__memset16)
192
193__MEMSET 32,4,st
194EXPORT_SYMBOL(__memset32)
195
196__MEMSET 64,8,stg
197EXPORT_SYMBOL(__memset64)
198