1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * String handling functions. 4 * 5 * Copyright IBM Corp. 2012 6 */ 7 8#include <linux/linkage.h> 9#include <asm/export.h> 10#include <asm/nospec-insn.h> 11 12 GEN_BR_THUNK %r14 13 14/* 15 * void *memmove(void *dest, const void *src, size_t n) 16 */ 17WEAK(memmove) 18ENTRY(__memmove) 19 ltgr %r4,%r4 20 lgr %r1,%r2 21 jz .Lmemmove_exit 22 aghi %r4,-1 23 clgr %r2,%r3 24 jnh .Lmemmove_forward 25 la %r5,1(%r4,%r3) 26 clgr %r2,%r5 27 jl .Lmemmove_reverse 28.Lmemmove_forward: 29 srlg %r0,%r4,8 30 ltgr %r0,%r0 31 jz .Lmemmove_forward_remainder 32.Lmemmove_forward_loop: 33 mvc 0(256,%r1),0(%r3) 34 la %r1,256(%r1) 35 la %r3,256(%r3) 36 brctg %r0,.Lmemmove_forward_loop 37.Lmemmove_forward_remainder: 38 larl %r5,.Lmemmove_mvc 39 ex %r4,0(%r5) 40.Lmemmove_exit: 41 BR_EX %r14 42.Lmemmove_reverse: 43 ic %r0,0(%r4,%r3) 44 stc %r0,0(%r4,%r1) 45 brctg %r4,.Lmemmove_reverse 46 ic %r0,0(%r4,%r3) 47 stc %r0,0(%r4,%r1) 48 BR_EX %r14 49.Lmemmove_mvc: 50 mvc 0(1,%r1),0(%r3) 51ENDPROC(__memmove) 52EXPORT_SYMBOL(memmove) 53 54/* 55 * memset implementation 56 * 57 * This code corresponds to the C construct below. We do distinguish 58 * between clearing (c == 0) and setting a memory array (c != 0) simply 59 * because nearly all memset invocations in the kernel clear memory and 60 * the xc instruction is preferred in such cases. 61 * 62 * void *memset(void *s, int c, size_t n) 63 * { 64 * if (likely(c == 0)) 65 * return __builtin_memset(s, 0, n); 66 * return __builtin_memset(s, c, n); 67 * } 68 */ 69WEAK(memset) 70ENTRY(__memset) 71 ltgr %r4,%r4 72 jz .Lmemset_exit 73 ltgr %r3,%r3 74 jnz .Lmemset_fill 75 aghi %r4,-1 76 srlg %r3,%r4,8 77 ltgr %r3,%r3 78 lgr %r1,%r2 79 jz .Lmemset_clear_remainder 80.Lmemset_clear_loop: 81 xc 0(256,%r1),0(%r1) 82 la %r1,256(%r1) 83 brctg %r3,.Lmemset_clear_loop 84.Lmemset_clear_remainder: 85 larl %r3,.Lmemset_xc 86 ex %r4,0(%r3) 87.Lmemset_exit: 88 BR_EX %r14 89.Lmemset_fill: 90 cghi %r4,1 91 lgr %r1,%r2 92 je .Lmemset_fill_exit 93 aghi %r4,-2 94 srlg %r5,%r4,8 95 ltgr %r5,%r5 96 jz .Lmemset_fill_remainder 97.Lmemset_fill_loop: 98 stc %r3,0(%r1) 99 mvc 1(255,%r1),0(%r1) 100 la %r1,256(%r1) 101 brctg %r5,.Lmemset_fill_loop 102.Lmemset_fill_remainder: 103 stc %r3,0(%r1) 104 larl %r5,.Lmemset_mvc 105 ex %r4,0(%r5) 106 BR_EX %r14 107.Lmemset_fill_exit: 108 stc %r3,0(%r1) 109 BR_EX %r14 110.Lmemset_xc: 111 xc 0(1,%r1),0(%r1) 112.Lmemset_mvc: 113 mvc 1(1,%r1),0(%r1) 114ENDPROC(__memset) 115EXPORT_SYMBOL(memset) 116 117/* 118 * memcpy implementation 119 * 120 * void *memcpy(void *dest, const void *src, size_t n) 121 */ 122WEAK(memcpy) 123ENTRY(__memcpy) 124 ltgr %r4,%r4 125 jz .Lmemcpy_exit 126 aghi %r4,-1 127 srlg %r5,%r4,8 128 ltgr %r5,%r5 129 lgr %r1,%r2 130 jnz .Lmemcpy_loop 131.Lmemcpy_remainder: 132 larl %r5,.Lmemcpy_mvc 133 ex %r4,0(%r5) 134.Lmemcpy_exit: 135 BR_EX %r14 136.Lmemcpy_loop: 137 mvc 0(256,%r1),0(%r3) 138 la %r1,256(%r1) 139 la %r3,256(%r3) 140 brctg %r5,.Lmemcpy_loop 141 j .Lmemcpy_remainder 142.Lmemcpy_mvc: 143 mvc 0(1,%r1),0(%r3) 144ENDPROC(__memcpy) 145EXPORT_SYMBOL(memcpy) 146 147/* 148 * __memset16/32/64 149 * 150 * void *__memset16(uint16_t *s, uint16_t v, size_t count) 151 * void *__memset32(uint32_t *s, uint32_t v, size_t count) 152 * void *__memset64(uint64_t *s, uint64_t v, size_t count) 153 */ 154.macro __MEMSET bits,bytes,insn 155ENTRY(__memset\bits) 156 ltgr %r4,%r4 157 jz .L__memset_exit\bits 158 cghi %r4,\bytes 159 je .L__memset_store\bits 160 aghi %r4,-(\bytes+1) 161 srlg %r5,%r4,8 162 ltgr %r5,%r5 163 lgr %r1,%r2 164 jz .L__memset_remainder\bits 165.L__memset_loop\bits: 166 \insn %r3,0(%r1) 167 mvc \bytes(256-\bytes,%r1),0(%r1) 168 la %r1,256(%r1) 169 brctg %r5,.L__memset_loop\bits 170.L__memset_remainder\bits: 171 \insn %r3,0(%r1) 172 larl %r5,.L__memset_mvc\bits 173 ex %r4,0(%r5) 174 BR_EX %r14 175.L__memset_store\bits: 176 \insn %r3,0(%r2) 177.L__memset_exit\bits: 178 BR_EX %r14 179.L__memset_mvc\bits: 180 mvc \bytes(1,%r1),0(%r1) 181ENDPROC(__memset\bits) 182.endm 183 184__MEMSET 16,2,sth 185EXPORT_SYMBOL(__memset16) 186 187__MEMSET 32,4,st 188EXPORT_SYMBOL(__memset32) 189 190__MEMSET 64,8,stg 191EXPORT_SYMBOL(__memset64) 192