1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * String handling functions. 4 * 5 * Copyright IBM Corp. 2012 6 */ 7 8#include <linux/linkage.h> 9#include <asm/export.h> 10#include <asm/nospec-insn.h> 11 12 GEN_BR_THUNK %r14 13 14/* 15 * void *memmove(void *dest, const void *src, size_t n) 16 */ 17ENTRY(memmove) 18 ltgr %r4,%r4 19 lgr %r1,%r2 20 jz .Lmemmove_exit 21 aghi %r4,-1 22 clgr %r2,%r3 23 jnh .Lmemmove_forward 24 la %r5,1(%r4,%r3) 25 clgr %r2,%r5 26 jl .Lmemmove_reverse 27.Lmemmove_forward: 28 srlg %r0,%r4,8 29 ltgr %r0,%r0 30 jz .Lmemmove_forward_remainder 31.Lmemmove_forward_loop: 32 mvc 0(256,%r1),0(%r3) 33 la %r1,256(%r1) 34 la %r3,256(%r3) 35 brctg %r0,.Lmemmove_forward_loop 36.Lmemmove_forward_remainder: 37 larl %r5,.Lmemmove_mvc 38 ex %r4,0(%r5) 39.Lmemmove_exit: 40 BR_EX %r14 41.Lmemmove_reverse: 42 ic %r0,0(%r4,%r3) 43 stc %r0,0(%r4,%r1) 44 brctg %r4,.Lmemmove_reverse 45 ic %r0,0(%r4,%r3) 46 stc %r0,0(%r4,%r1) 47 BR_EX %r14 48.Lmemmove_mvc: 49 mvc 0(1,%r1),0(%r3) 50EXPORT_SYMBOL(memmove) 51 52/* 53 * memset implementation 54 * 55 * This code corresponds to the C construct below. We do distinguish 56 * between clearing (c == 0) and setting a memory array (c != 0) simply 57 * because nearly all memset invocations in the kernel clear memory and 58 * the xc instruction is preferred in such cases. 59 * 60 * void *memset(void *s, int c, size_t n) 61 * { 62 * if (likely(c == 0)) 63 * return __builtin_memset(s, 0, n); 64 * return __builtin_memset(s, c, n); 65 * } 66 */ 67ENTRY(memset) 68 ltgr %r4,%r4 69 jz .Lmemset_exit 70 ltgr %r3,%r3 71 jnz .Lmemset_fill 72 aghi %r4,-1 73 srlg %r3,%r4,8 74 ltgr %r3,%r3 75 lgr %r1,%r2 76 jz .Lmemset_clear_remainder 77.Lmemset_clear_loop: 78 xc 0(256,%r1),0(%r1) 79 la %r1,256(%r1) 80 brctg %r3,.Lmemset_clear_loop 81.Lmemset_clear_remainder: 82 larl %r3,.Lmemset_xc 83 ex %r4,0(%r3) 84.Lmemset_exit: 85 BR_EX %r14 86.Lmemset_fill: 87 cghi %r4,1 88 lgr %r1,%r2 89 je .Lmemset_fill_exit 90 aghi %r4,-2 91 srlg %r5,%r4,8 92 ltgr %r5,%r5 93 jz .Lmemset_fill_remainder 94.Lmemset_fill_loop: 95 stc %r3,0(%r1) 96 mvc 1(255,%r1),0(%r1) 97 la %r1,256(%r1) 98 brctg %r5,.Lmemset_fill_loop 99.Lmemset_fill_remainder: 100 stc %r3,0(%r1) 101 larl %r5,.Lmemset_mvc 102 ex %r4,0(%r5) 103 BR_EX %r14 104.Lmemset_fill_exit: 105 stc %r3,0(%r1) 106 BR_EX %r14 107.Lmemset_xc: 108 xc 0(1,%r1),0(%r1) 109.Lmemset_mvc: 110 mvc 1(1,%r1),0(%r1) 111EXPORT_SYMBOL(memset) 112 113/* 114 * memcpy implementation 115 * 116 * void *memcpy(void *dest, const void *src, size_t n) 117 */ 118ENTRY(memcpy) 119 ltgr %r4,%r4 120 jz .Lmemcpy_exit 121 aghi %r4,-1 122 srlg %r5,%r4,8 123 ltgr %r5,%r5 124 lgr %r1,%r2 125 jnz .Lmemcpy_loop 126.Lmemcpy_remainder: 127 larl %r5,.Lmemcpy_mvc 128 ex %r4,0(%r5) 129.Lmemcpy_exit: 130 BR_EX %r14 131.Lmemcpy_loop: 132 mvc 0(256,%r1),0(%r3) 133 la %r1,256(%r1) 134 la %r3,256(%r3) 135 brctg %r5,.Lmemcpy_loop 136 j .Lmemcpy_remainder 137.Lmemcpy_mvc: 138 mvc 0(1,%r1),0(%r3) 139EXPORT_SYMBOL(memcpy) 140 141/* 142 * __memset16/32/64 143 * 144 * void *__memset16(uint16_t *s, uint16_t v, size_t count) 145 * void *__memset32(uint32_t *s, uint32_t v, size_t count) 146 * void *__memset64(uint64_t *s, uint64_t v, size_t count) 147 */ 148.macro __MEMSET bits,bytes,insn 149ENTRY(__memset\bits) 150 ltgr %r4,%r4 151 jz .L__memset_exit\bits 152 cghi %r4,\bytes 153 je .L__memset_store\bits 154 aghi %r4,-(\bytes+1) 155 srlg %r5,%r4,8 156 ltgr %r5,%r5 157 lgr %r1,%r2 158 jz .L__memset_remainder\bits 159.L__memset_loop\bits: 160 \insn %r3,0(%r1) 161 mvc \bytes(256-\bytes,%r1),0(%r1) 162 la %r1,256(%r1) 163 brctg %r5,.L__memset_loop\bits 164.L__memset_remainder\bits: 165 \insn %r3,0(%r1) 166 larl %r5,.L__memset_mvc\bits 167 ex %r4,0(%r5) 168 BR_EX %r14 169.L__memset_store\bits: 170 \insn %r3,0(%r2) 171.L__memset_exit\bits: 172 BR_EX %r14 173.L__memset_mvc\bits: 174 mvc \bytes(1,%r1),0(%r1) 175.endm 176 177__MEMSET 16,2,sth 178EXPORT_SYMBOL(__memset16) 179 180__MEMSET 32,4,st 181EXPORT_SYMBOL(__memset32) 182 183__MEMSET 64,8,stg 184EXPORT_SYMBOL(__memset64) 185