1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * String handling functions. 4 * 5 * Copyright IBM Corp. 2012 6 */ 7 8#include <linux/linkage.h> 9#include <asm/export.h> 10#include <asm/nospec-insn.h> 11 12 GEN_BR_THUNK %r14 13 14/* 15 * void *memmove(void *dest, const void *src, size_t n) 16 */ 17ENTRY(memmove) 18 ltgr %r4,%r4 19 lgr %r1,%r2 20 bzr %r14 21 aghi %r4,-1 22 clgr %r2,%r3 23 jnh .Lmemmove_forward 24 la %r5,1(%r4,%r3) 25 clgr %r2,%r5 26 jl .Lmemmove_reverse 27.Lmemmove_forward: 28 srlg %r0,%r4,8 29 ltgr %r0,%r0 30 jz .Lmemmove_forward_remainder 31.Lmemmove_forward_loop: 32 mvc 0(256,%r1),0(%r3) 33 la %r1,256(%r1) 34 la %r3,256(%r3) 35 brctg %r0,.Lmemmove_forward_loop 36.Lmemmove_forward_remainder: 37 larl %r5,.Lmemmove_mvc 38 ex %r4,0(%r5) 39 BR_EX %r14 40.Lmemmove_reverse: 41 ic %r0,0(%r4,%r3) 42 stc %r0,0(%r4,%r1) 43 brctg %r4,.Lmemmove_reverse 44 ic %r0,0(%r4,%r3) 45 stc %r0,0(%r4,%r1) 46 BR_EX %r14 47.Lmemmove_mvc: 48 mvc 0(1,%r1),0(%r3) 49EXPORT_SYMBOL(memmove) 50 51/* 52 * memset implementation 53 * 54 * This code corresponds to the C construct below. We do distinguish 55 * between clearing (c == 0) and setting a memory array (c != 0) simply 56 * because nearly all memset invocations in the kernel clear memory and 57 * the xc instruction is preferred in such cases. 58 * 59 * void *memset(void *s, int c, size_t n) 60 * { 61 * if (likely(c == 0)) 62 * return __builtin_memset(s, 0, n); 63 * return __builtin_memset(s, c, n); 64 * } 65 */ 66ENTRY(memset) 67 ltgr %r4,%r4 68 bzr %r14 69 ltgr %r3,%r3 70 jnz .Lmemset_fill 71 aghi %r4,-1 72 srlg %r3,%r4,8 73 ltgr %r3,%r3 74 lgr %r1,%r2 75 jz .Lmemset_clear_remainder 76.Lmemset_clear_loop: 77 xc 0(256,%r1),0(%r1) 78 la %r1,256(%r1) 79 brctg %r3,.Lmemset_clear_loop 80.Lmemset_clear_remainder: 81 larl %r3,.Lmemset_xc 82 ex %r4,0(%r3) 83 BR_EX %r14 84.Lmemset_fill: 85 cghi %r4,1 86 lgr %r1,%r2 87 je .Lmemset_fill_exit 88 aghi %r4,-2 89 srlg %r5,%r4,8 90 ltgr %r5,%r5 91 jz .Lmemset_fill_remainder 92.Lmemset_fill_loop: 93 stc %r3,0(%r1) 94 mvc 1(255,%r1),0(%r1) 95 la %r1,256(%r1) 96 brctg %r5,.Lmemset_fill_loop 97.Lmemset_fill_remainder: 98 stc %r3,0(%r1) 99 larl %r5,.Lmemset_mvc 100 ex %r4,0(%r5) 101 BR_EX %r14 102.Lmemset_fill_exit: 103 stc %r3,0(%r1) 104 BR_EX %r14 105.Lmemset_xc: 106 xc 0(1,%r1),0(%r1) 107.Lmemset_mvc: 108 mvc 1(1,%r1),0(%r1) 109EXPORT_SYMBOL(memset) 110 111/* 112 * memcpy implementation 113 * 114 * void *memcpy(void *dest, const void *src, size_t n) 115 */ 116ENTRY(memcpy) 117 ltgr %r4,%r4 118 bzr %r14 119 aghi %r4,-1 120 srlg %r5,%r4,8 121 ltgr %r5,%r5 122 lgr %r1,%r2 123 jnz .Lmemcpy_loop 124.Lmemcpy_remainder: 125 larl %r5,.Lmemcpy_mvc 126 ex %r4,0(%r5) 127 BR_EX %r14 128.Lmemcpy_loop: 129 mvc 0(256,%r1),0(%r3) 130 la %r1,256(%r1) 131 la %r3,256(%r3) 132 brctg %r5,.Lmemcpy_loop 133 j .Lmemcpy_remainder 134.Lmemcpy_mvc: 135 mvc 0(1,%r1),0(%r3) 136EXPORT_SYMBOL(memcpy) 137 138/* 139 * __memset16/32/64 140 * 141 * void *__memset16(uint16_t *s, uint16_t v, size_t count) 142 * void *__memset32(uint32_t *s, uint32_t v, size_t count) 143 * void *__memset64(uint64_t *s, uint64_t v, size_t count) 144 */ 145.macro __MEMSET bits,bytes,insn 146ENTRY(__memset\bits) 147 ltgr %r4,%r4 148 bzr %r14 149 cghi %r4,\bytes 150 je .L__memset_exit\bits 151 aghi %r4,-(\bytes+1) 152 srlg %r5,%r4,8 153 ltgr %r5,%r5 154 lgr %r1,%r2 155 jz .L__memset_remainder\bits 156.L__memset_loop\bits: 157 \insn %r3,0(%r1) 158 mvc \bytes(256-\bytes,%r1),0(%r1) 159 la %r1,256(%r1) 160 brctg %r5,.L__memset_loop\bits 161.L__memset_remainder\bits: 162 \insn %r3,0(%r1) 163 larl %r5,.L__memset_mvc\bits 164 ex %r4,0(%r5) 165 BR_EX %r14 166.L__memset_exit\bits: 167 \insn %r3,0(%r2) 168 BR_EX %r14 169.L__memset_mvc\bits: 170 mvc \bytes(1,%r1),0(%r1) 171.endm 172 173__MEMSET 16,2,sth 174EXPORT_SYMBOL(__memset16) 175 176__MEMSET 32,4,st 177EXPORT_SYMBOL(__memset32) 178 179__MEMSET 64,8,stg 180EXPORT_SYMBOL(__memset64) 181