1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_STRING_64_H 3 #define _ASM_X86_STRING_64_H 4 5 #ifdef __KERNEL__ 6 #include <linux/jump_label.h> 7 8 /* Written 2002 by Andi Kleen */ 9 10 /* Even with __builtin_ the compiler may decide to use the out of line 11 function. */ 12 13 #if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) 14 #include <linux/kmsan_string.h> 15 #endif 16 17 #define __HAVE_ARCH_MEMCPY 1 18 #if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) 19 #undef memcpy 20 #define memcpy __msan_memcpy 21 #else 22 extern void *memcpy(void *to, const void *from, size_t len); 23 #endif 24 extern void *__memcpy(void *to, const void *from, size_t len); 25 26 #define __HAVE_ARCH_MEMSET 27 #if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) 28 extern void *__msan_memset(void *s, int c, size_t n); 29 #undef memset 30 #define memset __msan_memset 31 #else 32 void *memset(void *s, int c, size_t n); 33 #endif 34 void *__memset(void *s, int c, size_t n); 35 36 #define __HAVE_ARCH_MEMSET16 37 static inline void *memset16(uint16_t *s, uint16_t v, size_t n) 38 { 39 long d0, d1; 40 asm volatile("rep\n\t" 41 "stosw" 42 : "=&c" (d0), "=&D" (d1) 43 : "a" (v), "1" (s), "0" (n) 44 : "memory"); 45 return s; 46 } 47 48 #define __HAVE_ARCH_MEMSET32 49 static inline void *memset32(uint32_t *s, uint32_t v, size_t n) 50 { 51 long d0, d1; 52 asm volatile("rep\n\t" 53 "stosl" 54 : "=&c" (d0), "=&D" (d1) 55 : "a" (v), "1" (s), "0" (n) 56 : "memory"); 57 return s; 58 } 59 60 #define __HAVE_ARCH_MEMSET64 61 static inline void *memset64(uint64_t *s, uint64_t v, size_t n) 62 { 63 long d0, d1; 64 asm volatile("rep\n\t" 65 "stosq" 66 : "=&c" (d0), "=&D" (d1) 67 : "a" (v), "1" (s), "0" (n) 68 : "memory"); 69 return s; 70 } 71 72 #define __HAVE_ARCH_MEMMOVE 73 #if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) 74 #undef memmove 75 void *__msan_memmove(void *dest, const void *src, size_t len); 76 #define memmove __msan_memmove 77 #else 78 void *memmove(void *dest, const void *src, size_t count); 79 #endif 80 void *__memmove(void *dest, const void *src, size_t count); 81 82 int memcmp(const void *cs, const void *ct, size_t count); 83 size_t strlen(const char *s); 84 char *strcpy(char *dest, const char *src); 85 char *strcat(char *dest, const char *src); 86 int strcmp(const char *cs, const char *ct); 87 88 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 89 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 90 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); 91 static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) 92 { 93 if (__builtin_constant_p(cnt)) { 94 switch (cnt) { 95 case 4: 96 asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src)); 97 return; 98 case 8: 99 asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src)); 100 return; 101 case 16: 102 asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src)); 103 asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8))); 104 return; 105 } 106 } 107 __memcpy_flushcache(dst, src, cnt); 108 } 109 #endif 110 111 #endif /* __KERNEL__ */ 112 113 #endif /* _ASM_X86_STRING_64_H */ 114