1 #include <linux/string.h> 2 #include <linux/module.h> 3 #include <linux/io.h> 4 #include <linux/kmsan-checks.h> 5 6 #define movs(type,to,from) \ 7 asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory") 8 9 /* Originally from i386/string.h */ 10 static __always_inline void rep_movs(void *to, const void *from, size_t n) 11 { 12 unsigned long d0, d1, d2; 13 asm volatile("rep ; movsl\n\t" 14 "testb $2,%b4\n\t" 15 "je 1f\n\t" 16 "movsw\n" 17 "1:\ttestb $1,%b4\n\t" 18 "je 2f\n\t" 19 "movsb\n" 20 "2:" 21 : "=&c" (d0), "=&D" (d1), "=&S" (d2) 22 : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from) 23 : "memory"); 24 } 25 26 static void string_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) 27 { 28 if (unlikely(!n)) 29 return; 30 31 /* Align any unaligned source IO */ 32 if (unlikely(1 & (unsigned long)from)) { 33 movs("b", to, from); 34 n--; 35 } 36 if (n > 1 && unlikely(2 & (unsigned long)from)) { 37 movs("w", to, from); 38 n-=2; 39 } 40 rep_movs(to, (const void *)from, n); 41 /* KMSAN must treat values read from devices as initialized. */ 42 kmsan_unpoison_memory(to, n); 43 } 44 45 static void string_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) 46 { 47 if (unlikely(!n)) 48 return; 49 50 /* Make sure uninitialized memory isn't copied to devices. */ 51 kmsan_check_memory(from, n); 52 /* Align any unaligned destination IO */ 53 if (unlikely(1 & (unsigned long)to)) { 54 movs("b", to, from); 55 n--; 56 } 57 if (n > 1 && unlikely(2 & (unsigned long)to)) { 58 movs("w", to, from); 59 n-=2; 60 } 61 rep_movs((void *)to, (const void *) from, n); 62 } 63 64 static void unrolled_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) 65 { 66 const volatile char __iomem *in = from; 67 char *out = to; 68 int i; 69 70 for (i = 0; i < n; ++i) 71 out[i] = readb(&in[i]); 72 } 73 74 static void unrolled_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) 75 { 76 volatile char __iomem *out = to; 77 const char *in = from; 78 int i; 79 80 for (i = 0; i < n; ++i) 81 writeb(in[i], &out[i]); 82 } 83 84 static void unrolled_memset_io(volatile void __iomem *a, int b, size_t c) 85 { 86 volatile char __iomem *mem = a; 87 int i; 88 89 for (i = 0; i < c; ++i) 90 writeb(b, &mem[i]); 91 } 92 93 void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) 94 { 95 if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) 96 unrolled_memcpy_fromio(to, from, n); 97 else 98 string_memcpy_fromio(to, from, n); 99 } 100 EXPORT_SYMBOL(memcpy_fromio); 101 102 void memcpy_toio(volatile void __iomem *to, const void *from, size_t n) 103 { 104 if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) 105 unrolled_memcpy_toio(to, from, n); 106 else 107 string_memcpy_toio(to, from, n); 108 } 109 EXPORT_SYMBOL(memcpy_toio); 110 111 void memset_io(volatile void __iomem *a, int b, size_t c) 112 { 113 if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) { 114 unrolled_memset_io(a, b, c); 115 } else { 116 /* 117 * TODO: memset can mangle the IO patterns quite a bit. 118 * perhaps it would be better to use a dumb one: 119 */ 120 memset((void *)a, b, c); 121 } 122 } 123 EXPORT_SYMBOL(memset_io); 124