1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Access kernel memory without faulting -- s390 specific implementation. 4 * 5 * Copyright IBM Corp. 2009, 2015 6 * 7 */ 8 9 #include <linux/uaccess.h> 10 #include <linux/kernel.h> 11 #include <linux/types.h> 12 #include <linux/errno.h> 13 #include <linux/gfp.h> 14 #include <linux/cpu.h> 15 #include <linux/uio.h> 16 #include <linux/io.h> 17 #include <asm/asm-extable.h> 18 #include <asm/ctl_reg.h> 19 #include <asm/abs_lowcore.h> 20 #include <asm/stacktrace.h> 21 #include <asm/maccess.h> 22 23 unsigned long __bootdata_preserved(__memcpy_real_area); 24 pte_t *__bootdata_preserved(memcpy_real_ptep); 25 static DEFINE_MUTEX(memcpy_real_mutex); 26 27 static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) 28 { 29 unsigned long aligned, offset, count; 30 char tmp[8]; 31 32 aligned = (unsigned long) dst & ~7UL; 33 offset = (unsigned long) dst & 7UL; 34 size = min(8UL - offset, size); 35 count = size - 1; 36 asm volatile( 37 " bras 1,0f\n" 38 " mvc 0(1,%4),0(%5)\n" 39 "0: mvc 0(8,%3),0(%0)\n" 40 " ex %1,0(1)\n" 41 " lg %1,0(%3)\n" 42 " lra %0,0(%0)\n" 43 " sturg %1,%0\n" 44 : "+&a" (aligned), "+&a" (count), "=m" (tmp) 45 : "a" (&tmp), "a" (&tmp[offset]), "a" (src) 46 : "cc", "memory", "1"); 47 return size; 48 } 49 50 /* 51 * s390_kernel_write - write to kernel memory bypassing DAT 52 * @dst: destination address 53 * @src: source address 54 * @size: number of bytes to copy 55 * 56 * This function writes to kernel memory bypassing DAT and possible page table 57 * write protection. It writes to the destination using the sturg instruction. 58 * Therefore we have a read-modify-write sequence: the function reads eight 59 * bytes from destination at an eight byte boundary, modifies the bytes 60 * requested and writes the result back in a loop. 61 */ 62 static DEFINE_SPINLOCK(s390_kernel_write_lock); 63 64 notrace void *s390_kernel_write(void *dst, const void *src, size_t size) 65 { 66 void *tmp = dst; 67 unsigned long flags; 68 long copied; 69 70 spin_lock_irqsave(&s390_kernel_write_lock, flags); 71 while (size) { 72 copied = s390_kernel_write_odd(tmp, src, size); 73 tmp += copied; 74 src += copied; 75 size -= copied; 76 } 77 spin_unlock_irqrestore(&s390_kernel_write_lock, flags); 78 79 return dst; 80 } 81 82 size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) 83 { 84 size_t len, copied, res = 0; 85 unsigned long phys, offset; 86 void *chunk; 87 pte_t pte; 88 89 while (count) { 90 phys = src & PAGE_MASK; 91 offset = src & ~PAGE_MASK; 92 chunk = (void *)(__memcpy_real_area + offset); 93 len = min(count, PAGE_SIZE - offset); 94 pte = mk_pte_phys(phys, PAGE_KERNEL_RO); 95 96 mutex_lock(&memcpy_real_mutex); 97 if (pte_val(pte) != pte_val(*memcpy_real_ptep)) { 98 __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); 99 set_pte(memcpy_real_ptep, pte); 100 } 101 copied = copy_to_iter(chunk, len, iter); 102 mutex_unlock(&memcpy_real_mutex); 103 104 count -= copied; 105 src += copied; 106 res += copied; 107 if (copied < len) 108 break; 109 } 110 return res; 111 } 112 113 int memcpy_real(void *dest, unsigned long src, size_t count) 114 { 115 struct iov_iter iter; 116 struct kvec kvec; 117 118 kvec.iov_base = dest; 119 kvec.iov_len = count; 120 iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count); 121 if (memcpy_real_iter(&iter, src, count) < count) 122 return -EFAULT; 123 return 0; 124 } 125 126 /* 127 * Find CPU that owns swapped prefix page 128 */ 129 static int get_swapped_owner(phys_addr_t addr) 130 { 131 phys_addr_t lc; 132 int cpu; 133 134 for_each_online_cpu(cpu) { 135 lc = virt_to_phys(lowcore_ptr[cpu]); 136 if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) 137 continue; 138 return cpu; 139 } 140 return -1; 141 } 142 143 /* 144 * Convert a physical pointer for /dev/mem access 145 * 146 * For swapped prefix pages a new buffer is returned that contains a copy of 147 * the absolute memory. The buffer size is maximum one page large. 148 */ 149 void *xlate_dev_mem_ptr(phys_addr_t addr) 150 { 151 void *ptr = phys_to_virt(addr); 152 void *bounce = ptr; 153 struct lowcore *abs_lc; 154 unsigned long size; 155 int this_cpu, cpu; 156 157 cpus_read_lock(); 158 this_cpu = get_cpu(); 159 if (addr >= sizeof(struct lowcore)) { 160 cpu = get_swapped_owner(addr); 161 if (cpu < 0) 162 goto out; 163 } 164 bounce = (void *)__get_free_page(GFP_ATOMIC); 165 if (!bounce) 166 goto out; 167 size = PAGE_SIZE - (addr & ~PAGE_MASK); 168 if (addr < sizeof(struct lowcore)) { 169 abs_lc = get_abs_lowcore(); 170 ptr = (void *)abs_lc + addr; 171 memcpy(bounce, ptr, size); 172 put_abs_lowcore(abs_lc); 173 } else if (cpu == this_cpu) { 174 ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); 175 memcpy(bounce, ptr, size); 176 } else { 177 memcpy(bounce, ptr, size); 178 } 179 out: 180 put_cpu(); 181 cpus_read_unlock(); 182 return bounce; 183 } 184 185 /* 186 * Free converted buffer for /dev/mem access (if necessary) 187 */ 188 void unxlate_dev_mem_ptr(phys_addr_t addr, void *ptr) 189 { 190 if (addr != virt_to_phys(ptr)) 191 free_page((unsigned long)ptr); 192 } 193