1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Access kernel memory without faulting -- s390 specific implementation. 4 * 5 * Copyright IBM Corp. 2009, 2015 6 * 7 */ 8 9 #include <linux/uaccess.h> 10 #include <linux/kernel.h> 11 #include <linux/types.h> 12 #include <linux/errno.h> 13 #include <linux/gfp.h> 14 #include <linux/cpu.h> 15 #include <linux/uio.h> 16 #include <asm/asm-extable.h> 17 #include <asm/ctl_reg.h> 18 #include <asm/io.h> 19 #include <asm/abs_lowcore.h> 20 #include <asm/stacktrace.h> 21 #include <asm/maccess.h> 22 23 unsigned long __bootdata_preserved(__memcpy_real_area); 24 static __ro_after_init pte_t *memcpy_real_ptep; 25 static DEFINE_MUTEX(memcpy_real_mutex); 26 27 static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) 28 { 29 unsigned long aligned, offset, count; 30 char tmp[8]; 31 32 aligned = (unsigned long) dst & ~7UL; 33 offset = (unsigned long) dst & 7UL; 34 size = min(8UL - offset, size); 35 count = size - 1; 36 asm volatile( 37 " bras 1,0f\n" 38 " mvc 0(1,%4),0(%5)\n" 39 "0: mvc 0(8,%3),0(%0)\n" 40 " ex %1,0(1)\n" 41 " lg %1,0(%3)\n" 42 " lra %0,0(%0)\n" 43 " sturg %1,%0\n" 44 : "+&a" (aligned), "+&a" (count), "=m" (tmp) 45 : "a" (&tmp), "a" (&tmp[offset]), "a" (src) 46 : "cc", "memory", "1"); 47 return size; 48 } 49 50 /* 51 * s390_kernel_write - write to kernel memory bypassing DAT 52 * @dst: destination address 53 * @src: source address 54 * @size: number of bytes to copy 55 * 56 * This function writes to kernel memory bypassing DAT and possible page table 57 * write protection. It writes to the destination using the sturg instruction. 58 * Therefore we have a read-modify-write sequence: the function reads eight 59 * bytes from destination at an eight byte boundary, modifies the bytes 60 * requested and writes the result back in a loop. 61 */ 62 static DEFINE_SPINLOCK(s390_kernel_write_lock); 63 64 notrace void *s390_kernel_write(void *dst, const void *src, size_t size) 65 { 66 void *tmp = dst; 67 unsigned long flags; 68 long copied; 69 70 spin_lock_irqsave(&s390_kernel_write_lock, flags); 71 if (!(flags & PSW_MASK_DAT)) { 72 memcpy(dst, src, size); 73 } else { 74 while (size) { 75 copied = s390_kernel_write_odd(tmp, src, size); 76 tmp += copied; 77 src += copied; 78 size -= copied; 79 } 80 } 81 spin_unlock_irqrestore(&s390_kernel_write_lock, flags); 82 83 return dst; 84 } 85 86 void __init memcpy_real_init(void) 87 { 88 memcpy_real_ptep = vmem_get_alloc_pte(__memcpy_real_area, true); 89 if (!memcpy_real_ptep) 90 panic("Couldn't setup memcpy real area"); 91 } 92 93 size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) 94 { 95 size_t len, copied, res = 0; 96 unsigned long phys, offset; 97 void *chunk; 98 pte_t pte; 99 100 while (count) { 101 phys = src & PAGE_MASK; 102 offset = src & ~PAGE_MASK; 103 chunk = (void *)(__memcpy_real_area + offset); 104 len = min(count, PAGE_SIZE - offset); 105 pte = mk_pte_phys(phys, PAGE_KERNEL_RO); 106 107 mutex_lock(&memcpy_real_mutex); 108 if (pte_val(pte) != pte_val(*memcpy_real_ptep)) { 109 __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); 110 set_pte(memcpy_real_ptep, pte); 111 } 112 copied = copy_to_iter(chunk, len, iter); 113 mutex_unlock(&memcpy_real_mutex); 114 115 count -= copied; 116 src += copied; 117 res += copied; 118 if (copied < len) 119 break; 120 } 121 return res; 122 } 123 124 int memcpy_real(void *dest, unsigned long src, size_t count) 125 { 126 struct iov_iter iter; 127 struct kvec kvec; 128 129 kvec.iov_base = dest; 130 kvec.iov_len = count; 131 iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count); 132 if (memcpy_real_iter(&iter, src, count) < count) 133 return -EFAULT; 134 return 0; 135 } 136 137 /* 138 * Find CPU that owns swapped prefix page 139 */ 140 static int get_swapped_owner(phys_addr_t addr) 141 { 142 phys_addr_t lc; 143 int cpu; 144 145 for_each_online_cpu(cpu) { 146 lc = virt_to_phys(lowcore_ptr[cpu]); 147 if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) 148 continue; 149 return cpu; 150 } 151 return -1; 152 } 153 154 /* 155 * Convert a physical pointer for /dev/mem access 156 * 157 * For swapped prefix pages a new buffer is returned that contains a copy of 158 * the absolute memory. The buffer size is maximum one page large. 159 */ 160 void *xlate_dev_mem_ptr(phys_addr_t addr) 161 { 162 void *ptr = phys_to_virt(addr); 163 void *bounce = ptr; 164 struct lowcore *abs_lc; 165 unsigned long flags; 166 unsigned long size; 167 int this_cpu, cpu; 168 169 cpus_read_lock(); 170 this_cpu = get_cpu(); 171 if (addr >= sizeof(struct lowcore)) { 172 cpu = get_swapped_owner(addr); 173 if (cpu < 0) 174 goto out; 175 } 176 bounce = (void *)__get_free_page(GFP_ATOMIC); 177 if (!bounce) 178 goto out; 179 size = PAGE_SIZE - (addr & ~PAGE_MASK); 180 if (addr < sizeof(struct lowcore)) { 181 abs_lc = get_abs_lowcore(&flags); 182 ptr = (void *)abs_lc + addr; 183 memcpy(bounce, ptr, size); 184 put_abs_lowcore(abs_lc, flags); 185 } else if (cpu == this_cpu) { 186 ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); 187 memcpy(bounce, ptr, size); 188 } else { 189 memcpy(bounce, ptr, size); 190 } 191 out: 192 put_cpu(); 193 cpus_read_unlock(); 194 return bounce; 195 } 196 197 /* 198 * Free converted buffer for /dev/mem access (if necessary) 199 */ 200 void unxlate_dev_mem_ptr(phys_addr_t addr, void *ptr) 201 { 202 if (addr != virt_to_phys(ptr)) 203 free_page((unsigned long)ptr); 204 } 205