1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Access to PCI I/O memory from user space programs. 4 * 5 * Copyright IBM Corp. 2014 6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com> 7 */ 8 #include <linux/kernel.h> 9 #include <linux/syscalls.h> 10 #include <linux/init.h> 11 #include <linux/mm.h> 12 #include <linux/errno.h> 13 #include <linux/pci.h> 14 #include <asm/pci_io.h> 15 #include <asm/pci_debug.h> 16 17 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) 18 { 19 struct { 20 u64 offset; 21 u8 cc; 22 u8 status; 23 } data = {offset, cc, status}; 24 25 zpci_err_hex(&data, sizeof(data)); 26 } 27 28 static inline int __pcistb_mio_inuser( 29 void __iomem *ioaddr, const void __user *src, 30 u64 len, u8 *status) 31 { 32 int cc = -ENXIO; 33 34 asm volatile ( 35 " sacf 256\n" 36 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" 37 "1: ipm %[cc]\n" 38 " srl %[cc],28\n" 39 "2: sacf 768\n" 40 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) 41 : [cc] "+d" (cc), [len] "+d" (len) 42 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) 43 : "cc", "memory"); 44 *status = len >> 24 & 0xff; 45 return cc; 46 } 47 48 static inline int __pcistg_mio_inuser( 49 void __iomem *ioaddr, const void __user *src, 50 u64 ulen, u8 *status) 51 { 52 register u64 addr asm("2") = (u64 __force) ioaddr; 53 register u64 len asm("3") = ulen; 54 int cc = -ENXIO; 55 u64 val = 0; 56 u64 cnt = ulen; 57 u8 tmp; 58 59 /* 60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of 61 * a register, then store it to PCI at @ioaddr while in secondary 62 * address space. pcistg then uses the user mappings. 63 */ 64 asm volatile ( 65 " sacf 256\n" 66 "0: llgc %[tmp],0(%[src])\n" 67 " sllg %[val],%[val],8\n" 68 " aghi %[src],1\n" 69 " ogr %[val],%[tmp]\n" 70 " brctg %[cnt],0b\n" 71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr]\n" 72 "2: ipm %[cc]\n" 73 " srl %[cc],28\n" 74 "3: sacf 768\n" 75 EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) 76 : 77 [src] "+a" (src), [cnt] "+d" (cnt), 78 [val] "+d" (val), [tmp] "=d" (tmp), 79 [len] "+d" (len), [cc] "+d" (cc), 80 [ioaddr] "+a" (addr) 81 :: "cc", "memory"); 82 *status = len >> 24 & 0xff; 83 84 /* did we read everything from user memory? */ 85 if (!cc && cnt != 0) 86 cc = -EFAULT; 87 88 return cc; 89 } 90 91 static inline int __memcpy_toio_inuser(void __iomem *dst, 92 const void __user *src, size_t n) 93 { 94 int size, rc = 0; 95 u8 status = 0; 96 mm_segment_t old_fs; 97 98 if (!src) 99 return -EINVAL; 100 101 old_fs = enable_sacf_uaccess(); 102 while (n > 0) { 103 size = zpci_get_max_write_size((u64 __force) dst, 104 (u64 __force) src, n, 105 ZPCI_MAX_WRITE_SIZE); 106 if (size > 8) /* main path */ 107 rc = __pcistb_mio_inuser(dst, src, size, &status); 108 else 109 rc = __pcistg_mio_inuser(dst, src, size, &status); 110 if (rc) 111 break; 112 src += size; 113 dst += size; 114 n -= size; 115 } 116 disable_sacf_uaccess(old_fs); 117 if (rc) 118 zpci_err_mmio(rc, status, (__force u64) dst); 119 return rc; 120 } 121 122 static long get_pfn(unsigned long user_addr, unsigned long access, 123 unsigned long *pfn) 124 { 125 struct vm_area_struct *vma; 126 long ret; 127 128 mmap_read_lock(current->mm); 129 ret = -EINVAL; 130 vma = find_vma(current->mm, user_addr); 131 if (!vma) 132 goto out; 133 ret = -EACCES; 134 if (!(vma->vm_flags & access)) 135 goto out; 136 ret = follow_pfn(vma, user_addr, pfn); 137 out: 138 mmap_read_unlock(current->mm); 139 return ret; 140 } 141 142 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, 143 const void __user *, user_buffer, size_t, length) 144 { 145 u8 local_buf[64]; 146 void __iomem *io_addr; 147 void *buf; 148 unsigned long pfn; 149 long ret; 150 151 if (!zpci_is_enabled()) 152 return -ENODEV; 153 154 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 155 return -EINVAL; 156 157 /* 158 * We only support write access to MIO capable devices if we are on 159 * a MIO enabled system. Otherwise we would have to check for every 160 * address if it is a special ZPCI_ADDR and would have to do 161 * a get_pfn() which we don't need for MIO capable devices. Currently 162 * ISM devices are the only devices without MIO support and there is no 163 * known need for accessing these from userspace. 164 */ 165 if (static_branch_likely(&have_mio)) { 166 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr, 167 user_buffer, 168 length); 169 return ret; 170 } 171 172 if (length > 64) { 173 buf = kmalloc(length, GFP_KERNEL); 174 if (!buf) 175 return -ENOMEM; 176 } else 177 buf = local_buf; 178 179 ret = get_pfn(mmio_addr, VM_WRITE, &pfn); 180 if (ret) 181 goto out; 182 io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | 183 (mmio_addr & ~PAGE_MASK)); 184 185 ret = -EFAULT; 186 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) 187 goto out; 188 189 if (copy_from_user(buf, user_buffer, length)) 190 goto out; 191 192 ret = zpci_memcpy_toio(io_addr, buf, length); 193 out: 194 if (buf != local_buf) 195 kfree(buf); 196 return ret; 197 } 198 199 static inline int __pcilg_mio_inuser( 200 void __user *dst, const void __iomem *ioaddr, 201 u64 ulen, u8 *status) 202 { 203 register u64 addr asm("2") = (u64 __force) ioaddr; 204 register u64 len asm("3") = ulen; 205 u64 cnt = ulen; 206 int shift = ulen * 8; 207 int cc = -ENXIO; 208 u64 val, tmp; 209 210 /* 211 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in 212 * user space) into a register using pcilg then store these bytes at 213 * user address @dst 214 */ 215 asm volatile ( 216 " sacf 256\n" 217 "0: .insn rre,0xb9d60000,%[val],%[ioaddr]\n" 218 "1: ipm %[cc]\n" 219 " srl %[cc],28\n" 220 " ltr %[cc],%[cc]\n" 221 " jne 4f\n" 222 "2: ahi %[shift],-8\n" 223 " srlg %[tmp],%[val],0(%[shift])\n" 224 "3: stc %[tmp],0(%[dst])\n" 225 " aghi %[dst],1\n" 226 " brctg %[cnt],2b\n" 227 "4: sacf 768\n" 228 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) 229 : 230 [cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len), 231 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), 232 [shift] "+d" (shift) 233 : 234 [ioaddr] "a" (addr) 235 : "cc", "memory"); 236 237 /* did we write everything to the user space buffer? */ 238 if (!cc && cnt != 0) 239 cc = -EFAULT; 240 241 *status = len >> 24 & 0xff; 242 return cc; 243 } 244 245 static inline int __memcpy_fromio_inuser(void __user *dst, 246 const void __iomem *src, 247 unsigned long n) 248 { 249 int size, rc = 0; 250 u8 status; 251 mm_segment_t old_fs; 252 253 old_fs = enable_sacf_uaccess(); 254 while (n > 0) { 255 size = zpci_get_max_write_size((u64 __force) src, 256 (u64 __force) dst, n, 257 ZPCI_MAX_READ_SIZE); 258 rc = __pcilg_mio_inuser(dst, src, size, &status); 259 if (rc) 260 break; 261 src += size; 262 dst += size; 263 n -= size; 264 } 265 disable_sacf_uaccess(old_fs); 266 if (rc) 267 zpci_err_mmio(rc, status, (__force u64) dst); 268 return rc; 269 } 270 271 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, 272 void __user *, user_buffer, size_t, length) 273 { 274 u8 local_buf[64]; 275 void __iomem *io_addr; 276 void *buf; 277 unsigned long pfn; 278 long ret; 279 280 if (!zpci_is_enabled()) 281 return -ENODEV; 282 283 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 284 return -EINVAL; 285 286 /* 287 * We only support read access to MIO capable devices if we are on 288 * a MIO enabled system. Otherwise we would have to check for every 289 * address if it is a special ZPCI_ADDR and would have to do 290 * a get_pfn() which we don't need for MIO capable devices. Currently 291 * ISM devices are the only devices without MIO support and there is no 292 * known need for accessing these from userspace. 293 */ 294 if (static_branch_likely(&have_mio)) { 295 ret = __memcpy_fromio_inuser( 296 user_buffer, (const void __iomem *)mmio_addr, 297 length); 298 return ret; 299 } 300 301 if (length > 64) { 302 buf = kmalloc(length, GFP_KERNEL); 303 if (!buf) 304 return -ENOMEM; 305 } else { 306 buf = local_buf; 307 } 308 309 ret = get_pfn(mmio_addr, VM_READ, &pfn); 310 if (ret) 311 goto out; 312 io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); 313 314 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { 315 ret = -EFAULT; 316 goto out; 317 } 318 ret = zpci_memcpy_fromio(buf, io_addr, length); 319 if (ret) 320 goto out; 321 if (copy_to_user(user_buffer, buf, length)) 322 ret = -EFAULT; 323 324 out: 325 if (buf != local_buf) 326 kfree(buf); 327 return ret; 328 } 329