1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Access to PCI I/O memory from user space programs. 4 * 5 * Copyright IBM Corp. 2014 6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com> 7 */ 8 #include <linux/kernel.h> 9 #include <linux/syscalls.h> 10 #include <linux/init.h> 11 #include <linux/mm.h> 12 #include <linux/errno.h> 13 #include <linux/pci.h> 14 #include <asm/pci_io.h> 15 #include <asm/pci_debug.h> 16 17 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) 18 { 19 struct { 20 u64 offset; 21 u8 cc; 22 u8 status; 23 } data = {offset, cc, status}; 24 25 zpci_err_hex(&data, sizeof(data)); 26 } 27 28 static inline int __pcistb_mio_inuser( 29 void __iomem *ioaddr, const void __user *src, 30 u64 len, u8 *status) 31 { 32 int cc = -ENXIO; 33 34 asm volatile ( 35 " sacf 256\n" 36 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" 37 "1: ipm %[cc]\n" 38 " srl %[cc],28\n" 39 "2: sacf 768\n" 40 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) 41 : [cc] "+d" (cc), [len] "+d" (len) 42 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) 43 : "cc", "memory"); 44 *status = len >> 24 & 0xff; 45 return cc; 46 } 47 48 static inline int __pcistg_mio_inuser( 49 void __iomem *ioaddr, const void __user *src, 50 u64 ulen, u8 *status) 51 { 52 register u64 addr asm("2") = (u64 __force) ioaddr; 53 register u64 len asm("3") = ulen; 54 int cc = -ENXIO; 55 u64 val = 0; 56 u64 cnt = ulen; 57 u8 tmp; 58 59 /* 60 * copy 0 < @len <= 8 bytes from @src into the right most bytes of 61 * a register, then store it to PCI at @ioaddr while in secondary 62 * address space. pcistg then uses the user mappings. 63 */ 64 asm volatile ( 65 " sacf 256\n" 66 "0: llgc %[tmp],0(%[src])\n" 67 " sllg %[val],%[val],8\n" 68 " aghi %[src],1\n" 69 " ogr %[val],%[tmp]\n" 70 " brctg %[cnt],0b\n" 71 "1: .insn rre,0xb9d40000,%[val],%[ioaddr]\n" 72 "2: ipm %[cc]\n" 73 " srl %[cc],28\n" 74 "3: sacf 768\n" 75 EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) 76 : 77 [src] "+a" (src), [cnt] "+d" (cnt), 78 [val] "+d" (val), [tmp] "=d" (tmp), 79 [len] "+d" (len), [cc] "+d" (cc), 80 [ioaddr] "+a" (addr) 81 :: "cc", "memory"); 82 *status = len >> 24 & 0xff; 83 84 /* did we read everything from user memory? */ 85 if (!cc && cnt != 0) 86 cc = -EFAULT; 87 88 return cc; 89 } 90 91 static inline int __memcpy_toio_inuser(void __iomem *dst, 92 const void __user *src, size_t n) 93 { 94 int size, rc = 0; 95 u8 status = 0; 96 97 if (!src) 98 return -EINVAL; 99 100 while (n > 0) { 101 size = zpci_get_max_write_size((u64 __force) dst, 102 (u64 __force) src, n, 103 ZPCI_MAX_WRITE_SIZE); 104 if (size > 8) /* main path */ 105 rc = __pcistb_mio_inuser(dst, src, size, &status); 106 else 107 rc = __pcistg_mio_inuser(dst, src, size, &status); 108 if (rc) 109 break; 110 src += size; 111 dst += size; 112 n -= size; 113 } 114 if (rc) 115 zpci_err_mmio(rc, status, (__force u64) dst); 116 return rc; 117 } 118 119 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, 120 const void __user *, user_buffer, size_t, length) 121 { 122 u8 local_buf[64]; 123 void __iomem *io_addr; 124 void *buf; 125 struct vm_area_struct *vma; 126 pte_t *ptep; 127 spinlock_t *ptl; 128 long ret; 129 130 if (!zpci_is_enabled()) 131 return -ENODEV; 132 133 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 134 return -EINVAL; 135 136 /* 137 * We only support write access to MIO capable devices if we are on 138 * a MIO enabled system. Otherwise we would have to check for every 139 * address if it is a special ZPCI_ADDR and would have to do 140 * a pfn lookup which we don't need for MIO capable devices. Currently 141 * ISM devices are the only devices without MIO support and there is no 142 * known need for accessing these from userspace. 143 */ 144 if (static_branch_likely(&have_mio)) { 145 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr, 146 user_buffer, 147 length); 148 return ret; 149 } 150 151 if (length > 64) { 152 buf = kmalloc(length, GFP_KERNEL); 153 if (!buf) 154 return -ENOMEM; 155 } else 156 buf = local_buf; 157 158 ret = -EFAULT; 159 if (copy_from_user(buf, user_buffer, length)) 160 goto out_free; 161 162 mmap_read_lock(current->mm); 163 ret = -EINVAL; 164 vma = find_vma(current->mm, mmio_addr); 165 if (!vma) 166 goto out_unlock_mmap; 167 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 168 goto out_unlock_mmap; 169 ret = -EACCES; 170 if (!(vma->vm_flags & VM_WRITE)) 171 goto out_unlock_mmap; 172 173 ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); 174 if (ret) 175 goto out_unlock_mmap; 176 177 io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | 178 (mmio_addr & ~PAGE_MASK)); 179 180 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) 181 goto out_unlock_pt; 182 183 ret = zpci_memcpy_toio(io_addr, buf, length); 184 out_unlock_pt: 185 pte_unmap_unlock(ptep, ptl); 186 out_unlock_mmap: 187 mmap_read_unlock(current->mm); 188 out_free: 189 if (buf != local_buf) 190 kfree(buf); 191 return ret; 192 } 193 194 static inline int __pcilg_mio_inuser( 195 void __user *dst, const void __iomem *ioaddr, 196 u64 ulen, u8 *status) 197 { 198 register u64 addr asm("2") = (u64 __force) ioaddr; 199 register u64 len asm("3") = ulen; 200 u64 cnt = ulen; 201 int shift = ulen * 8; 202 int cc = -ENXIO; 203 u64 val, tmp; 204 205 /* 206 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in 207 * user space) into a register using pcilg then store these bytes at 208 * user address @dst 209 */ 210 asm volatile ( 211 " sacf 256\n" 212 "0: .insn rre,0xb9d60000,%[val],%[ioaddr]\n" 213 "1: ipm %[cc]\n" 214 " srl %[cc],28\n" 215 " ltr %[cc],%[cc]\n" 216 " jne 4f\n" 217 "2: ahi %[shift],-8\n" 218 " srlg %[tmp],%[val],0(%[shift])\n" 219 "3: stc %[tmp],0(%[dst])\n" 220 " aghi %[dst],1\n" 221 " brctg %[cnt],2b\n" 222 "4: sacf 768\n" 223 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) 224 : 225 [cc] "+d" (cc), [val] "=d" (val), [len] "+d" (len), 226 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), 227 [shift] "+d" (shift) 228 : 229 [ioaddr] "a" (addr) 230 : "cc", "memory"); 231 232 /* did we write everything to the user space buffer? */ 233 if (!cc && cnt != 0) 234 cc = -EFAULT; 235 236 *status = len >> 24 & 0xff; 237 return cc; 238 } 239 240 static inline int __memcpy_fromio_inuser(void __user *dst, 241 const void __iomem *src, 242 unsigned long n) 243 { 244 int size, rc = 0; 245 u8 status; 246 247 while (n > 0) { 248 size = zpci_get_max_write_size((u64 __force) src, 249 (u64 __force) dst, n, 250 ZPCI_MAX_READ_SIZE); 251 rc = __pcilg_mio_inuser(dst, src, size, &status); 252 if (rc) 253 break; 254 src += size; 255 dst += size; 256 n -= size; 257 } 258 if (rc) 259 zpci_err_mmio(rc, status, (__force u64) dst); 260 return rc; 261 } 262 263 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, 264 void __user *, user_buffer, size_t, length) 265 { 266 u8 local_buf[64]; 267 void __iomem *io_addr; 268 void *buf; 269 struct vm_area_struct *vma; 270 pte_t *ptep; 271 spinlock_t *ptl; 272 long ret; 273 274 if (!zpci_is_enabled()) 275 return -ENODEV; 276 277 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 278 return -EINVAL; 279 280 /* 281 * We only support read access to MIO capable devices if we are on 282 * a MIO enabled system. Otherwise we would have to check for every 283 * address if it is a special ZPCI_ADDR and would have to do 284 * a pfn lookup which we don't need for MIO capable devices. Currently 285 * ISM devices are the only devices without MIO support and there is no 286 * known need for accessing these from userspace. 287 */ 288 if (static_branch_likely(&have_mio)) { 289 ret = __memcpy_fromio_inuser( 290 user_buffer, (const void __iomem *)mmio_addr, 291 length); 292 return ret; 293 } 294 295 if (length > 64) { 296 buf = kmalloc(length, GFP_KERNEL); 297 if (!buf) 298 return -ENOMEM; 299 } else { 300 buf = local_buf; 301 } 302 303 mmap_read_lock(current->mm); 304 ret = -EINVAL; 305 vma = find_vma(current->mm, mmio_addr); 306 if (!vma) 307 goto out_unlock_mmap; 308 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 309 goto out_unlock_mmap; 310 ret = -EACCES; 311 if (!(vma->vm_flags & VM_WRITE)) 312 goto out_unlock_mmap; 313 314 ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); 315 if (ret) 316 goto out_unlock_mmap; 317 318 io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | 319 (mmio_addr & ~PAGE_MASK)); 320 321 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { 322 ret = -EFAULT; 323 goto out_unlock_pt; 324 } 325 ret = zpci_memcpy_fromio(buf, io_addr, length); 326 327 out_unlock_pt: 328 pte_unmap_unlock(ptep, ptl); 329 out_unlock_mmap: 330 mmap_read_unlock(current->mm); 331 332 if (!ret && copy_to_user(user_buffer, buf, length)) 333 ret = -EFAULT; 334 335 if (buf != local_buf) 336 kfree(buf); 337 return ret; 338 } 339