1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * IOMMU API for s390 PCI devices 4 * 5 * Copyright IBM Corp. 2015 6 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> 7 */ 8 9 #include <linux/pci.h> 10 #include <linux/iommu.h> 11 #include <linux/iommu-helper.h> 12 #include <linux/sizes.h> 13 #include <linux/rculist.h> 14 #include <linux/rcupdate.h> 15 #include <asm/pci_dma.h> 16 17 static const struct iommu_ops s390_iommu_ops; 18 19 struct s390_domain { 20 struct iommu_domain domain; 21 struct list_head devices; 22 unsigned long *dma_table; 23 spinlock_t list_lock; 24 struct rcu_head rcu; 25 }; 26 27 static struct s390_domain *to_s390_domain(struct iommu_domain *dom) 28 { 29 return container_of(dom, struct s390_domain, domain); 30 } 31 32 static bool s390_iommu_capable(struct device *dev, enum iommu_cap cap) 33 { 34 switch (cap) { 35 case IOMMU_CAP_CACHE_COHERENCY: 36 return true; 37 case IOMMU_CAP_INTR_REMAP: 38 return true; 39 default: 40 return false; 41 } 42 } 43 44 static struct iommu_domain *s390_domain_alloc(unsigned domain_type) 45 { 46 struct s390_domain *s390_domain; 47 48 if (domain_type != IOMMU_DOMAIN_UNMANAGED) 49 return NULL; 50 51 s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL); 52 if (!s390_domain) 53 return NULL; 54 55 s390_domain->dma_table = dma_alloc_cpu_table(); 56 if (!s390_domain->dma_table) { 57 kfree(s390_domain); 58 return NULL; 59 } 60 s390_domain->domain.geometry.force_aperture = true; 61 s390_domain->domain.geometry.aperture_start = 0; 62 s390_domain->domain.geometry.aperture_end = ZPCI_TABLE_SIZE_RT - 1; 63 64 spin_lock_init(&s390_domain->list_lock); 65 INIT_LIST_HEAD_RCU(&s390_domain->devices); 66 67 return &s390_domain->domain; 68 } 69 70 static void s390_iommu_rcu_free_domain(struct rcu_head *head) 71 { 72 struct s390_domain *s390_domain = container_of(head, struct s390_domain, rcu); 73 74 dma_cleanup_tables(s390_domain->dma_table); 75 kfree(s390_domain); 76 } 77 78 static void s390_domain_free(struct iommu_domain *domain) 79 { 80 struct s390_domain *s390_domain = to_s390_domain(domain); 81 82 rcu_read_lock(); 83 WARN_ON(!list_empty(&s390_domain->devices)); 84 rcu_read_unlock(); 85 86 call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain); 87 } 88 89 static void __s390_iommu_detach_device(struct zpci_dev *zdev) 90 { 91 struct s390_domain *s390_domain = zdev->s390_domain; 92 unsigned long flags; 93 94 if (!s390_domain) 95 return; 96 97 spin_lock_irqsave(&s390_domain->list_lock, flags); 98 list_del_rcu(&zdev->iommu_list); 99 spin_unlock_irqrestore(&s390_domain->list_lock, flags); 100 101 zpci_unregister_ioat(zdev, 0); 102 zdev->s390_domain = NULL; 103 zdev->dma_table = NULL; 104 } 105 106 static int s390_iommu_attach_device(struct iommu_domain *domain, 107 struct device *dev) 108 { 109 struct s390_domain *s390_domain = to_s390_domain(domain); 110 struct zpci_dev *zdev = to_zpci_dev(dev); 111 unsigned long flags; 112 u8 status; 113 int cc; 114 115 if (!zdev) 116 return -ENODEV; 117 118 if (WARN_ON(domain->geometry.aperture_start > zdev->end_dma || 119 domain->geometry.aperture_end < zdev->start_dma)) 120 return -EINVAL; 121 122 if (zdev->s390_domain) 123 __s390_iommu_detach_device(zdev); 124 else if (zdev->dma_table) 125 zpci_dma_exit_device(zdev); 126 127 cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, 128 virt_to_phys(s390_domain->dma_table), &status); 129 /* 130 * If the device is undergoing error recovery the reset code 131 * will re-establish the new domain. 132 */ 133 if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL) 134 return -EIO; 135 zdev->dma_table = s390_domain->dma_table; 136 137 zdev->dma_table = s390_domain->dma_table; 138 zdev->s390_domain = s390_domain; 139 140 spin_lock_irqsave(&s390_domain->list_lock, flags); 141 list_add_rcu(&zdev->iommu_list, &s390_domain->devices); 142 spin_unlock_irqrestore(&s390_domain->list_lock, flags); 143 144 return 0; 145 } 146 147 static void s390_iommu_detach_device(struct iommu_domain *domain, 148 struct device *dev) 149 { 150 struct zpci_dev *zdev = to_zpci_dev(dev); 151 152 WARN_ON(zdev->s390_domain != to_s390_domain(domain)); 153 154 __s390_iommu_detach_device(zdev); 155 zpci_dma_init_device(zdev); 156 } 157 158 static void s390_iommu_get_resv_regions(struct device *dev, 159 struct list_head *list) 160 { 161 struct zpci_dev *zdev = to_zpci_dev(dev); 162 struct iommu_resv_region *region; 163 164 if (zdev->start_dma) { 165 region = iommu_alloc_resv_region(0, zdev->start_dma, 0, 166 IOMMU_RESV_RESERVED, GFP_KERNEL); 167 if (!region) 168 return; 169 list_add_tail(®ion->list, list); 170 } 171 172 if (zdev->end_dma < ZPCI_TABLE_SIZE_RT - 1) { 173 region = iommu_alloc_resv_region(zdev->end_dma + 1, 174 ZPCI_TABLE_SIZE_RT - zdev->end_dma - 1, 175 0, IOMMU_RESV_RESERVED, GFP_KERNEL); 176 if (!region) 177 return; 178 list_add_tail(®ion->list, list); 179 } 180 } 181 182 static struct iommu_device *s390_iommu_probe_device(struct device *dev) 183 { 184 struct zpci_dev *zdev; 185 186 if (!dev_is_pci(dev)) 187 return ERR_PTR(-ENODEV); 188 189 zdev = to_zpci_dev(dev); 190 191 if (zdev->start_dma > zdev->end_dma || 192 zdev->start_dma > ZPCI_TABLE_SIZE_RT - 1) 193 return ERR_PTR(-EINVAL); 194 195 if (zdev->end_dma > ZPCI_TABLE_SIZE_RT - 1) 196 zdev->end_dma = ZPCI_TABLE_SIZE_RT - 1; 197 198 return &zdev->iommu_dev; 199 } 200 201 static void s390_iommu_release_device(struct device *dev) 202 { 203 struct zpci_dev *zdev = to_zpci_dev(dev); 204 205 /* 206 * release_device is expected to detach any domain currently attached 207 * to the device, but keep it attached to other devices in the group. 208 */ 209 if (zdev) 210 __s390_iommu_detach_device(zdev); 211 } 212 213 static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain) 214 { 215 struct s390_domain *s390_domain = to_s390_domain(domain); 216 struct zpci_dev *zdev; 217 218 rcu_read_lock(); 219 list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { 220 zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma, 221 zdev->end_dma - zdev->start_dma + 1); 222 } 223 rcu_read_unlock(); 224 } 225 226 static void s390_iommu_iotlb_sync(struct iommu_domain *domain, 227 struct iommu_iotlb_gather *gather) 228 { 229 struct s390_domain *s390_domain = to_s390_domain(domain); 230 size_t size = gather->end - gather->start + 1; 231 struct zpci_dev *zdev; 232 233 /* If gather was never added to there is nothing to flush */ 234 if (!gather->end) 235 return; 236 237 rcu_read_lock(); 238 list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { 239 zpci_refresh_trans((u64)zdev->fh << 32, gather->start, 240 size); 241 } 242 rcu_read_unlock(); 243 } 244 245 static void s390_iommu_iotlb_sync_map(struct iommu_domain *domain, 246 unsigned long iova, size_t size) 247 { 248 struct s390_domain *s390_domain = to_s390_domain(domain); 249 struct zpci_dev *zdev; 250 251 rcu_read_lock(); 252 list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { 253 if (!zdev->tlb_refresh) 254 continue; 255 zpci_refresh_trans((u64)zdev->fh << 32, 256 iova, size); 257 } 258 rcu_read_unlock(); 259 } 260 261 static int s390_iommu_validate_trans(struct s390_domain *s390_domain, 262 phys_addr_t pa, dma_addr_t dma_addr, 263 unsigned long nr_pages, int flags) 264 { 265 phys_addr_t page_addr = pa & PAGE_MASK; 266 unsigned long *entry; 267 unsigned long i; 268 int rc; 269 270 for (i = 0; i < nr_pages; i++) { 271 entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr); 272 if (unlikely(!entry)) { 273 rc = -ENOMEM; 274 goto undo_cpu_trans; 275 } 276 dma_update_cpu_trans(entry, page_addr, flags); 277 page_addr += PAGE_SIZE; 278 dma_addr += PAGE_SIZE; 279 } 280 281 return 0; 282 283 undo_cpu_trans: 284 while (i-- > 0) { 285 dma_addr -= PAGE_SIZE; 286 entry = dma_walk_cpu_trans(s390_domain->dma_table, 287 dma_addr); 288 if (!entry) 289 break; 290 dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID); 291 } 292 293 return rc; 294 } 295 296 static int s390_iommu_invalidate_trans(struct s390_domain *s390_domain, 297 dma_addr_t dma_addr, unsigned long nr_pages) 298 { 299 unsigned long *entry; 300 unsigned long i; 301 int rc = 0; 302 303 for (i = 0; i < nr_pages; i++) { 304 entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr); 305 if (unlikely(!entry)) { 306 rc = -EINVAL; 307 break; 308 } 309 dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID); 310 dma_addr += PAGE_SIZE; 311 } 312 313 return rc; 314 } 315 316 static int s390_iommu_map_pages(struct iommu_domain *domain, 317 unsigned long iova, phys_addr_t paddr, 318 size_t pgsize, size_t pgcount, 319 int prot, gfp_t gfp, size_t *mapped) 320 { 321 struct s390_domain *s390_domain = to_s390_domain(domain); 322 size_t size = pgcount << __ffs(pgsize); 323 int flags = ZPCI_PTE_VALID, rc = 0; 324 325 if (pgsize != SZ_4K) 326 return -EINVAL; 327 328 if (iova < s390_domain->domain.geometry.aperture_start || 329 (iova + size - 1) > s390_domain->domain.geometry.aperture_end) 330 return -EINVAL; 331 332 if (!IS_ALIGNED(iova | paddr, pgsize)) 333 return -EINVAL; 334 335 if (!(prot & IOMMU_READ)) 336 return -EINVAL; 337 338 if (!(prot & IOMMU_WRITE)) 339 flags |= ZPCI_TABLE_PROTECTED; 340 341 rc = s390_iommu_validate_trans(s390_domain, paddr, iova, 342 pgcount, flags); 343 if (!rc) 344 *mapped = size; 345 346 return rc; 347 } 348 349 static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain, 350 dma_addr_t iova) 351 { 352 struct s390_domain *s390_domain = to_s390_domain(domain); 353 unsigned long *rto, *sto, *pto; 354 unsigned long ste, pte, rte; 355 unsigned int rtx, sx, px; 356 phys_addr_t phys = 0; 357 358 if (iova < domain->geometry.aperture_start || 359 iova > domain->geometry.aperture_end) 360 return 0; 361 362 rtx = calc_rtx(iova); 363 sx = calc_sx(iova); 364 px = calc_px(iova); 365 rto = s390_domain->dma_table; 366 367 rte = READ_ONCE(rto[rtx]); 368 if (reg_entry_isvalid(rte)) { 369 sto = get_rt_sto(rte); 370 ste = READ_ONCE(sto[sx]); 371 if (reg_entry_isvalid(ste)) { 372 pto = get_st_pto(ste); 373 pte = READ_ONCE(pto[px]); 374 if (pt_entry_isvalid(pte)) 375 phys = pte & ZPCI_PTE_ADDR_MASK; 376 } 377 } 378 379 return phys; 380 } 381 382 static size_t s390_iommu_unmap_pages(struct iommu_domain *domain, 383 unsigned long iova, 384 size_t pgsize, size_t pgcount, 385 struct iommu_iotlb_gather *gather) 386 { 387 struct s390_domain *s390_domain = to_s390_domain(domain); 388 size_t size = pgcount << __ffs(pgsize); 389 int rc; 390 391 if (WARN_ON(iova < s390_domain->domain.geometry.aperture_start || 392 (iova + size - 1) > s390_domain->domain.geometry.aperture_end)) 393 return 0; 394 395 rc = s390_iommu_invalidate_trans(s390_domain, iova, pgcount); 396 if (rc) 397 return 0; 398 399 iommu_iotlb_gather_add_range(gather, iova, size); 400 401 return size; 402 } 403 404 int zpci_init_iommu(struct zpci_dev *zdev) 405 { 406 int rc = 0; 407 408 rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL, 409 "s390-iommu.%08x", zdev->fid); 410 if (rc) 411 goto out_err; 412 413 rc = iommu_device_register(&zdev->iommu_dev, &s390_iommu_ops, NULL); 414 if (rc) 415 goto out_sysfs; 416 417 return 0; 418 419 out_sysfs: 420 iommu_device_sysfs_remove(&zdev->iommu_dev); 421 422 out_err: 423 return rc; 424 } 425 426 void zpci_destroy_iommu(struct zpci_dev *zdev) 427 { 428 iommu_device_unregister(&zdev->iommu_dev); 429 iommu_device_sysfs_remove(&zdev->iommu_dev); 430 } 431 432 static const struct iommu_ops s390_iommu_ops = { 433 .capable = s390_iommu_capable, 434 .domain_alloc = s390_domain_alloc, 435 .probe_device = s390_iommu_probe_device, 436 .release_device = s390_iommu_release_device, 437 .device_group = generic_device_group, 438 .pgsize_bitmap = SZ_4K, 439 .get_resv_regions = s390_iommu_get_resv_regions, 440 .default_domain_ops = &(const struct iommu_domain_ops) { 441 .attach_dev = s390_iommu_attach_device, 442 .detach_dev = s390_iommu_detach_device, 443 .map_pages = s390_iommu_map_pages, 444 .unmap_pages = s390_iommu_unmap_pages, 445 .flush_iotlb_all = s390_iommu_flush_iotlb_all, 446 .iotlb_sync = s390_iommu_iotlb_sync, 447 .iotlb_sync_map = s390_iommu_iotlb_sync_map, 448 .iova_to_phys = s390_iommu_iova_to_phys, 449 .free = s390_domain_free, 450 } 451 }; 452