1 /* 2 * A fairly generic DMA-API to IOMMU-API glue layer. 3 * 4 * Copyright (C) 2014-2015 ARM Ltd. 5 * 6 * based in part on arch/arm/mm/dma-mapping.c: 7 * Copyright (C) 2000-2004 Russell King 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program. If not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include <linux/device.h> 23 #include <linux/dma-iommu.h> 24 #include <linux/gfp.h> 25 #include <linux/huge_mm.h> 26 #include <linux/iommu.h> 27 #include <linux/iova.h> 28 #include <linux/irq.h> 29 #include <linux/mm.h> 30 #include <linux/pci.h> 31 #include <linux/scatterlist.h> 32 #include <linux/vmalloc.h> 33 34 struct iommu_dma_msi_page { 35 struct list_head list; 36 dma_addr_t iova; 37 phys_addr_t phys; 38 }; 39 40 struct iommu_dma_cookie { 41 struct iova_domain iovad; 42 struct list_head msi_page_list; 43 spinlock_t msi_lock; 44 }; 45 46 static inline struct iova_domain *cookie_iovad(struct iommu_domain *domain) 47 { 48 return &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad; 49 } 50 51 int iommu_dma_init(void) 52 { 53 return iova_cache_get(); 54 } 55 56 /** 57 * iommu_get_dma_cookie - Acquire DMA-API resources for a domain 58 * @domain: IOMMU domain to prepare for DMA-API usage 59 * 60 * IOMMU drivers should normally call this from their domain_alloc 61 * callback when domain->type == IOMMU_DOMAIN_DMA. 62 */ 63 int iommu_get_dma_cookie(struct iommu_domain *domain) 64 { 65 struct iommu_dma_cookie *cookie; 66 67 if (domain->iova_cookie) 68 return -EEXIST; 69 70 cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); 71 if (!cookie) 72 return -ENOMEM; 73 74 spin_lock_init(&cookie->msi_lock); 75 INIT_LIST_HEAD(&cookie->msi_page_list); 76 domain->iova_cookie = cookie; 77 return 0; 78 } 79 EXPORT_SYMBOL(iommu_get_dma_cookie); 80 81 /** 82 * iommu_put_dma_cookie - Release a domain's DMA mapping resources 83 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() 84 * 85 * IOMMU drivers should normally call this from their domain_free callback. 86 */ 87 void iommu_put_dma_cookie(struct iommu_domain *domain) 88 { 89 struct iommu_dma_cookie *cookie = domain->iova_cookie; 90 struct iommu_dma_msi_page *msi, *tmp; 91 92 if (!cookie) 93 return; 94 95 if (cookie->iovad.granule) 96 put_iova_domain(&cookie->iovad); 97 98 list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { 99 list_del(&msi->list); 100 kfree(msi); 101 } 102 kfree(cookie); 103 domain->iova_cookie = NULL; 104 } 105 EXPORT_SYMBOL(iommu_put_dma_cookie); 106 107 static void iova_reserve_pci_windows(struct pci_dev *dev, 108 struct iova_domain *iovad) 109 { 110 struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); 111 struct resource_entry *window; 112 unsigned long lo, hi; 113 114 resource_list_for_each_entry(window, &bridge->windows) { 115 if (resource_type(window->res) != IORESOURCE_MEM && 116 resource_type(window->res) != IORESOURCE_IO) 117 continue; 118 119 lo = iova_pfn(iovad, window->res->start - window->offset); 120 hi = iova_pfn(iovad, window->res->end - window->offset); 121 reserve_iova(iovad, lo, hi); 122 } 123 } 124 125 /** 126 * iommu_dma_init_domain - Initialise a DMA mapping domain 127 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() 128 * @base: IOVA at which the mappable address space starts 129 * @size: Size of IOVA space 130 * @dev: Device the domain is being initialised for 131 * 132 * @base and @size should be exact multiples of IOMMU page granularity to 133 * avoid rounding surprises. If necessary, we reserve the page at address 0 134 * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but 135 * any change which could make prior IOVAs invalid will fail. 136 */ 137 int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, 138 u64 size, struct device *dev) 139 { 140 struct iova_domain *iovad = cookie_iovad(domain); 141 unsigned long order, base_pfn, end_pfn; 142 143 if (!iovad) 144 return -ENODEV; 145 146 /* Use the smallest supported page size for IOVA granularity */ 147 order = __ffs(domain->pgsize_bitmap); 148 base_pfn = max_t(unsigned long, 1, base >> order); 149 end_pfn = (base + size - 1) >> order; 150 151 /* Check the domain allows at least some access to the device... */ 152 if (domain->geometry.force_aperture) { 153 if (base > domain->geometry.aperture_end || 154 base + size <= domain->geometry.aperture_start) { 155 pr_warn("specified DMA range outside IOMMU capability\n"); 156 return -EFAULT; 157 } 158 /* ...then finally give it a kicking to make sure it fits */ 159 base_pfn = max_t(unsigned long, base_pfn, 160 domain->geometry.aperture_start >> order); 161 end_pfn = min_t(unsigned long, end_pfn, 162 domain->geometry.aperture_end >> order); 163 } 164 165 /* All we can safely do with an existing domain is enlarge it */ 166 if (iovad->start_pfn) { 167 if (1UL << order != iovad->granule || 168 base_pfn != iovad->start_pfn || 169 end_pfn < iovad->dma_32bit_pfn) { 170 pr_warn("Incompatible range for DMA domain\n"); 171 return -EFAULT; 172 } 173 iovad->dma_32bit_pfn = end_pfn; 174 } else { 175 init_iova_domain(iovad, 1UL << order, base_pfn, end_pfn); 176 if (dev && dev_is_pci(dev)) 177 iova_reserve_pci_windows(to_pci_dev(dev), iovad); 178 } 179 return 0; 180 } 181 EXPORT_SYMBOL(iommu_dma_init_domain); 182 183 /** 184 * dma_direction_to_prot - Translate DMA API directions to IOMMU API page flags 185 * @dir: Direction of DMA transfer 186 * @coherent: Is the DMA master cache-coherent? 187 * 188 * Return: corresponding IOMMU API page protection flags 189 */ 190 int dma_direction_to_prot(enum dma_data_direction dir, bool coherent) 191 { 192 int prot = coherent ? IOMMU_CACHE : 0; 193 194 switch (dir) { 195 case DMA_BIDIRECTIONAL: 196 return prot | IOMMU_READ | IOMMU_WRITE; 197 case DMA_TO_DEVICE: 198 return prot | IOMMU_READ; 199 case DMA_FROM_DEVICE: 200 return prot | IOMMU_WRITE; 201 default: 202 return 0; 203 } 204 } 205 206 static struct iova *__alloc_iova(struct iommu_domain *domain, size_t size, 207 dma_addr_t dma_limit) 208 { 209 struct iova_domain *iovad = cookie_iovad(domain); 210 unsigned long shift = iova_shift(iovad); 211 unsigned long length = iova_align(iovad, size) >> shift; 212 213 if (domain->geometry.force_aperture) 214 dma_limit = min(dma_limit, domain->geometry.aperture_end); 215 /* 216 * Enforce size-alignment to be safe - there could perhaps be an 217 * attribute to control this per-device, or at least per-domain... 218 */ 219 return alloc_iova(iovad, length, dma_limit >> shift, true); 220 } 221 222 /* The IOVA allocator knows what we mapped, so just unmap whatever that was */ 223 static void __iommu_dma_unmap(struct iommu_domain *domain, dma_addr_t dma_addr) 224 { 225 struct iova_domain *iovad = cookie_iovad(domain); 226 unsigned long shift = iova_shift(iovad); 227 unsigned long pfn = dma_addr >> shift; 228 struct iova *iova = find_iova(iovad, pfn); 229 size_t size; 230 231 if (WARN_ON(!iova)) 232 return; 233 234 size = iova_size(iova) << shift; 235 size -= iommu_unmap(domain, pfn << shift, size); 236 /* ...and if we can't, then something is horribly, horribly wrong */ 237 WARN_ON(size > 0); 238 __free_iova(iovad, iova); 239 } 240 241 static void __iommu_dma_free_pages(struct page **pages, int count) 242 { 243 while (count--) 244 __free_page(pages[count]); 245 kvfree(pages); 246 } 247 248 static struct page **__iommu_dma_alloc_pages(unsigned int count, 249 unsigned long order_mask, gfp_t gfp) 250 { 251 struct page **pages; 252 unsigned int i = 0, array_size = count * sizeof(*pages); 253 254 order_mask &= (2U << MAX_ORDER) - 1; 255 if (!order_mask) 256 return NULL; 257 258 if (array_size <= PAGE_SIZE) 259 pages = kzalloc(array_size, GFP_KERNEL); 260 else 261 pages = vzalloc(array_size); 262 if (!pages) 263 return NULL; 264 265 /* IOMMU can map any pages, so himem can also be used here */ 266 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 267 268 while (count) { 269 struct page *page = NULL; 270 unsigned int order_size; 271 272 /* 273 * Higher-order allocations are a convenience rather 274 * than a necessity, hence using __GFP_NORETRY until 275 * falling back to minimum-order allocations. 276 */ 277 for (order_mask &= (2U << __fls(count)) - 1; 278 order_mask; order_mask &= ~order_size) { 279 unsigned int order = __fls(order_mask); 280 281 order_size = 1U << order; 282 page = alloc_pages((order_mask - order_size) ? 283 gfp | __GFP_NORETRY : gfp, order); 284 if (!page) 285 continue; 286 if (!order) 287 break; 288 if (!PageCompound(page)) { 289 split_page(page, order); 290 break; 291 } else if (!split_huge_page(page)) { 292 break; 293 } 294 __free_pages(page, order); 295 } 296 if (!page) { 297 __iommu_dma_free_pages(pages, i); 298 return NULL; 299 } 300 count -= order_size; 301 while (order_size--) 302 pages[i++] = page++; 303 } 304 return pages; 305 } 306 307 /** 308 * iommu_dma_free - Free a buffer allocated by iommu_dma_alloc() 309 * @dev: Device which owns this buffer 310 * @pages: Array of buffer pages as returned by iommu_dma_alloc() 311 * @size: Size of buffer in bytes 312 * @handle: DMA address of buffer 313 * 314 * Frees both the pages associated with the buffer, and the array 315 * describing them 316 */ 317 void iommu_dma_free(struct device *dev, struct page **pages, size_t size, 318 dma_addr_t *handle) 319 { 320 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), *handle); 321 __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); 322 *handle = DMA_ERROR_CODE; 323 } 324 325 /** 326 * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space 327 * @dev: Device to allocate memory for. Must be a real device 328 * attached to an iommu_dma_domain 329 * @size: Size of buffer in bytes 330 * @gfp: Allocation flags 331 * @attrs: DMA attributes for this allocation 332 * @prot: IOMMU mapping flags 333 * @handle: Out argument for allocated DMA handle 334 * @flush_page: Arch callback which must ensure PAGE_SIZE bytes from the 335 * given VA/PA are visible to the given non-coherent device. 336 * 337 * If @size is less than PAGE_SIZE, then a full CPU page will be allocated, 338 * but an IOMMU which supports smaller pages might not map the whole thing. 339 * 340 * Return: Array of struct page pointers describing the buffer, 341 * or NULL on failure. 342 */ 343 struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, 344 unsigned long attrs, int prot, dma_addr_t *handle, 345 void (*flush_page)(struct device *, const void *, phys_addr_t)) 346 { 347 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 348 struct iova_domain *iovad = cookie_iovad(domain); 349 struct iova *iova; 350 struct page **pages; 351 struct sg_table sgt; 352 dma_addr_t dma_addr; 353 unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; 354 355 *handle = DMA_ERROR_CODE; 356 357 min_size = alloc_sizes & -alloc_sizes; 358 if (min_size < PAGE_SIZE) { 359 min_size = PAGE_SIZE; 360 alloc_sizes |= PAGE_SIZE; 361 } else { 362 size = ALIGN(size, min_size); 363 } 364 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) 365 alloc_sizes = min_size; 366 367 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 368 pages = __iommu_dma_alloc_pages(count, alloc_sizes >> PAGE_SHIFT, gfp); 369 if (!pages) 370 return NULL; 371 372 iova = __alloc_iova(domain, size, dev->coherent_dma_mask); 373 if (!iova) 374 goto out_free_pages; 375 376 size = iova_align(iovad, size); 377 if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, GFP_KERNEL)) 378 goto out_free_iova; 379 380 if (!(prot & IOMMU_CACHE)) { 381 struct sg_mapping_iter miter; 382 /* 383 * The CPU-centric flushing implied by SG_MITER_TO_SG isn't 384 * sufficient here, so skip it by using the "wrong" direction. 385 */ 386 sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, SG_MITER_FROM_SG); 387 while (sg_miter_next(&miter)) 388 flush_page(dev, miter.addr, page_to_phys(miter.page)); 389 sg_miter_stop(&miter); 390 } 391 392 dma_addr = iova_dma_addr(iovad, iova); 393 if (iommu_map_sg(domain, dma_addr, sgt.sgl, sgt.orig_nents, prot) 394 < size) 395 goto out_free_sg; 396 397 *handle = dma_addr; 398 sg_free_table(&sgt); 399 return pages; 400 401 out_free_sg: 402 sg_free_table(&sgt); 403 out_free_iova: 404 __free_iova(iovad, iova); 405 out_free_pages: 406 __iommu_dma_free_pages(pages, count); 407 return NULL; 408 } 409 410 /** 411 * iommu_dma_mmap - Map a buffer into provided user VMA 412 * @pages: Array representing buffer from iommu_dma_alloc() 413 * @size: Size of buffer in bytes 414 * @vma: VMA describing requested userspace mapping 415 * 416 * Maps the pages of the buffer in @pages into @vma. The caller is responsible 417 * for verifying the correct size and protection of @vma beforehand. 418 */ 419 420 int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma) 421 { 422 unsigned long uaddr = vma->vm_start; 423 unsigned int i, count = PAGE_ALIGN(size) >> PAGE_SHIFT; 424 int ret = -ENXIO; 425 426 for (i = vma->vm_pgoff; i < count && uaddr < vma->vm_end; i++) { 427 ret = vm_insert_page(vma, uaddr, pages[i]); 428 if (ret) 429 break; 430 uaddr += PAGE_SIZE; 431 } 432 return ret; 433 } 434 435 dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 436 unsigned long offset, size_t size, int prot) 437 { 438 dma_addr_t dma_addr; 439 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 440 struct iova_domain *iovad = cookie_iovad(domain); 441 phys_addr_t phys = page_to_phys(page) + offset; 442 size_t iova_off = iova_offset(iovad, phys); 443 size_t len = iova_align(iovad, size + iova_off); 444 struct iova *iova = __alloc_iova(domain, len, dma_get_mask(dev)); 445 446 if (!iova) 447 return DMA_ERROR_CODE; 448 449 dma_addr = iova_dma_addr(iovad, iova); 450 if (iommu_map(domain, dma_addr, phys - iova_off, len, prot)) { 451 __free_iova(iovad, iova); 452 return DMA_ERROR_CODE; 453 } 454 return dma_addr + iova_off; 455 } 456 457 void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, 458 enum dma_data_direction dir, unsigned long attrs) 459 { 460 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle); 461 } 462 463 /* 464 * Prepare a successfully-mapped scatterlist to give back to the caller. 465 * 466 * At this point the segments are already laid out by iommu_dma_map_sg() to 467 * avoid individually crossing any boundaries, so we merely need to check a 468 * segment's start address to avoid concatenating across one. 469 */ 470 static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, 471 dma_addr_t dma_addr) 472 { 473 struct scatterlist *s, *cur = sg; 474 unsigned long seg_mask = dma_get_seg_boundary(dev); 475 unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); 476 int i, count = 0; 477 478 for_each_sg(sg, s, nents, i) { 479 /* Restore this segment's original unaligned fields first */ 480 unsigned int s_iova_off = sg_dma_address(s); 481 unsigned int s_length = sg_dma_len(s); 482 unsigned int s_iova_len = s->length; 483 484 s->offset += s_iova_off; 485 s->length = s_length; 486 sg_dma_address(s) = DMA_ERROR_CODE; 487 sg_dma_len(s) = 0; 488 489 /* 490 * Now fill in the real DMA data. If... 491 * - there is a valid output segment to append to 492 * - and this segment starts on an IOVA page boundary 493 * - but doesn't fall at a segment boundary 494 * - and wouldn't make the resulting output segment too long 495 */ 496 if (cur_len && !s_iova_off && (dma_addr & seg_mask) && 497 (cur_len + s_length <= max_len)) { 498 /* ...then concatenate it with the previous one */ 499 cur_len += s_length; 500 } else { 501 /* Otherwise start the next output segment */ 502 if (i > 0) 503 cur = sg_next(cur); 504 cur_len = s_length; 505 count++; 506 507 sg_dma_address(cur) = dma_addr + s_iova_off; 508 } 509 510 sg_dma_len(cur) = cur_len; 511 dma_addr += s_iova_len; 512 513 if (s_length + s_iova_off < s_iova_len) 514 cur_len = 0; 515 } 516 return count; 517 } 518 519 /* 520 * If mapping failed, then just restore the original list, 521 * but making sure the DMA fields are invalidated. 522 */ 523 static void __invalidate_sg(struct scatterlist *sg, int nents) 524 { 525 struct scatterlist *s; 526 int i; 527 528 for_each_sg(sg, s, nents, i) { 529 if (sg_dma_address(s) != DMA_ERROR_CODE) 530 s->offset += sg_dma_address(s); 531 if (sg_dma_len(s)) 532 s->length = sg_dma_len(s); 533 sg_dma_address(s) = DMA_ERROR_CODE; 534 sg_dma_len(s) = 0; 535 } 536 } 537 538 /* 539 * The DMA API client is passing in a scatterlist which could describe 540 * any old buffer layout, but the IOMMU API requires everything to be 541 * aligned to IOMMU pages. Hence the need for this complicated bit of 542 * impedance-matching, to be able to hand off a suitably-aligned list, 543 * but still preserve the original offsets and sizes for the caller. 544 */ 545 int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, 546 int nents, int prot) 547 { 548 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 549 struct iova_domain *iovad = cookie_iovad(domain); 550 struct iova *iova; 551 struct scatterlist *s, *prev = NULL; 552 dma_addr_t dma_addr; 553 size_t iova_len = 0; 554 unsigned long mask = dma_get_seg_boundary(dev); 555 int i; 556 557 /* 558 * Work out how much IOVA space we need, and align the segments to 559 * IOVA granules for the IOMMU driver to handle. With some clever 560 * trickery we can modify the list in-place, but reversibly, by 561 * stashing the unaligned parts in the as-yet-unused DMA fields. 562 */ 563 for_each_sg(sg, s, nents, i) { 564 size_t s_iova_off = iova_offset(iovad, s->offset); 565 size_t s_length = s->length; 566 size_t pad_len = (mask - iova_len + 1) & mask; 567 568 sg_dma_address(s) = s_iova_off; 569 sg_dma_len(s) = s_length; 570 s->offset -= s_iova_off; 571 s_length = iova_align(iovad, s_length + s_iova_off); 572 s->length = s_length; 573 574 /* 575 * Due to the alignment of our single IOVA allocation, we can 576 * depend on these assumptions about the segment boundary mask: 577 * - If mask size >= IOVA size, then the IOVA range cannot 578 * possibly fall across a boundary, so we don't care. 579 * - If mask size < IOVA size, then the IOVA range must start 580 * exactly on a boundary, therefore we can lay things out 581 * based purely on segment lengths without needing to know 582 * the actual addresses beforehand. 583 * - The mask must be a power of 2, so pad_len == 0 if 584 * iova_len == 0, thus we cannot dereference prev the first 585 * time through here (i.e. before it has a meaningful value). 586 */ 587 if (pad_len && pad_len < s_length - 1) { 588 prev->length += pad_len; 589 iova_len += pad_len; 590 } 591 592 iova_len += s_length; 593 prev = s; 594 } 595 596 iova = __alloc_iova(domain, iova_len, dma_get_mask(dev)); 597 if (!iova) 598 goto out_restore_sg; 599 600 /* 601 * We'll leave any physical concatenation to the IOMMU driver's 602 * implementation - it knows better than we do. 603 */ 604 dma_addr = iova_dma_addr(iovad, iova); 605 if (iommu_map_sg(domain, dma_addr, sg, nents, prot) < iova_len) 606 goto out_free_iova; 607 608 return __finalise_sg(dev, sg, nents, dma_addr); 609 610 out_free_iova: 611 __free_iova(iovad, iova); 612 out_restore_sg: 613 __invalidate_sg(sg, nents); 614 return 0; 615 } 616 617 void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 618 enum dma_data_direction dir, unsigned long attrs) 619 { 620 /* 621 * The scatterlist segments are mapped into a single 622 * contiguous IOVA allocation, so this is incredibly easy. 623 */ 624 __iommu_dma_unmap(iommu_get_domain_for_dev(dev), sg_dma_address(sg)); 625 } 626 627 int iommu_dma_supported(struct device *dev, u64 mask) 628 { 629 /* 630 * 'Special' IOMMUs which don't have the same addressing capability 631 * as the CPU will have to wait until we have some way to query that 632 * before they'll be able to use this framework. 633 */ 634 return 1; 635 } 636 637 int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 638 { 639 return dma_addr == DMA_ERROR_CODE; 640 } 641 642 static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, 643 phys_addr_t msi_addr, struct iommu_domain *domain) 644 { 645 struct iommu_dma_cookie *cookie = domain->iova_cookie; 646 struct iommu_dma_msi_page *msi_page; 647 struct iova_domain *iovad = &cookie->iovad; 648 struct iova *iova; 649 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 650 651 msi_addr &= ~(phys_addr_t)iova_mask(iovad); 652 list_for_each_entry(msi_page, &cookie->msi_page_list, list) 653 if (msi_page->phys == msi_addr) 654 return msi_page; 655 656 msi_page = kzalloc(sizeof(*msi_page), GFP_ATOMIC); 657 if (!msi_page) 658 return NULL; 659 660 iova = __alloc_iova(domain, iovad->granule, dma_get_mask(dev)); 661 if (!iova) 662 goto out_free_page; 663 664 msi_page->phys = msi_addr; 665 msi_page->iova = iova_dma_addr(iovad, iova); 666 if (iommu_map(domain, msi_page->iova, msi_addr, iovad->granule, prot)) 667 goto out_free_iova; 668 669 INIT_LIST_HEAD(&msi_page->list); 670 list_add(&msi_page->list, &cookie->msi_page_list); 671 return msi_page; 672 673 out_free_iova: 674 __free_iova(iovad, iova); 675 out_free_page: 676 kfree(msi_page); 677 return NULL; 678 } 679 680 void iommu_dma_map_msi_msg(int irq, struct msi_msg *msg) 681 { 682 struct device *dev = msi_desc_to_dev(irq_get_msi_desc(irq)); 683 struct iommu_domain *domain = iommu_get_domain_for_dev(dev); 684 struct iommu_dma_cookie *cookie; 685 struct iommu_dma_msi_page *msi_page; 686 phys_addr_t msi_addr = (u64)msg->address_hi << 32 | msg->address_lo; 687 unsigned long flags; 688 689 if (!domain || !domain->iova_cookie) 690 return; 691 692 cookie = domain->iova_cookie; 693 694 /* 695 * We disable IRQs to rule out a possible inversion against 696 * irq_desc_lock if, say, someone tries to retarget the affinity 697 * of an MSI from within an IPI handler. 698 */ 699 spin_lock_irqsave(&cookie->msi_lock, flags); 700 msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain); 701 spin_unlock_irqrestore(&cookie->msi_lock, flags); 702 703 if (WARN_ON(!msi_page)) { 704 /* 705 * We're called from a void callback, so the best we can do is 706 * 'fail' by filling the message with obviously bogus values. 707 * Since we got this far due to an IOMMU being present, it's 708 * not like the existing address would have worked anyway... 709 */ 710 msg->address_hi = ~0U; 711 msg->address_lo = ~0U; 712 msg->data = ~0U; 713 } else { 714 msg->address_hi = upper_32_bits(msi_page->iova); 715 msg->address_lo &= iova_mask(&cookie->iovad); 716 msg->address_lo += lower_32_bits(msi_page->iova); 717 } 718 } 719