1 /* 2 * linux/arch/alpha/kernel/pci_iommu.c 3 */ 4 5 #include <linux/kernel.h> 6 #include <linux/mm.h> 7 #include <linux/pci.h> 8 #include <linux/gfp.h> 9 #include <linux/bootmem.h> 10 #include <linux/export.h> 11 #include <linux/scatterlist.h> 12 #include <linux/log2.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/iommu-helper.h> 15 16 #include <asm/io.h> 17 #include <asm/hwrpb.h> 18 19 #include "proto.h" 20 #include "pci_impl.h" 21 22 23 #define DEBUG_ALLOC 0 24 #if DEBUG_ALLOC > 0 25 # define DBGA(args...) printk(KERN_DEBUG args) 26 #else 27 # define DBGA(args...) 28 #endif 29 #if DEBUG_ALLOC > 1 30 # define DBGA2(args...) printk(KERN_DEBUG args) 31 #else 32 # define DBGA2(args...) 33 #endif 34 35 #define DEBUG_NODIRECT 0 36 37 #define ISA_DMA_MASK 0x00ffffff 38 39 static inline unsigned long 40 mk_iommu_pte(unsigned long paddr) 41 { 42 return (paddr >> (PAGE_SHIFT-1)) | 1; 43 } 44 45 /* Return the minimum of MAX or the first power of two larger 46 than main memory. */ 47 48 unsigned long 49 size_for_memory(unsigned long max) 50 { 51 unsigned long mem = max_low_pfn << PAGE_SHIFT; 52 if (mem < max) 53 max = roundup_pow_of_two(mem); 54 return max; 55 } 56 57 struct pci_iommu_arena * __init 58 iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, 59 unsigned long window_size, unsigned long align) 60 { 61 unsigned long mem_size; 62 struct pci_iommu_arena *arena; 63 64 mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long)); 65 66 /* Note that the TLB lookup logic uses bitwise concatenation, 67 not addition, so the required arena alignment is based on 68 the size of the window. Retain the align parameter so that 69 particular systems can over-align the arena. */ 70 if (align < mem_size) 71 align = mem_size; 72 73 74 #ifdef CONFIG_DISCONTIGMEM 75 76 arena = alloc_bootmem_node(NODE_DATA(nid), sizeof(*arena)); 77 if (!NODE_DATA(nid) || !arena) { 78 printk("%s: couldn't allocate arena from node %d\n" 79 " falling back to system-wide allocation\n", 80 __func__, nid); 81 arena = alloc_bootmem(sizeof(*arena)); 82 } 83 84 arena->ptes = __alloc_bootmem_node(NODE_DATA(nid), mem_size, align, 0); 85 if (!NODE_DATA(nid) || !arena->ptes) { 86 printk("%s: couldn't allocate arena ptes from node %d\n" 87 " falling back to system-wide allocation\n", 88 __func__, nid); 89 arena->ptes = __alloc_bootmem(mem_size, align, 0); 90 } 91 92 #else /* CONFIG_DISCONTIGMEM */ 93 94 arena = alloc_bootmem(sizeof(*arena)); 95 arena->ptes = __alloc_bootmem(mem_size, align, 0); 96 97 #endif /* CONFIG_DISCONTIGMEM */ 98 99 spin_lock_init(&arena->lock); 100 arena->hose = hose; 101 arena->dma_base = base; 102 arena->size = window_size; 103 arena->next_entry = 0; 104 105 /* Align allocations to a multiple of a page size. Not needed 106 unless there are chip bugs. */ 107 arena->align_entry = 1; 108 109 return arena; 110 } 111 112 struct pci_iommu_arena * __init 113 iommu_arena_new(struct pci_controller *hose, dma_addr_t base, 114 unsigned long window_size, unsigned long align) 115 { 116 return iommu_arena_new_node(0, hose, base, window_size, align); 117 } 118 119 /* Must be called with the arena lock held */ 120 static long 121 iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena, 122 long n, long mask) 123 { 124 unsigned long *ptes; 125 long i, p, nent; 126 int pass = 0; 127 unsigned long base; 128 unsigned long boundary_size; 129 130 base = arena->dma_base >> PAGE_SHIFT; 131 if (dev) { 132 boundary_size = dma_get_seg_boundary(dev) + 1; 133 boundary_size >>= PAGE_SHIFT; 134 } else { 135 boundary_size = 1UL << (32 - PAGE_SHIFT); 136 } 137 138 /* Search forward for the first mask-aligned sequence of N free ptes */ 139 ptes = arena->ptes; 140 nent = arena->size >> PAGE_SHIFT; 141 p = ALIGN(arena->next_entry, mask + 1); 142 i = 0; 143 144 again: 145 while (i < n && p+i < nent) { 146 if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) { 147 p = ALIGN(p + 1, mask + 1); 148 goto again; 149 } 150 151 if (ptes[p+i]) 152 p = ALIGN(p + i + 1, mask + 1), i = 0; 153 else 154 i = i + 1; 155 } 156 157 if (i < n) { 158 if (pass < 1) { 159 /* 160 * Reached the end. Flush the TLB and restart 161 * the search from the beginning. 162 */ 163 alpha_mv.mv_pci_tbi(arena->hose, 0, -1); 164 165 pass++; 166 p = 0; 167 i = 0; 168 goto again; 169 } else 170 return -1; 171 } 172 173 /* Success. It's the responsibility of the caller to mark them 174 in use before releasing the lock */ 175 return p; 176 } 177 178 static long 179 iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n, 180 unsigned int align) 181 { 182 unsigned long flags; 183 unsigned long *ptes; 184 long i, p, mask; 185 186 spin_lock_irqsave(&arena->lock, flags); 187 188 /* Search for N empty ptes */ 189 ptes = arena->ptes; 190 mask = max(align, arena->align_entry) - 1; 191 p = iommu_arena_find_pages(dev, arena, n, mask); 192 if (p < 0) { 193 spin_unlock_irqrestore(&arena->lock, flags); 194 return -1; 195 } 196 197 /* Success. Mark them all in use, ie not zero and invalid 198 for the iommu tlb that could load them from under us. 199 The chip specific bits will fill this in with something 200 kosher when we return. */ 201 for (i = 0; i < n; ++i) 202 ptes[p+i] = IOMMU_INVALID_PTE; 203 204 arena->next_entry = p + n; 205 spin_unlock_irqrestore(&arena->lock, flags); 206 207 return p; 208 } 209 210 static void 211 iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n) 212 { 213 unsigned long *p; 214 long i; 215 216 p = arena->ptes + ofs; 217 for (i = 0; i < n; ++i) 218 p[i] = 0; 219 } 220 221 /* 222 * True if the machine supports DAC addressing, and DEV can 223 * make use of it given MASK. 224 */ 225 static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask) 226 { 227 dma_addr_t dac_offset = alpha_mv.pci_dac_offset; 228 int ok = 1; 229 230 /* If this is not set, the machine doesn't support DAC at all. */ 231 if (dac_offset == 0) 232 ok = 0; 233 234 /* The device has to be able to address our DAC bit. */ 235 if ((dac_offset & dev->dma_mask) != dac_offset) 236 ok = 0; 237 238 /* If both conditions above are met, we are fine. */ 239 DBGA("pci_dac_dma_supported %s from %p\n", 240 ok ? "yes" : "no", __builtin_return_address(0)); 241 242 return ok; 243 } 244 245 /* Map a single buffer of the indicated size for PCI DMA in streaming 246 mode. The 32-bit PCI bus mastering address to use is returned. 247 Once the device is given the dma address, the device owns this memory 248 until either pci_unmap_single or pci_dma_sync_single is performed. */ 249 250 static dma_addr_t 251 pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, 252 int dac_allowed) 253 { 254 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; 255 dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; 256 struct pci_iommu_arena *arena; 257 long npages, dma_ofs, i; 258 unsigned long paddr; 259 dma_addr_t ret; 260 unsigned int align = 0; 261 struct device *dev = pdev ? &pdev->dev : NULL; 262 263 paddr = __pa(cpu_addr); 264 265 #if !DEBUG_NODIRECT 266 /* First check to see if we can use the direct map window. */ 267 if (paddr + size + __direct_map_base - 1 <= max_dma 268 && paddr + size <= __direct_map_size) { 269 ret = paddr + __direct_map_base; 270 271 DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %p\n", 272 cpu_addr, size, ret, __builtin_return_address(0)); 273 274 return ret; 275 } 276 #endif 277 278 /* Next, use DAC if selected earlier. */ 279 if (dac_allowed) { 280 ret = paddr + alpha_mv.pci_dac_offset; 281 282 DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %p\n", 283 cpu_addr, size, ret, __builtin_return_address(0)); 284 285 return ret; 286 } 287 288 /* If the machine doesn't define a pci_tbi routine, we have to 289 assume it doesn't support sg mapping, and, since we tried to 290 use direct_map above, it now must be considered an error. */ 291 if (! alpha_mv.mv_pci_tbi) { 292 printk_once(KERN_WARNING "pci_map_single: no HW sg\n"); 293 return 0; 294 } 295 296 arena = hose->sg_pci; 297 if (!arena || arena->dma_base + arena->size - 1 > max_dma) 298 arena = hose->sg_isa; 299 300 npages = iommu_num_pages(paddr, size, PAGE_SIZE); 301 302 /* Force allocation to 64KB boundary for ISA bridges. */ 303 if (pdev && pdev == isa_bridge) 304 align = 8; 305 dma_ofs = iommu_arena_alloc(dev, arena, npages, align); 306 if (dma_ofs < 0) { 307 printk(KERN_WARNING "pci_map_single failed: " 308 "could not allocate dma page tables\n"); 309 return 0; 310 } 311 312 paddr &= PAGE_MASK; 313 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) 314 arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr); 315 316 ret = arena->dma_base + dma_ofs * PAGE_SIZE; 317 ret += (unsigned long)cpu_addr & ~PAGE_MASK; 318 319 DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %p\n", 320 cpu_addr, size, npages, ret, __builtin_return_address(0)); 321 322 return ret; 323 } 324 325 /* Helper for generic DMA-mapping functions. */ 326 static struct pci_dev *alpha_gendev_to_pci(struct device *dev) 327 { 328 if (dev && dev->bus == &pci_bus_type) 329 return to_pci_dev(dev); 330 331 /* Assume that non-PCI devices asking for DMA are either ISA or EISA, 332 BUG() otherwise. */ 333 BUG_ON(!isa_bridge); 334 335 /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA 336 bridge is bus master then). */ 337 if (!dev || !dev->dma_mask || !*dev->dma_mask) 338 return isa_bridge; 339 340 /* For EISA bus masters, return isa_bridge (it might have smaller 341 dma_mask due to wiring limitations). */ 342 if (*dev->dma_mask >= isa_bridge->dma_mask) 343 return isa_bridge; 344 345 /* This assumes ISA bus master with dma_mask 0xffffff. */ 346 return NULL; 347 } 348 349 static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, 350 unsigned long offset, size_t size, 351 enum dma_data_direction dir, 352 struct dma_attrs *attrs) 353 { 354 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 355 int dac_allowed; 356 357 if (dir == PCI_DMA_NONE) 358 BUG(); 359 360 dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; 361 return pci_map_single_1(pdev, (char *)page_address(page) + offset, 362 size, dac_allowed); 363 } 364 365 /* Unmap a single streaming mode DMA translation. The DMA_ADDR and 366 SIZE must match what was provided for in a previous pci_map_single 367 call. All other usages are undefined. After this call, reads by 368 the cpu to the buffer are guaranteed to see whatever the device 369 wrote there. */ 370 371 static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, 372 size_t size, enum dma_data_direction dir, 373 struct dma_attrs *attrs) 374 { 375 unsigned long flags; 376 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 377 struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; 378 struct pci_iommu_arena *arena; 379 long dma_ofs, npages; 380 381 if (dir == PCI_DMA_NONE) 382 BUG(); 383 384 if (dma_addr >= __direct_map_base 385 && dma_addr < __direct_map_base + __direct_map_size) { 386 /* Nothing to do. */ 387 388 DBGA2("pci_unmap_single: direct [%llx,%zx] from %p\n", 389 dma_addr, size, __builtin_return_address(0)); 390 391 return; 392 } 393 394 if (dma_addr > 0xffffffff) { 395 DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %p\n", 396 dma_addr, size, __builtin_return_address(0)); 397 return; 398 } 399 400 arena = hose->sg_pci; 401 if (!arena || dma_addr < arena->dma_base) 402 arena = hose->sg_isa; 403 404 dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT; 405 if (dma_ofs * PAGE_SIZE >= arena->size) { 406 printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx " 407 " base %llx size %x\n", 408 dma_addr, arena->dma_base, arena->size); 409 return; 410 BUG(); 411 } 412 413 npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); 414 415 spin_lock_irqsave(&arena->lock, flags); 416 417 iommu_arena_free(arena, dma_ofs, npages); 418 419 /* If we're freeing ptes above the `next_entry' pointer (they 420 may have snuck back into the TLB since the last wrap flush), 421 we need to flush the TLB before reallocating the latter. */ 422 if (dma_ofs >= arena->next_entry) 423 alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1); 424 425 spin_unlock_irqrestore(&arena->lock, flags); 426 427 DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %p\n", 428 dma_addr, size, npages, __builtin_return_address(0)); 429 } 430 431 /* Allocate and map kernel buffer using consistent mode DMA for PCI 432 device. Returns non-NULL cpu-view pointer to the buffer if 433 successful and sets *DMA_ADDRP to the pci side dma address as well, 434 else DMA_ADDRP is undefined. */ 435 436 static void *alpha_pci_alloc_coherent(struct device *dev, size_t size, 437 dma_addr_t *dma_addrp, gfp_t gfp) 438 { 439 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 440 void *cpu_addr; 441 long order = get_order(size); 442 443 gfp &= ~GFP_DMA; 444 445 try_again: 446 cpu_addr = (void *)__get_free_pages(gfp, order); 447 if (! cpu_addr) { 448 printk(KERN_INFO "pci_alloc_consistent: " 449 "get_free_pages failed from %p\n", 450 __builtin_return_address(0)); 451 /* ??? Really atomic allocation? Otherwise we could play 452 with vmalloc and sg if we can't find contiguous memory. */ 453 return NULL; 454 } 455 memset(cpu_addr, 0, size); 456 457 *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0); 458 if (*dma_addrp == 0) { 459 free_pages((unsigned long)cpu_addr, order); 460 if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA)) 461 return NULL; 462 /* The address doesn't fit required mask and we 463 do not have iommu. Try again with GFP_DMA. */ 464 gfp |= GFP_DMA; 465 goto try_again; 466 } 467 468 DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %p\n", 469 size, cpu_addr, *dma_addrp, __builtin_return_address(0)); 470 471 return cpu_addr; 472 } 473 474 /* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must 475 be values that were returned from pci_alloc_consistent. SIZE must 476 be the same as what as passed into pci_alloc_consistent. 477 References to the memory and mappings associated with CPU_ADDR or 478 DMA_ADDR past this call are illegal. */ 479 480 static void alpha_pci_free_coherent(struct device *dev, size_t size, 481 void *cpu_addr, dma_addr_t dma_addr) 482 { 483 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 484 pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); 485 free_pages((unsigned long)cpu_addr, get_order(size)); 486 487 DBGA2("pci_free_consistent: [%llx,%zx] from %p\n", 488 dma_addr, size, __builtin_return_address(0)); 489 } 490 491 /* Classify the elements of the scatterlist. Write dma_address 492 of each element with: 493 0 : Followers all physically adjacent. 494 1 : Followers all virtually adjacent. 495 -1 : Not leader, physically adjacent to previous. 496 -2 : Not leader, virtually adjacent to previous. 497 Write dma_length of each leader with the combined lengths of 498 the mergable followers. */ 499 500 #define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) 501 #define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG)) 502 503 static void 504 sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end, 505 int virt_ok) 506 { 507 unsigned long next_paddr; 508 struct scatterlist *leader; 509 long leader_flag, leader_length; 510 unsigned int max_seg_size; 511 512 leader = sg; 513 leader_flag = 0; 514 leader_length = leader->length; 515 next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length; 516 517 /* we will not marge sg without device. */ 518 max_seg_size = dev ? dma_get_max_seg_size(dev) : 0; 519 for (++sg; sg < end; ++sg) { 520 unsigned long addr, len; 521 addr = SG_ENT_PHYS_ADDRESS(sg); 522 len = sg->length; 523 524 if (leader_length + len > max_seg_size) 525 goto new_segment; 526 527 if (next_paddr == addr) { 528 sg->dma_address = -1; 529 leader_length += len; 530 } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) { 531 sg->dma_address = -2; 532 leader_flag = 1; 533 leader_length += len; 534 } else { 535 new_segment: 536 leader->dma_address = leader_flag; 537 leader->dma_length = leader_length; 538 leader = sg; 539 leader_flag = 0; 540 leader_length = len; 541 } 542 543 next_paddr = addr + len; 544 } 545 546 leader->dma_address = leader_flag; 547 leader->dma_length = leader_length; 548 } 549 550 /* Given a scatterlist leader, choose an allocation method and fill 551 in the blanks. */ 552 553 static int 554 sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end, 555 struct scatterlist *out, struct pci_iommu_arena *arena, 556 dma_addr_t max_dma, int dac_allowed) 557 { 558 unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader); 559 long size = leader->dma_length; 560 struct scatterlist *sg; 561 unsigned long *ptes; 562 long npages, dma_ofs, i; 563 564 #if !DEBUG_NODIRECT 565 /* If everything is physically contiguous, and the addresses 566 fall into the direct-map window, use it. */ 567 if (leader->dma_address == 0 568 && paddr + size + __direct_map_base - 1 <= max_dma 569 && paddr + size <= __direct_map_size) { 570 out->dma_address = paddr + __direct_map_base; 571 out->dma_length = size; 572 573 DBGA(" sg_fill: [%p,%lx] -> direct %llx\n", 574 __va(paddr), size, out->dma_address); 575 576 return 0; 577 } 578 #endif 579 580 /* If physically contiguous and DAC is available, use it. */ 581 if (leader->dma_address == 0 && dac_allowed) { 582 out->dma_address = paddr + alpha_mv.pci_dac_offset; 583 out->dma_length = size; 584 585 DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n", 586 __va(paddr), size, out->dma_address); 587 588 return 0; 589 } 590 591 /* Otherwise, we'll use the iommu to make the pages virtually 592 contiguous. */ 593 594 paddr &= ~PAGE_MASK; 595 npages = iommu_num_pages(paddr, size, PAGE_SIZE); 596 dma_ofs = iommu_arena_alloc(dev, arena, npages, 0); 597 if (dma_ofs < 0) { 598 /* If we attempted a direct map above but failed, die. */ 599 if (leader->dma_address == 0) 600 return -1; 601 602 /* Otherwise, break up the remaining virtually contiguous 603 hunks into individual direct maps and retry. */ 604 sg_classify(dev, leader, end, 0); 605 return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed); 606 } 607 608 out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr; 609 out->dma_length = size; 610 611 DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n", 612 __va(paddr), size, out->dma_address, npages); 613 614 /* All virtually contiguous. We need to find the length of each 615 physically contiguous subsegment to fill in the ptes. */ 616 ptes = &arena->ptes[dma_ofs]; 617 sg = leader; 618 do { 619 #if DEBUG_ALLOC > 0 620 struct scatterlist *last_sg = sg; 621 #endif 622 623 size = sg->length; 624 paddr = SG_ENT_PHYS_ADDRESS(sg); 625 626 while (sg+1 < end && (int) sg[1].dma_address == -1) { 627 size += sg[1].length; 628 sg++; 629 } 630 631 npages = iommu_num_pages(paddr, size, PAGE_SIZE); 632 633 paddr &= PAGE_MASK; 634 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) 635 *ptes++ = mk_iommu_pte(paddr); 636 637 #if DEBUG_ALLOC > 0 638 DBGA(" (%ld) [%p,%x] np %ld\n", 639 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), 640 last_sg->length, npages); 641 while (++last_sg <= sg) { 642 DBGA(" (%ld) [%p,%x] cont\n", 643 last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), 644 last_sg->length); 645 } 646 #endif 647 } while (++sg < end && (int) sg->dma_address < 0); 648 649 return 1; 650 } 651 652 static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, 653 int nents, enum dma_data_direction dir, 654 struct dma_attrs *attrs) 655 { 656 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 657 struct scatterlist *start, *end, *out; 658 struct pci_controller *hose; 659 struct pci_iommu_arena *arena; 660 dma_addr_t max_dma; 661 int dac_allowed; 662 663 if (dir == PCI_DMA_NONE) 664 BUG(); 665 666 dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; 667 668 /* Fast path single entry scatterlists. */ 669 if (nents == 1) { 670 sg->dma_length = sg->length; 671 sg->dma_address 672 = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg), 673 sg->length, dac_allowed); 674 return sg->dma_address != 0; 675 } 676 677 start = sg; 678 end = sg + nents; 679 680 /* First, prepare information about the entries. */ 681 sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0); 682 683 /* Second, figure out where we're going to map things. */ 684 if (alpha_mv.mv_pci_tbi) { 685 hose = pdev ? pdev->sysdata : pci_isa_hose; 686 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; 687 arena = hose->sg_pci; 688 if (!arena || arena->dma_base + arena->size - 1 > max_dma) 689 arena = hose->sg_isa; 690 } else { 691 max_dma = -1; 692 arena = NULL; 693 hose = NULL; 694 } 695 696 /* Third, iterate over the scatterlist leaders and allocate 697 dma space as needed. */ 698 for (out = sg; sg < end; ++sg) { 699 if ((int) sg->dma_address < 0) 700 continue; 701 if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0) 702 goto error; 703 out++; 704 } 705 706 /* Mark the end of the list for pci_unmap_sg. */ 707 if (out < end) 708 out->dma_length = 0; 709 710 if (out - start == 0) 711 printk(KERN_WARNING "pci_map_sg failed: no entries?\n"); 712 DBGA("pci_map_sg: %ld entries\n", out - start); 713 714 return out - start; 715 716 error: 717 printk(KERN_WARNING "pci_map_sg failed: " 718 "could not allocate dma page tables\n"); 719 720 /* Some allocation failed while mapping the scatterlist 721 entries. Unmap them now. */ 722 if (out > start) 723 pci_unmap_sg(pdev, start, out - start, dir); 724 return 0; 725 } 726 727 /* Unmap a set of streaming mode DMA translations. Again, cpu read 728 rules concerning calls here are the same as for pci_unmap_single() 729 above. */ 730 731 static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg, 732 int nents, enum dma_data_direction dir, 733 struct dma_attrs *attrs) 734 { 735 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 736 unsigned long flags; 737 struct pci_controller *hose; 738 struct pci_iommu_arena *arena; 739 struct scatterlist *end; 740 dma_addr_t max_dma; 741 dma_addr_t fbeg, fend; 742 743 if (dir == PCI_DMA_NONE) 744 BUG(); 745 746 if (! alpha_mv.mv_pci_tbi) 747 return; 748 749 hose = pdev ? pdev->sysdata : pci_isa_hose; 750 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; 751 arena = hose->sg_pci; 752 if (!arena || arena->dma_base + arena->size - 1 > max_dma) 753 arena = hose->sg_isa; 754 755 fbeg = -1, fend = 0; 756 757 spin_lock_irqsave(&arena->lock, flags); 758 759 for (end = sg + nents; sg < end; ++sg) { 760 dma_addr_t addr; 761 size_t size; 762 long npages, ofs; 763 dma_addr_t tend; 764 765 addr = sg->dma_address; 766 size = sg->dma_length; 767 if (!size) 768 break; 769 770 if (addr > 0xffffffff) { 771 /* It's a DAC address -- nothing to do. */ 772 DBGA(" (%ld) DAC [%llx,%zx]\n", 773 sg - end + nents, addr, size); 774 continue; 775 } 776 777 if (addr >= __direct_map_base 778 && addr < __direct_map_base + __direct_map_size) { 779 /* Nothing to do. */ 780 DBGA(" (%ld) direct [%llx,%zx]\n", 781 sg - end + nents, addr, size); 782 continue; 783 } 784 785 DBGA(" (%ld) sg [%llx,%zx]\n", 786 sg - end + nents, addr, size); 787 788 npages = iommu_num_pages(addr, size, PAGE_SIZE); 789 ofs = (addr - arena->dma_base) >> PAGE_SHIFT; 790 iommu_arena_free(arena, ofs, npages); 791 792 tend = addr + size - 1; 793 if (fbeg > addr) fbeg = addr; 794 if (fend < tend) fend = tend; 795 } 796 797 /* If we're freeing ptes above the `next_entry' pointer (they 798 may have snuck back into the TLB since the last wrap flush), 799 we need to flush the TLB before reallocating the latter. */ 800 if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry) 801 alpha_mv.mv_pci_tbi(hose, fbeg, fend); 802 803 spin_unlock_irqrestore(&arena->lock, flags); 804 805 DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg)); 806 } 807 808 /* Return whether the given PCI device DMA address mask can be 809 supported properly. */ 810 811 static int alpha_pci_supported(struct device *dev, u64 mask) 812 { 813 struct pci_dev *pdev = alpha_gendev_to_pci(dev); 814 struct pci_controller *hose; 815 struct pci_iommu_arena *arena; 816 817 /* If there exists a direct map, and the mask fits either 818 the entire direct mapped space or the total system memory as 819 shifted by the map base */ 820 if (__direct_map_size != 0 821 && (__direct_map_base + __direct_map_size - 1 <= mask || 822 __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask)) 823 return 1; 824 825 /* Check that we have a scatter-gather arena that fits. */ 826 hose = pdev ? pdev->sysdata : pci_isa_hose; 827 arena = hose->sg_isa; 828 if (arena && arena->dma_base + arena->size - 1 <= mask) 829 return 1; 830 arena = hose->sg_pci; 831 if (arena && arena->dma_base + arena->size - 1 <= mask) 832 return 1; 833 834 /* As last resort try ZONE_DMA. */ 835 if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask) 836 return 1; 837 838 return 0; 839 } 840 841 842 /* 843 * AGP GART extensions to the IOMMU 844 */ 845 int 846 iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask) 847 { 848 unsigned long flags; 849 unsigned long *ptes; 850 long i, p; 851 852 if (!arena) return -EINVAL; 853 854 spin_lock_irqsave(&arena->lock, flags); 855 856 /* Search for N empty ptes. */ 857 ptes = arena->ptes; 858 p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask); 859 if (p < 0) { 860 spin_unlock_irqrestore(&arena->lock, flags); 861 return -1; 862 } 863 864 /* Success. Mark them all reserved (ie not zero and invalid) 865 for the iommu tlb that could load them from under us. 866 They will be filled in with valid bits by _bind() */ 867 for (i = 0; i < pg_count; ++i) 868 ptes[p+i] = IOMMU_RESERVED_PTE; 869 870 arena->next_entry = p + pg_count; 871 spin_unlock_irqrestore(&arena->lock, flags); 872 873 return p; 874 } 875 876 int 877 iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count) 878 { 879 unsigned long *ptes; 880 long i; 881 882 if (!arena) return -EINVAL; 883 884 ptes = arena->ptes; 885 886 /* Make sure they're all reserved first... */ 887 for(i = pg_start; i < pg_start + pg_count; i++) 888 if (ptes[i] != IOMMU_RESERVED_PTE) 889 return -EBUSY; 890 891 iommu_arena_free(arena, pg_start, pg_count); 892 return 0; 893 } 894 895 int 896 iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count, 897 struct page **pages) 898 { 899 unsigned long flags; 900 unsigned long *ptes; 901 long i, j; 902 903 if (!arena) return -EINVAL; 904 905 spin_lock_irqsave(&arena->lock, flags); 906 907 ptes = arena->ptes; 908 909 for(j = pg_start; j < pg_start + pg_count; j++) { 910 if (ptes[j] != IOMMU_RESERVED_PTE) { 911 spin_unlock_irqrestore(&arena->lock, flags); 912 return -EBUSY; 913 } 914 } 915 916 for(i = 0, j = pg_start; i < pg_count; i++, j++) 917 ptes[j] = mk_iommu_pte(page_to_phys(pages[i])); 918 919 spin_unlock_irqrestore(&arena->lock, flags); 920 921 return 0; 922 } 923 924 int 925 iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count) 926 { 927 unsigned long *p; 928 long i; 929 930 if (!arena) return -EINVAL; 931 932 p = arena->ptes + pg_start; 933 for(i = 0; i < pg_count; i++) 934 p[i] = IOMMU_RESERVED_PTE; 935 936 return 0; 937 } 938 939 static int alpha_pci_mapping_error(struct device *dev, dma_addr_t dma_addr) 940 { 941 return dma_addr == 0; 942 } 943 944 static int alpha_pci_set_mask(struct device *dev, u64 mask) 945 { 946 if (!dev->dma_mask || 947 !pci_dma_supported(alpha_gendev_to_pci(dev), mask)) 948 return -EIO; 949 950 *dev->dma_mask = mask; 951 return 0; 952 } 953 954 struct dma_map_ops alpha_pci_ops = { 955 .alloc_coherent = alpha_pci_alloc_coherent, 956 .free_coherent = alpha_pci_free_coherent, 957 .map_page = alpha_pci_map_page, 958 .unmap_page = alpha_pci_unmap_page, 959 .map_sg = alpha_pci_map_sg, 960 .unmap_sg = alpha_pci_unmap_sg, 961 .mapping_error = alpha_pci_mapping_error, 962 .dma_supported = alpha_pci_supported, 963 .set_dma_mask = alpha_pci_set_mask, 964 }; 965 966 struct dma_map_ops *dma_ops = &alpha_pci_ops; 967 EXPORT_SYMBOL(dma_ops); 968