xref: /openbmc/linux/arch/alpha/kernel/pci_iommu.c (revision 05d2e16a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *	linux/arch/alpha/kernel/pci_iommu.c
4  */
5 
6 #include <linux/kernel.h>
7 #include <linux/mm.h>
8 #include <linux/pci.h>
9 #include <linux/gfp.h>
10 #include <linux/memblock.h>
11 #include <linux/export.h>
12 #include <linux/scatterlist.h>
13 #include <linux/log2.h>
14 #include <linux/dma-map-ops.h>
15 #include <linux/iommu-helper.h>
16 
17 #include <asm/io.h>
18 #include <asm/hwrpb.h>
19 
20 #include "proto.h"
21 #include "pci_impl.h"
22 
23 
24 #define DEBUG_ALLOC 0
25 #if DEBUG_ALLOC > 0
26 # define DBGA(args...)		printk(KERN_DEBUG args)
27 #else
28 # define DBGA(args...)
29 #endif
30 #if DEBUG_ALLOC > 1
31 # define DBGA2(args...)		printk(KERN_DEBUG args)
32 #else
33 # define DBGA2(args...)
34 #endif
35 
36 #define DEBUG_NODIRECT 0
37 
38 #define ISA_DMA_MASK		0x00ffffff
39 
40 static inline unsigned long
mk_iommu_pte(unsigned long paddr)41 mk_iommu_pte(unsigned long paddr)
42 {
43 	return (paddr >> (PAGE_SHIFT-1)) | 1;
44 }
45 
46 /* Return the minimum of MAX or the first power of two larger
47    than main memory.  */
48 
49 unsigned long
size_for_memory(unsigned long max)50 size_for_memory(unsigned long max)
51 {
52 	unsigned long mem = max_low_pfn << PAGE_SHIFT;
53 	if (mem < max)
54 		max = roundup_pow_of_two(mem);
55 	return max;
56 }
57 
58 struct pci_iommu_arena * __init
iommu_arena_new_node(int nid,struct pci_controller * hose,dma_addr_t base,unsigned long window_size,unsigned long align)59 iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base,
60 		     unsigned long window_size, unsigned long align)
61 {
62 	unsigned long mem_size;
63 	struct pci_iommu_arena *arena;
64 
65 	mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long));
66 
67 	/* Note that the TLB lookup logic uses bitwise concatenation,
68 	   not addition, so the required arena alignment is based on
69 	   the size of the window.  Retain the align parameter so that
70 	   particular systems can over-align the arena.  */
71 	if (align < mem_size)
72 		align = mem_size;
73 
74 	arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES);
75 	if (!arena)
76 		panic("%s: Failed to allocate %zu bytes\n", __func__,
77 		      sizeof(*arena));
78 	arena->ptes = memblock_alloc(mem_size, align);
79 	if (!arena->ptes)
80 		panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
81 		      __func__, mem_size, align);
82 
83 	spin_lock_init(&arena->lock);
84 	arena->hose = hose;
85 	arena->dma_base = base;
86 	arena->size = window_size;
87 	arena->next_entry = 0;
88 
89 	/* Align allocations to a multiple of a page size.  Not needed
90 	   unless there are chip bugs.  */
91 	arena->align_entry = 1;
92 
93 	return arena;
94 }
95 
96 struct pci_iommu_arena * __init
iommu_arena_new(struct pci_controller * hose,dma_addr_t base,unsigned long window_size,unsigned long align)97 iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
98 		unsigned long window_size, unsigned long align)
99 {
100 	return iommu_arena_new_node(0, hose, base, window_size, align);
101 }
102 
103 /* Must be called with the arena lock held */
104 static long
iommu_arena_find_pages(struct device * dev,struct pci_iommu_arena * arena,long n,long mask)105 iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
106 		       long n, long mask)
107 {
108 	unsigned long *ptes;
109 	long i, p, nent;
110 	int pass = 0;
111 	unsigned long base;
112 	unsigned long boundary_size;
113 
114 	base = arena->dma_base >> PAGE_SHIFT;
115 	boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
116 
117 	/* Search forward for the first mask-aligned sequence of N free ptes */
118 	ptes = arena->ptes;
119 	nent = arena->size >> PAGE_SHIFT;
120 	p = ALIGN(arena->next_entry, mask + 1);
121 	i = 0;
122 
123 again:
124 	while (i < n && p+i < nent) {
125 		if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) {
126 			p = ALIGN(p + 1, mask + 1);
127 			goto again;
128 		}
129 
130 		if (ptes[p+i]) {
131 			p = ALIGN(p + i + 1, mask + 1);
132 			i = 0;
133 		} else {
134 			i = i + 1;
135 		}
136 	}
137 
138 	if (i < n) {
139 		if (pass < 1) {
140 			/*
141 			 * Reached the end.  Flush the TLB and restart
142 			 * the search from the beginning.
143 			*/
144 			alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
145 
146 			pass++;
147 			p = 0;
148 			i = 0;
149 			goto again;
150 		} else
151 			return -1;
152 	}
153 
154 	/* Success. It's the responsibility of the caller to mark them
155 	   in use before releasing the lock */
156 	return p;
157 }
158 
159 static long
iommu_arena_alloc(struct device * dev,struct pci_iommu_arena * arena,long n,unsigned int align)160 iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
161 		  unsigned int align)
162 {
163 	unsigned long flags;
164 	unsigned long *ptes;
165 	long i, p, mask;
166 
167 	spin_lock_irqsave(&arena->lock, flags);
168 
169 	/* Search for N empty ptes */
170 	ptes = arena->ptes;
171 	mask = max(align, arena->align_entry) - 1;
172 	p = iommu_arena_find_pages(dev, arena, n, mask);
173 	if (p < 0) {
174 		spin_unlock_irqrestore(&arena->lock, flags);
175 		return -1;
176 	}
177 
178 	/* Success.  Mark them all in use, ie not zero and invalid
179 	   for the iommu tlb that could load them from under us.
180 	   The chip specific bits will fill this in with something
181 	   kosher when we return.  */
182 	for (i = 0; i < n; ++i)
183 		ptes[p+i] = IOMMU_INVALID_PTE;
184 
185 	arena->next_entry = p + n;
186 	spin_unlock_irqrestore(&arena->lock, flags);
187 
188 	return p;
189 }
190 
191 static void
iommu_arena_free(struct pci_iommu_arena * arena,long ofs,long n)192 iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n)
193 {
194 	unsigned long *p;
195 	long i;
196 
197 	p = arena->ptes + ofs;
198 	for (i = 0; i < n; ++i)
199 		p[i] = 0;
200 }
201 
202 /*
203  * True if the machine supports DAC addressing, and DEV can
204  * make use of it given MASK.
205  */
pci_dac_dma_supported(struct pci_dev * dev,u64 mask)206 static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
207 {
208 	dma_addr_t dac_offset = alpha_mv.pci_dac_offset;
209 	int ok = 1;
210 
211 	/* If this is not set, the machine doesn't support DAC at all.  */
212 	if (dac_offset == 0)
213 		ok = 0;
214 
215 	/* The device has to be able to address our DAC bit.  */
216 	if ((dac_offset & dev->dma_mask) != dac_offset)
217 		ok = 0;
218 
219 	/* If both conditions above are met, we are fine. */
220 	DBGA("pci_dac_dma_supported %s from %ps\n",
221 	     ok ? "yes" : "no", __builtin_return_address(0));
222 
223 	return ok;
224 }
225 
226 /* Map a single buffer of the indicated size for PCI DMA in streaming
227    mode.  The 32-bit PCI bus mastering address to use is returned.
228    Once the device is given the dma address, the device owns this memory
229    until either pci_unmap_single or pci_dma_sync_single is performed.  */
230 
231 static dma_addr_t
pci_map_single_1(struct pci_dev * pdev,void * cpu_addr,size_t size,int dac_allowed)232 pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size,
233 		 int dac_allowed)
234 {
235 	struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
236 	dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
237 	struct pci_iommu_arena *arena;
238 	long npages, dma_ofs, i;
239 	unsigned long paddr;
240 	dma_addr_t ret;
241 	unsigned int align = 0;
242 	struct device *dev = pdev ? &pdev->dev : NULL;
243 
244 	paddr = __pa(cpu_addr);
245 
246 #if !DEBUG_NODIRECT
247 	/* First check to see if we can use the direct map window.  */
248 	if (paddr + size + __direct_map_base - 1 <= max_dma
249 	    && paddr + size <= __direct_map_size) {
250 		ret = paddr + __direct_map_base;
251 
252 		DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %ps\n",
253 		      cpu_addr, size, ret, __builtin_return_address(0));
254 
255 		return ret;
256 	}
257 #endif
258 
259 	/* Next, use DAC if selected earlier.  */
260 	if (dac_allowed) {
261 		ret = paddr + alpha_mv.pci_dac_offset;
262 
263 		DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %ps\n",
264 		      cpu_addr, size, ret, __builtin_return_address(0));
265 
266 		return ret;
267 	}
268 
269 	/* If the machine doesn't define a pci_tbi routine, we have to
270 	   assume it doesn't support sg mapping, and, since we tried to
271 	   use direct_map above, it now must be considered an error. */
272 	if (! alpha_mv.mv_pci_tbi) {
273 		printk_once(KERN_WARNING "pci_map_single: no HW sg\n");
274 		return DMA_MAPPING_ERROR;
275 	}
276 
277 	arena = hose->sg_pci;
278 	if (!arena || arena->dma_base + arena->size - 1 > max_dma)
279 		arena = hose->sg_isa;
280 
281 	npages = iommu_num_pages(paddr, size, PAGE_SIZE);
282 
283 	/* Force allocation to 64KB boundary for ISA bridges. */
284 	if (pdev && pdev == isa_bridge)
285 		align = 8;
286 	dma_ofs = iommu_arena_alloc(dev, arena, npages, align);
287 	if (dma_ofs < 0) {
288 		printk(KERN_WARNING "pci_map_single failed: "
289 		       "could not allocate dma page tables\n");
290 		return DMA_MAPPING_ERROR;
291 	}
292 
293 	paddr &= PAGE_MASK;
294 	for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
295 		arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr);
296 
297 	ret = arena->dma_base + dma_ofs * PAGE_SIZE;
298 	ret += (unsigned long)cpu_addr & ~PAGE_MASK;
299 
300 	DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %ps\n",
301 	      cpu_addr, size, npages, ret, __builtin_return_address(0));
302 
303 	return ret;
304 }
305 
306 /* Helper for generic DMA-mapping functions. */
alpha_gendev_to_pci(struct device * dev)307 static struct pci_dev *alpha_gendev_to_pci(struct device *dev)
308 {
309 	if (dev && dev_is_pci(dev))
310 		return to_pci_dev(dev);
311 
312 	/* Assume that non-PCI devices asking for DMA are either ISA or EISA,
313 	   BUG() otherwise. */
314 	BUG_ON(!isa_bridge);
315 
316 	/* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA
317 	   bridge is bus master then). */
318 	if (!dev || !dev->dma_mask || !*dev->dma_mask)
319 		return isa_bridge;
320 
321 	/* For EISA bus masters, return isa_bridge (it might have smaller
322 	   dma_mask due to wiring limitations). */
323 	if (*dev->dma_mask >= isa_bridge->dma_mask)
324 		return isa_bridge;
325 
326 	/* This assumes ISA bus master with dma_mask 0xffffff. */
327 	return NULL;
328 }
329 
alpha_pci_map_page(struct device * dev,struct page * page,unsigned long offset,size_t size,enum dma_data_direction dir,unsigned long attrs)330 static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page,
331 				     unsigned long offset, size_t size,
332 				     enum dma_data_direction dir,
333 				     unsigned long attrs)
334 {
335 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
336 	int dac_allowed;
337 
338 	BUG_ON(dir == DMA_NONE);
339 
340 	dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
341 	return pci_map_single_1(pdev, (char *)page_address(page) + offset,
342 				size, dac_allowed);
343 }
344 
345 /* Unmap a single streaming mode DMA translation.  The DMA_ADDR and
346    SIZE must match what was provided for in a previous pci_map_single
347    call.  All other usages are undefined.  After this call, reads by
348    the cpu to the buffer are guaranteed to see whatever the device
349    wrote there.  */
350 
alpha_pci_unmap_page(struct device * dev,dma_addr_t dma_addr,size_t size,enum dma_data_direction dir,unsigned long attrs)351 static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr,
352 				 size_t size, enum dma_data_direction dir,
353 				 unsigned long attrs)
354 {
355 	unsigned long flags;
356 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
357 	struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
358 	struct pci_iommu_arena *arena;
359 	long dma_ofs, npages;
360 
361 	BUG_ON(dir == DMA_NONE);
362 
363 	if (dma_addr >= __direct_map_base
364 	    && dma_addr < __direct_map_base + __direct_map_size) {
365 		/* Nothing to do.  */
366 
367 		DBGA2("pci_unmap_single: direct [%llx,%zx] from %ps\n",
368 		      dma_addr, size, __builtin_return_address(0));
369 
370 		return;
371 	}
372 
373 	if (dma_addr > 0xffffffff) {
374 		DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %ps\n",
375 		      dma_addr, size, __builtin_return_address(0));
376 		return;
377 	}
378 
379 	arena = hose->sg_pci;
380 	if (!arena || dma_addr < arena->dma_base)
381 		arena = hose->sg_isa;
382 
383 	dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
384 	if (dma_ofs * PAGE_SIZE >= arena->size) {
385 		printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx "
386 		       " base %llx size %x\n",
387 		       dma_addr, arena->dma_base, arena->size);
388 		return;
389 		BUG();
390 	}
391 
392 	npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
393 
394 	spin_lock_irqsave(&arena->lock, flags);
395 
396 	iommu_arena_free(arena, dma_ofs, npages);
397 
398         /* If we're freeing ptes above the `next_entry' pointer (they
399            may have snuck back into the TLB since the last wrap flush),
400            we need to flush the TLB before reallocating the latter.  */
401 	if (dma_ofs >= arena->next_entry)
402 		alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
403 
404 	spin_unlock_irqrestore(&arena->lock, flags);
405 
406 	DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %ps\n",
407 	      dma_addr, size, npages, __builtin_return_address(0));
408 }
409 
410 /* Allocate and map kernel buffer using consistent mode DMA for PCI
411    device.  Returns non-NULL cpu-view pointer to the buffer if
412    successful and sets *DMA_ADDRP to the pci side dma address as well,
413    else DMA_ADDRP is undefined.  */
414 
alpha_pci_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_addrp,gfp_t gfp,unsigned long attrs)415 static void *alpha_pci_alloc_coherent(struct device *dev, size_t size,
416 				      dma_addr_t *dma_addrp, gfp_t gfp,
417 				      unsigned long attrs)
418 {
419 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
420 	void *cpu_addr;
421 	long order = get_order(size);
422 
423 	gfp &= ~GFP_DMA;
424 
425 try_again:
426 	cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order);
427 	if (! cpu_addr) {
428 		printk(KERN_INFO "pci_alloc_consistent: "
429 		       "get_free_pages failed from %ps\n",
430 			__builtin_return_address(0));
431 		/* ??? Really atomic allocation?  Otherwise we could play
432 		   with vmalloc and sg if we can't find contiguous memory.  */
433 		return NULL;
434 	}
435 	memset(cpu_addr, 0, size);
436 
437 	*dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0);
438 	if (*dma_addrp == DMA_MAPPING_ERROR) {
439 		free_pages((unsigned long)cpu_addr, order);
440 		if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA))
441 			return NULL;
442 		/* The address doesn't fit required mask and we
443 		   do not have iommu. Try again with GFP_DMA. */
444 		gfp |= GFP_DMA;
445 		goto try_again;
446 	}
447 
448 	DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n",
449 	      size, cpu_addr, *dma_addrp, __builtin_return_address(0));
450 
451 	return cpu_addr;
452 }
453 
454 /* Free and unmap a consistent DMA buffer.  CPU_ADDR and DMA_ADDR must
455    be values that were returned from pci_alloc_consistent.  SIZE must
456    be the same as what as passed into pci_alloc_consistent.
457    References to the memory and mappings associated with CPU_ADDR or
458    DMA_ADDR past this call are illegal.  */
459 
alpha_pci_free_coherent(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_addr,unsigned long attrs)460 static void alpha_pci_free_coherent(struct device *dev, size_t size,
461 				    void *cpu_addr, dma_addr_t dma_addr,
462 				    unsigned long attrs)
463 {
464 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
465 	dma_unmap_single(&pdev->dev, dma_addr, size, DMA_BIDIRECTIONAL);
466 	free_pages((unsigned long)cpu_addr, get_order(size));
467 
468 	DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n",
469 	      dma_addr, size, __builtin_return_address(0));
470 }
471 
472 /* Classify the elements of the scatterlist.  Write dma_address
473    of each element with:
474 	0   : Followers all physically adjacent.
475 	1   : Followers all virtually adjacent.
476 	-1  : Not leader, physically adjacent to previous.
477 	-2  : Not leader, virtually adjacent to previous.
478    Write dma_length of each leader with the combined lengths of
479    the mergable followers.  */
480 
481 #define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG)))
482 #define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG))
483 
484 static void
sg_classify(struct device * dev,struct scatterlist * sg,struct scatterlist * end,int virt_ok)485 sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end,
486 	    int virt_ok)
487 {
488 	unsigned long next_paddr;
489 	struct scatterlist *leader;
490 	long leader_flag, leader_length;
491 	unsigned int max_seg_size;
492 
493 	leader = sg;
494 	leader_flag = 0;
495 	leader_length = leader->length;
496 	next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
497 
498 	/* we will not marge sg without device. */
499 	max_seg_size = dev ? dma_get_max_seg_size(dev) : 0;
500 	for (++sg; sg < end; ++sg) {
501 		unsigned long addr, len;
502 		addr = SG_ENT_PHYS_ADDRESS(sg);
503 		len = sg->length;
504 
505 		if (leader_length + len > max_seg_size)
506 			goto new_segment;
507 
508 		if (next_paddr == addr) {
509 			sg->dma_address = -1;
510 			leader_length += len;
511 		} else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
512 			sg->dma_address = -2;
513 			leader_flag = 1;
514 			leader_length += len;
515 		} else {
516 new_segment:
517 			leader->dma_address = leader_flag;
518 			leader->dma_length = leader_length;
519 			leader = sg;
520 			leader_flag = 0;
521 			leader_length = len;
522 		}
523 
524 		next_paddr = addr + len;
525 	}
526 
527 	leader->dma_address = leader_flag;
528 	leader->dma_length = leader_length;
529 }
530 
531 /* Given a scatterlist leader, choose an allocation method and fill
532    in the blanks.  */
533 
534 static int
sg_fill(struct device * dev,struct scatterlist * leader,struct scatterlist * end,struct scatterlist * out,struct pci_iommu_arena * arena,dma_addr_t max_dma,int dac_allowed)535 sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end,
536 	struct scatterlist *out, struct pci_iommu_arena *arena,
537 	dma_addr_t max_dma, int dac_allowed)
538 {
539 	unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
540 	long size = leader->dma_length;
541 	struct scatterlist *sg;
542 	unsigned long *ptes;
543 	long npages, dma_ofs, i;
544 
545 #if !DEBUG_NODIRECT
546 	/* If everything is physically contiguous, and the addresses
547 	   fall into the direct-map window, use it.  */
548 	if (leader->dma_address == 0
549 	    && paddr + size + __direct_map_base - 1 <= max_dma
550 	    && paddr + size <= __direct_map_size) {
551 		out->dma_address = paddr + __direct_map_base;
552 		out->dma_length = size;
553 
554 		DBGA("    sg_fill: [%p,%lx] -> direct %llx\n",
555 		     __va(paddr), size, out->dma_address);
556 
557 		return 0;
558 	}
559 #endif
560 
561 	/* If physically contiguous and DAC is available, use it.  */
562 	if (leader->dma_address == 0 && dac_allowed) {
563 		out->dma_address = paddr + alpha_mv.pci_dac_offset;
564 		out->dma_length = size;
565 
566 		DBGA("    sg_fill: [%p,%lx] -> DAC %llx\n",
567 		     __va(paddr), size, out->dma_address);
568 
569 		return 0;
570 	}
571 
572 	/* Otherwise, we'll use the iommu to make the pages virtually
573 	   contiguous.  */
574 
575 	paddr &= ~PAGE_MASK;
576 	npages = iommu_num_pages(paddr, size, PAGE_SIZE);
577 	dma_ofs = iommu_arena_alloc(dev, arena, npages, 0);
578 	if (dma_ofs < 0) {
579 		/* If we attempted a direct map above but failed, die.  */
580 		if (leader->dma_address == 0)
581 			return -1;
582 
583 		/* Otherwise, break up the remaining virtually contiguous
584 		   hunks into individual direct maps and retry.  */
585 		sg_classify(dev, leader, end, 0);
586 		return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed);
587 	}
588 
589 	out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
590 	out->dma_length = size;
591 
592 	DBGA("    sg_fill: [%p,%lx] -> sg %llx np %ld\n",
593 	     __va(paddr), size, out->dma_address, npages);
594 
595 	/* All virtually contiguous.  We need to find the length of each
596 	   physically contiguous subsegment to fill in the ptes.  */
597 	ptes = &arena->ptes[dma_ofs];
598 	sg = leader;
599 	do {
600 #if DEBUG_ALLOC > 0
601 		struct scatterlist *last_sg = sg;
602 #endif
603 
604 		size = sg->length;
605 		paddr = SG_ENT_PHYS_ADDRESS(sg);
606 
607 		while (sg+1 < end && (int) sg[1].dma_address == -1) {
608 			size += sg[1].length;
609 			sg = sg_next(sg);
610 		}
611 
612 		npages = iommu_num_pages(paddr, size, PAGE_SIZE);
613 
614 		paddr &= PAGE_MASK;
615 		for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
616 			*ptes++ = mk_iommu_pte(paddr);
617 
618 #if DEBUG_ALLOC > 0
619 		DBGA("    (%ld) [%p,%x] np %ld\n",
620 		     last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
621 		     last_sg->length, npages);
622 		while (++last_sg <= sg) {
623 			DBGA("        (%ld) [%p,%x] cont\n",
624 			     last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
625 			     last_sg->length);
626 		}
627 #endif
628 	} while (++sg < end && (int) sg->dma_address < 0);
629 
630 	return 1;
631 }
632 
alpha_pci_map_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)633 static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg,
634 			    int nents, enum dma_data_direction dir,
635 			    unsigned long attrs)
636 {
637 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
638 	struct scatterlist *start, *end, *out;
639 	struct pci_controller *hose;
640 	struct pci_iommu_arena *arena;
641 	dma_addr_t max_dma;
642 	int dac_allowed;
643 
644 	BUG_ON(dir == DMA_NONE);
645 
646 	dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
647 
648 	/* Fast path single entry scatterlists.  */
649 	if (nents == 1) {
650 		sg->dma_length = sg->length;
651 		sg->dma_address
652 		  = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg),
653 				     sg->length, dac_allowed);
654 		if (sg->dma_address == DMA_MAPPING_ERROR)
655 			return -EIO;
656 		return 1;
657 	}
658 
659 	start = sg;
660 	end = sg + nents;
661 
662 	/* First, prepare information about the entries.  */
663 	sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0);
664 
665 	/* Second, figure out where we're going to map things.  */
666 	if (alpha_mv.mv_pci_tbi) {
667 		hose = pdev ? pdev->sysdata : pci_isa_hose;
668 		max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
669 		arena = hose->sg_pci;
670 		if (!arena || arena->dma_base + arena->size - 1 > max_dma)
671 			arena = hose->sg_isa;
672 	} else {
673 		max_dma = -1;
674 		arena = NULL;
675 		hose = NULL;
676 	}
677 
678 	/* Third, iterate over the scatterlist leaders and allocate
679 	   dma space as needed.  */
680 	for (out = sg; sg < end; ++sg) {
681 		if ((int) sg->dma_address < 0)
682 			continue;
683 		if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0)
684 			goto error;
685 		out++;
686 	}
687 
688 	/* Mark the end of the list for pci_unmap_sg.  */
689 	if (out < end)
690 		out->dma_length = 0;
691 
692 	if (out - start == 0) {
693 		printk(KERN_WARNING "pci_map_sg failed: no entries?\n");
694 		return -ENOMEM;
695 	}
696 	DBGA("pci_map_sg: %ld entries\n", out - start);
697 
698 	return out - start;
699 
700  error:
701 	printk(KERN_WARNING "pci_map_sg failed: "
702 	       "could not allocate dma page tables\n");
703 
704 	/* Some allocation failed while mapping the scatterlist
705 	   entries.  Unmap them now.  */
706 	if (out > start)
707 		dma_unmap_sg(&pdev->dev, start, out - start, dir);
708 	return -ENOMEM;
709 }
710 
711 /* Unmap a set of streaming mode DMA translations.  Again, cpu read
712    rules concerning calls here are the same as for pci_unmap_single()
713    above.  */
714 
alpha_pci_unmap_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)715 static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg,
716 			       int nents, enum dma_data_direction dir,
717 			       unsigned long attrs)
718 {
719 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
720 	unsigned long flags;
721 	struct pci_controller *hose;
722 	struct pci_iommu_arena *arena;
723 	struct scatterlist *end;
724 	dma_addr_t max_dma;
725 	dma_addr_t fbeg, fend;
726 
727 	BUG_ON(dir == DMA_NONE);
728 
729 	if (! alpha_mv.mv_pci_tbi)
730 		return;
731 
732 	hose = pdev ? pdev->sysdata : pci_isa_hose;
733 	max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
734 	arena = hose->sg_pci;
735 	if (!arena || arena->dma_base + arena->size - 1 > max_dma)
736 		arena = hose->sg_isa;
737 
738 	fbeg = -1, fend = 0;
739 
740 	spin_lock_irqsave(&arena->lock, flags);
741 
742 	for (end = sg + nents; sg < end; ++sg) {
743 		dma_addr_t addr;
744 		size_t size;
745 		long npages, ofs;
746 		dma_addr_t tend;
747 
748 		addr = sg->dma_address;
749 		size = sg->dma_length;
750 		if (!size)
751 			break;
752 
753 		if (addr > 0xffffffff) {
754 			/* It's a DAC address -- nothing to do.  */
755 			DBGA("    (%ld) DAC [%llx,%zx]\n",
756 			      sg - end + nents, addr, size);
757 			continue;
758 		}
759 
760 		if (addr >= __direct_map_base
761 		    && addr < __direct_map_base + __direct_map_size) {
762 			/* Nothing to do.  */
763 			DBGA("    (%ld) direct [%llx,%zx]\n",
764 			      sg - end + nents, addr, size);
765 			continue;
766 		}
767 
768 		DBGA("    (%ld) sg [%llx,%zx]\n",
769 		     sg - end + nents, addr, size);
770 
771 		npages = iommu_num_pages(addr, size, PAGE_SIZE);
772 		ofs = (addr - arena->dma_base) >> PAGE_SHIFT;
773 		iommu_arena_free(arena, ofs, npages);
774 
775 		tend = addr + size - 1;
776 		if (fbeg > addr) fbeg = addr;
777 		if (fend < tend) fend = tend;
778 	}
779 
780         /* If we're freeing ptes above the `next_entry' pointer (they
781            may have snuck back into the TLB since the last wrap flush),
782            we need to flush the TLB before reallocating the latter.  */
783 	if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
784 		alpha_mv.mv_pci_tbi(hose, fbeg, fend);
785 
786 	spin_unlock_irqrestore(&arena->lock, flags);
787 
788 	DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
789 }
790 
791 /* Return whether the given PCI device DMA address mask can be
792    supported properly.  */
793 
alpha_pci_supported(struct device * dev,u64 mask)794 static int alpha_pci_supported(struct device *dev, u64 mask)
795 {
796 	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
797 	struct pci_controller *hose;
798 	struct pci_iommu_arena *arena;
799 
800 	/* If there exists a direct map, and the mask fits either
801 	   the entire direct mapped space or the total system memory as
802 	   shifted by the map base */
803 	if (__direct_map_size != 0
804 	    && (__direct_map_base + __direct_map_size - 1 <= mask ||
805 		__direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask))
806 		return 1;
807 
808 	/* Check that we have a scatter-gather arena that fits.  */
809 	hose = pdev ? pdev->sysdata : pci_isa_hose;
810 	arena = hose->sg_isa;
811 	if (arena && arena->dma_base + arena->size - 1 <= mask)
812 		return 1;
813 	arena = hose->sg_pci;
814 	if (arena && arena->dma_base + arena->size - 1 <= mask)
815 		return 1;
816 
817 	/* As last resort try ZONE_DMA.  */
818 	if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask)
819 		return 1;
820 
821 	return 0;
822 }
823 
824 
825 /*
826  * AGP GART extensions to the IOMMU
827  */
828 int
iommu_reserve(struct pci_iommu_arena * arena,long pg_count,long align_mask)829 iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask)
830 {
831 	unsigned long flags;
832 	unsigned long *ptes;
833 	long i, p;
834 
835 	if (!arena) return -EINVAL;
836 
837 	spin_lock_irqsave(&arena->lock, flags);
838 
839 	/* Search for N empty ptes.  */
840 	ptes = arena->ptes;
841 	p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask);
842 	if (p < 0) {
843 		spin_unlock_irqrestore(&arena->lock, flags);
844 		return -1;
845 	}
846 
847 	/* Success.  Mark them all reserved (ie not zero and invalid)
848 	   for the iommu tlb that could load them from under us.
849 	   They will be filled in with valid bits by _bind() */
850 	for (i = 0; i < pg_count; ++i)
851 		ptes[p+i] = IOMMU_RESERVED_PTE;
852 
853 	arena->next_entry = p + pg_count;
854 	spin_unlock_irqrestore(&arena->lock, flags);
855 
856 	return p;
857 }
858 
859 int
iommu_release(struct pci_iommu_arena * arena,long pg_start,long pg_count)860 iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count)
861 {
862 	unsigned long *ptes;
863 	long i;
864 
865 	if (!arena) return -EINVAL;
866 
867 	ptes = arena->ptes;
868 
869 	/* Make sure they're all reserved first... */
870 	for(i = pg_start; i < pg_start + pg_count; i++)
871 		if (ptes[i] != IOMMU_RESERVED_PTE)
872 			return -EBUSY;
873 
874 	iommu_arena_free(arena, pg_start, pg_count);
875 	return 0;
876 }
877 
878 int
iommu_bind(struct pci_iommu_arena * arena,long pg_start,long pg_count,struct page ** pages)879 iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count,
880 	   struct page **pages)
881 {
882 	unsigned long flags;
883 	unsigned long *ptes;
884 	long i, j;
885 
886 	if (!arena) return -EINVAL;
887 
888 	spin_lock_irqsave(&arena->lock, flags);
889 
890 	ptes = arena->ptes;
891 
892 	for(j = pg_start; j < pg_start + pg_count; j++) {
893 		if (ptes[j] != IOMMU_RESERVED_PTE) {
894 			spin_unlock_irqrestore(&arena->lock, flags);
895 			return -EBUSY;
896 		}
897 	}
898 
899 	for(i = 0, j = pg_start; i < pg_count; i++, j++)
900 		ptes[j] = mk_iommu_pte(page_to_phys(pages[i]));
901 
902 	spin_unlock_irqrestore(&arena->lock, flags);
903 
904 	return 0;
905 }
906 
907 int
iommu_unbind(struct pci_iommu_arena * arena,long pg_start,long pg_count)908 iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count)
909 {
910 	unsigned long *p;
911 	long i;
912 
913 	if (!arena) return -EINVAL;
914 
915 	p = arena->ptes + pg_start;
916 	for(i = 0; i < pg_count; i++)
917 		p[i] = IOMMU_RESERVED_PTE;
918 
919 	return 0;
920 }
921 
922 const struct dma_map_ops alpha_pci_ops = {
923 	.alloc			= alpha_pci_alloc_coherent,
924 	.free			= alpha_pci_free_coherent,
925 	.map_page		= alpha_pci_map_page,
926 	.unmap_page		= alpha_pci_unmap_page,
927 	.map_sg			= alpha_pci_map_sg,
928 	.unmap_sg		= alpha_pci_unmap_sg,
929 	.dma_supported		= alpha_pci_supported,
930 	.mmap			= dma_common_mmap,
931 	.get_sgtable		= dma_common_get_sgtable,
932 	.alloc_pages		= dma_common_alloc_pages,
933 	.free_pages		= dma_common_free_pages,
934 };
935 EXPORT_SYMBOL(alpha_pci_ops);
936