1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2a88b5ba8SSam Ravnborg /* iommu.c: Generic sparc64 IOMMU support.
3a88b5ba8SSam Ravnborg *
4a88b5ba8SSam Ravnborg * Copyright (C) 1999, 2007, 2008 David S. Miller (davem@davemloft.net)
5a88b5ba8SSam Ravnborg * Copyright (C) 1999, 2000 Jakub Jelinek (jakub@redhat.com)
6a88b5ba8SSam Ravnborg */
7a88b5ba8SSam Ravnborg
8a88b5ba8SSam Ravnborg #include <linux/kernel.h>
9066bcacaSPaul Gortmaker #include <linux/export.h>
105a0e3ad6STejun Heo #include <linux/slab.h>
11a88b5ba8SSam Ravnborg #include <linux/delay.h>
12a88b5ba8SSam Ravnborg #include <linux/device.h>
130a0f0d8bSChristoph Hellwig #include <linux/dma-map-ops.h>
14a88b5ba8SSam Ravnborg #include <linux/errno.h>
15a88b5ba8SSam Ravnborg #include <linux/iommu-helper.h>
16a66022c4SAkinobu Mita #include <linux/bitmap.h>
170d3fdb15SChristoph Hellwig #include <asm/iommu-common.h>
18a88b5ba8SSam Ravnborg
19a88b5ba8SSam Ravnborg #ifdef CONFIG_PCI
20a88b5ba8SSam Ravnborg #include <linux/pci.h>
21a88b5ba8SSam Ravnborg #endif
22a88b5ba8SSam Ravnborg
23a88b5ba8SSam Ravnborg #include <asm/iommu.h>
24a88b5ba8SSam Ravnborg
25a88b5ba8SSam Ravnborg #include "iommu_common.h"
264ac7b826SSam Ravnborg #include "kernel.h"
27a88b5ba8SSam Ravnborg
28a88b5ba8SSam Ravnborg #define STC_CTXMATCH_ADDR(STC, CTX) \
29a88b5ba8SSam Ravnborg ((STC)->strbuf_ctxmatch_base + ((CTX) << 3))
30a88b5ba8SSam Ravnborg #define STC_FLUSHFLAG_INIT(STC) \
31a88b5ba8SSam Ravnborg (*((STC)->strbuf_flushflag) = 0UL)
32a88b5ba8SSam Ravnborg #define STC_FLUSHFLAG_SET(STC) \
33a88b5ba8SSam Ravnborg (*((STC)->strbuf_flushflag) != 0UL)
34a88b5ba8SSam Ravnborg
35a88b5ba8SSam Ravnborg #define iommu_read(__reg) \
36a88b5ba8SSam Ravnborg ({ u64 __ret; \
37a88b5ba8SSam Ravnborg __asm__ __volatile__("ldxa [%1] %2, %0" \
38a88b5ba8SSam Ravnborg : "=r" (__ret) \
39a88b5ba8SSam Ravnborg : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \
40a88b5ba8SSam Ravnborg : "memory"); \
41a88b5ba8SSam Ravnborg __ret; \
42a88b5ba8SSam Ravnborg })
43a88b5ba8SSam Ravnborg #define iommu_write(__reg, __val) \
44a88b5ba8SSam Ravnborg __asm__ __volatile__("stxa %0, [%1] %2" \
45a88b5ba8SSam Ravnborg : /* no outputs */ \
46a88b5ba8SSam Ravnborg : "r" (__val), "r" (__reg), \
47a88b5ba8SSam Ravnborg "i" (ASI_PHYS_BYPASS_EC_E))
48a88b5ba8SSam Ravnborg
49a88b5ba8SSam Ravnborg /* Must be invoked under the IOMMU lock. */
iommu_flushall(struct iommu_map_table * iommu_map_table)50bb620c3dSSowmini Varadhan static void iommu_flushall(struct iommu_map_table *iommu_map_table)
51a88b5ba8SSam Ravnborg {
52bb620c3dSSowmini Varadhan struct iommu *iommu = container_of(iommu_map_table, struct iommu, tbl);
53a88b5ba8SSam Ravnborg if (iommu->iommu_flushinv) {
54a88b5ba8SSam Ravnborg iommu_write(iommu->iommu_flushinv, ~(u64)0);
55a88b5ba8SSam Ravnborg } else {
56a88b5ba8SSam Ravnborg unsigned long tag;
57a88b5ba8SSam Ravnborg int entry;
58a88b5ba8SSam Ravnborg
59a88b5ba8SSam Ravnborg tag = iommu->iommu_tags;
60a88b5ba8SSam Ravnborg for (entry = 0; entry < 16; entry++) {
61a88b5ba8SSam Ravnborg iommu_write(tag, 0);
62a88b5ba8SSam Ravnborg tag += 8;
63a88b5ba8SSam Ravnborg }
64a88b5ba8SSam Ravnborg
65a88b5ba8SSam Ravnborg /* Ensure completion of previous PIO writes. */
66a88b5ba8SSam Ravnborg (void) iommu_read(iommu->write_complete_reg);
67a88b5ba8SSam Ravnborg }
68a88b5ba8SSam Ravnborg }
69a88b5ba8SSam Ravnborg
70a88b5ba8SSam Ravnborg #define IOPTE_CONSISTENT(CTX) \
71a88b5ba8SSam Ravnborg (IOPTE_VALID | IOPTE_CACHE | \
72a88b5ba8SSam Ravnborg (((CTX) << 47) & IOPTE_CONTEXT))
73a88b5ba8SSam Ravnborg
74a88b5ba8SSam Ravnborg #define IOPTE_STREAMING(CTX) \
75a88b5ba8SSam Ravnborg (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
76a88b5ba8SSam Ravnborg
77a88b5ba8SSam Ravnborg /* Existing mappings are never marked invalid, instead they
78a88b5ba8SSam Ravnborg * are pointed to a dummy page.
79a88b5ba8SSam Ravnborg */
80a88b5ba8SSam Ravnborg #define IOPTE_IS_DUMMY(iommu, iopte) \
81a88b5ba8SSam Ravnborg ((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
82a88b5ba8SSam Ravnborg
iopte_make_dummy(struct iommu * iommu,iopte_t * iopte)83a88b5ba8SSam Ravnborg static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte)
84a88b5ba8SSam Ravnborg {
85a88b5ba8SSam Ravnborg unsigned long val = iopte_val(*iopte);
86a88b5ba8SSam Ravnborg
87a88b5ba8SSam Ravnborg val &= ~IOPTE_PAGE;
88a88b5ba8SSam Ravnborg val |= iommu->dummy_page_pa;
89a88b5ba8SSam Ravnborg
90a88b5ba8SSam Ravnborg iopte_val(*iopte) = val;
91a88b5ba8SSam Ravnborg }
92a88b5ba8SSam Ravnborg
iommu_table_init(struct iommu * iommu,int tsbsize,u32 dma_offset,u32 dma_addr_mask,int numa_node)93a88b5ba8SSam Ravnborg int iommu_table_init(struct iommu *iommu, int tsbsize,
94a88b5ba8SSam Ravnborg u32 dma_offset, u32 dma_addr_mask,
95a88b5ba8SSam Ravnborg int numa_node)
96a88b5ba8SSam Ravnborg {
97a88b5ba8SSam Ravnborg unsigned long i, order, sz, num_tsb_entries;
98a88b5ba8SSam Ravnborg struct page *page;
99a88b5ba8SSam Ravnborg
100a88b5ba8SSam Ravnborg num_tsb_entries = tsbsize / sizeof(iopte_t);
101a88b5ba8SSam Ravnborg
102a88b5ba8SSam Ravnborg /* Setup initial software IOMMU state. */
103a88b5ba8SSam Ravnborg spin_lock_init(&iommu->lock);
104a88b5ba8SSam Ravnborg iommu->ctx_lowest_free = 1;
105bb620c3dSSowmini Varadhan iommu->tbl.table_map_base = dma_offset;
106a88b5ba8SSam Ravnborg iommu->dma_addr_mask = dma_addr_mask;
107a88b5ba8SSam Ravnborg
108a88b5ba8SSam Ravnborg /* Allocate and initialize the free area map. */
109a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8;
110a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL;
11186322ba9SSabyasachi Gupta iommu->tbl.map = kzalloc_node(sz, GFP_KERNEL, numa_node);
112bb620c3dSSowmini Varadhan if (!iommu->tbl.map)
113a88b5ba8SSam Ravnborg return -ENOMEM;
114f1600e54SSowmini Varadhan
115bb620c3dSSowmini Varadhan iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,
116bb620c3dSSowmini Varadhan (tlb_type != hypervisor ? iommu_flushall : NULL),
117bb620c3dSSowmini Varadhan false, 1, false);
118a88b5ba8SSam Ravnborg
119a88b5ba8SSam Ravnborg /* Allocate and initialize the dummy page which we
120a88b5ba8SSam Ravnborg * set inactive IO PTEs to point to.
121a88b5ba8SSam Ravnborg */
122a88b5ba8SSam Ravnborg page = alloc_pages_node(numa_node, GFP_KERNEL, 0);
123a88b5ba8SSam Ravnborg if (!page) {
124a88b5ba8SSam Ravnborg printk(KERN_ERR "IOMMU: Error, gfp(dummy_page) failed.\n");
125a88b5ba8SSam Ravnborg goto out_free_map;
126a88b5ba8SSam Ravnborg }
127a88b5ba8SSam Ravnborg iommu->dummy_page = (unsigned long) page_address(page);
128a88b5ba8SSam Ravnborg memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
129a88b5ba8SSam Ravnborg iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
130a88b5ba8SSam Ravnborg
131a88b5ba8SSam Ravnborg /* Now allocate and setup the IOMMU page table itself. */
132a88b5ba8SSam Ravnborg order = get_order(tsbsize);
133a88b5ba8SSam Ravnborg page = alloc_pages_node(numa_node, GFP_KERNEL, order);
134a88b5ba8SSam Ravnborg if (!page) {
135a88b5ba8SSam Ravnborg printk(KERN_ERR "IOMMU: Error, gfp(tsb) failed.\n");
136a88b5ba8SSam Ravnborg goto out_free_dummy_page;
137a88b5ba8SSam Ravnborg }
138a88b5ba8SSam Ravnborg iommu->page_table = (iopte_t *)page_address(page);
139a88b5ba8SSam Ravnborg
140a88b5ba8SSam Ravnborg for (i = 0; i < num_tsb_entries; i++)
141a88b5ba8SSam Ravnborg iopte_make_dummy(iommu, &iommu->page_table[i]);
142a88b5ba8SSam Ravnborg
143a88b5ba8SSam Ravnborg return 0;
144a88b5ba8SSam Ravnborg
145a88b5ba8SSam Ravnborg out_free_dummy_page:
146a88b5ba8SSam Ravnborg free_page(iommu->dummy_page);
147a88b5ba8SSam Ravnborg iommu->dummy_page = 0UL;
148a88b5ba8SSam Ravnborg
149a88b5ba8SSam Ravnborg out_free_map:
150bb620c3dSSowmini Varadhan kfree(iommu->tbl.map);
151bb620c3dSSowmini Varadhan iommu->tbl.map = NULL;
152a88b5ba8SSam Ravnborg
153a88b5ba8SSam Ravnborg return -ENOMEM;
154a88b5ba8SSam Ravnborg }
155a88b5ba8SSam Ravnborg
alloc_npages(struct device * dev,struct iommu * iommu,unsigned long npages)156bb620c3dSSowmini Varadhan static inline iopte_t *alloc_npages(struct device *dev,
157bb620c3dSSowmini Varadhan struct iommu *iommu,
158a88b5ba8SSam Ravnborg unsigned long npages)
159a88b5ba8SSam Ravnborg {
160a88b5ba8SSam Ravnborg unsigned long entry;
161a88b5ba8SSam Ravnborg
162bb620c3dSSowmini Varadhan entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,
163bb620c3dSSowmini Varadhan (unsigned long)(-1), 0);
164d618382bSDavid S. Miller if (unlikely(entry == IOMMU_ERROR_CODE))
165a88b5ba8SSam Ravnborg return NULL;
166a88b5ba8SSam Ravnborg
167a88b5ba8SSam Ravnborg return iommu->page_table + entry;
168a88b5ba8SSam Ravnborg }
169a88b5ba8SSam Ravnborg
iommu_alloc_ctx(struct iommu * iommu)170a88b5ba8SSam Ravnborg static int iommu_alloc_ctx(struct iommu *iommu)
171a88b5ba8SSam Ravnborg {
172a88b5ba8SSam Ravnborg int lowest = iommu->ctx_lowest_free;
173711c71a0SAkinobu Mita int n = find_next_zero_bit(iommu->ctx_bitmap, IOMMU_NUM_CTXS, lowest);
174a88b5ba8SSam Ravnborg
175711c71a0SAkinobu Mita if (unlikely(n == IOMMU_NUM_CTXS)) {
176a88b5ba8SSam Ravnborg n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1);
177a88b5ba8SSam Ravnborg if (unlikely(n == lowest)) {
178a88b5ba8SSam Ravnborg printk(KERN_WARNING "IOMMU: Ran out of contexts.\n");
179a88b5ba8SSam Ravnborg n = 0;
180a88b5ba8SSam Ravnborg }
181a88b5ba8SSam Ravnborg }
182a88b5ba8SSam Ravnborg if (n)
183a88b5ba8SSam Ravnborg __set_bit(n, iommu->ctx_bitmap);
184a88b5ba8SSam Ravnborg
185a88b5ba8SSam Ravnborg return n;
186a88b5ba8SSam Ravnborg }
187a88b5ba8SSam Ravnborg
iommu_free_ctx(struct iommu * iommu,int ctx)188a88b5ba8SSam Ravnborg static inline void iommu_free_ctx(struct iommu *iommu, int ctx)
189a88b5ba8SSam Ravnborg {
190a88b5ba8SSam Ravnborg if (likely(ctx)) {
191a88b5ba8SSam Ravnborg __clear_bit(ctx, iommu->ctx_bitmap);
192a88b5ba8SSam Ravnborg if (ctx < iommu->ctx_lowest_free)
193a88b5ba8SSam Ravnborg iommu->ctx_lowest_free = ctx;
194a88b5ba8SSam Ravnborg }
195a88b5ba8SSam Ravnborg }
196a88b5ba8SSam Ravnborg
dma_4u_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_addrp,gfp_t gfp,unsigned long attrs)197a88b5ba8SSam Ravnborg static void *dma_4u_alloc_coherent(struct device *dev, size_t size,
198c416258aSAndrzej Pietrasiewicz dma_addr_t *dma_addrp, gfp_t gfp,
19900085f1eSKrzysztof Kozlowski unsigned long attrs)
200a88b5ba8SSam Ravnborg {
201bb620c3dSSowmini Varadhan unsigned long order, first_page;
202a88b5ba8SSam Ravnborg struct iommu *iommu;
203a88b5ba8SSam Ravnborg struct page *page;
204a88b5ba8SSam Ravnborg int npages, nid;
205a88b5ba8SSam Ravnborg iopte_t *iopte;
206a88b5ba8SSam Ravnborg void *ret;
207a88b5ba8SSam Ravnborg
208a88b5ba8SSam Ravnborg size = IO_PAGE_ALIGN(size);
209a88b5ba8SSam Ravnborg order = get_order(size);
210a88b5ba8SSam Ravnborg if (order >= 10)
211a88b5ba8SSam Ravnborg return NULL;
212a88b5ba8SSam Ravnborg
213a88b5ba8SSam Ravnborg nid = dev->archdata.numa_node;
214a88b5ba8SSam Ravnborg page = alloc_pages_node(nid, gfp, order);
215a88b5ba8SSam Ravnborg if (unlikely(!page))
216a88b5ba8SSam Ravnborg return NULL;
217a88b5ba8SSam Ravnborg
218a88b5ba8SSam Ravnborg first_page = (unsigned long) page_address(page);
219a88b5ba8SSam Ravnborg memset((char *)first_page, 0, PAGE_SIZE << order);
220a88b5ba8SSam Ravnborg
221a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
222a88b5ba8SSam Ravnborg
223a88b5ba8SSam Ravnborg iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT);
224a88b5ba8SSam Ravnborg
225a88b5ba8SSam Ravnborg if (unlikely(iopte == NULL)) {
226a88b5ba8SSam Ravnborg free_pages(first_page, order);
227a88b5ba8SSam Ravnborg return NULL;
228a88b5ba8SSam Ravnborg }
229a88b5ba8SSam Ravnborg
230bb620c3dSSowmini Varadhan *dma_addrp = (iommu->tbl.table_map_base +
231a88b5ba8SSam Ravnborg ((iopte - iommu->page_table) << IO_PAGE_SHIFT));
232a88b5ba8SSam Ravnborg ret = (void *) first_page;
233a88b5ba8SSam Ravnborg npages = size >> IO_PAGE_SHIFT;
234a88b5ba8SSam Ravnborg first_page = __pa(first_page);
235a88b5ba8SSam Ravnborg while (npages--) {
236a88b5ba8SSam Ravnborg iopte_val(*iopte) = (IOPTE_CONSISTENT(0UL) |
237a88b5ba8SSam Ravnborg IOPTE_WRITE |
238a88b5ba8SSam Ravnborg (first_page & IOPTE_PAGE));
239a88b5ba8SSam Ravnborg iopte++;
240a88b5ba8SSam Ravnborg first_page += IO_PAGE_SIZE;
241a88b5ba8SSam Ravnborg }
242a88b5ba8SSam Ravnborg
243a88b5ba8SSam Ravnborg return ret;
244a88b5ba8SSam Ravnborg }
245a88b5ba8SSam Ravnborg
dma_4u_free_coherent(struct device * dev,size_t size,void * cpu,dma_addr_t dvma,unsigned long attrs)246a88b5ba8SSam Ravnborg static void dma_4u_free_coherent(struct device *dev, size_t size,
247c416258aSAndrzej Pietrasiewicz void *cpu, dma_addr_t dvma,
24800085f1eSKrzysztof Kozlowski unsigned long attrs)
249a88b5ba8SSam Ravnborg {
250a88b5ba8SSam Ravnborg struct iommu *iommu;
251bb620c3dSSowmini Varadhan unsigned long order, npages;
252a88b5ba8SSam Ravnborg
253a88b5ba8SSam Ravnborg npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
254a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
255a88b5ba8SSam Ravnborg
256d618382bSDavid S. Miller iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE);
257a88b5ba8SSam Ravnborg
258a88b5ba8SSam Ravnborg order = get_order(size);
259a88b5ba8SSam Ravnborg if (order < 10)
260a88b5ba8SSam Ravnborg free_pages((unsigned long)cpu, order);
261a88b5ba8SSam Ravnborg }
262a88b5ba8SSam Ravnborg
dma_4u_map_page(struct device * dev,struct page * page,unsigned long offset,size_t sz,enum dma_data_direction direction,unsigned long attrs)263797a7568SFUJITA Tomonori static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page,
264797a7568SFUJITA Tomonori unsigned long offset, size_t sz,
265bc0a14f1SFUJITA Tomonori enum dma_data_direction direction,
26600085f1eSKrzysztof Kozlowski unsigned long attrs)
267a88b5ba8SSam Ravnborg {
268a88b5ba8SSam Ravnborg struct iommu *iommu;
269a88b5ba8SSam Ravnborg struct strbuf *strbuf;
270a88b5ba8SSam Ravnborg iopte_t *base;
271a88b5ba8SSam Ravnborg unsigned long flags, npages, oaddr;
272a88b5ba8SSam Ravnborg unsigned long i, base_paddr, ctx;
273a88b5ba8SSam Ravnborg u32 bus_addr, ret;
274a88b5ba8SSam Ravnborg unsigned long iopte_protection;
275a88b5ba8SSam Ravnborg
276a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
277a88b5ba8SSam Ravnborg strbuf = dev->archdata.stc;
278a88b5ba8SSam Ravnborg
279a88b5ba8SSam Ravnborg if (unlikely(direction == DMA_NONE))
280a88b5ba8SSam Ravnborg goto bad_no_ctx;
281a88b5ba8SSam Ravnborg
282797a7568SFUJITA Tomonori oaddr = (unsigned long)(page_address(page) + offset);
283a88b5ba8SSam Ravnborg npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
284a88b5ba8SSam Ravnborg npages >>= IO_PAGE_SHIFT;
285a88b5ba8SSam Ravnborg
286c12f048fSDavid S. Miller base = alloc_npages(dev, iommu, npages);
287bb620c3dSSowmini Varadhan spin_lock_irqsave(&iommu->lock, flags);
288a88b5ba8SSam Ravnborg ctx = 0;
289a88b5ba8SSam Ravnborg if (iommu->iommu_ctxflush)
290a88b5ba8SSam Ravnborg ctx = iommu_alloc_ctx(iommu);
291a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
292a88b5ba8SSam Ravnborg
293a88b5ba8SSam Ravnborg if (unlikely(!base))
294a88b5ba8SSam Ravnborg goto bad;
295a88b5ba8SSam Ravnborg
296bb620c3dSSowmini Varadhan bus_addr = (iommu->tbl.table_map_base +
297a88b5ba8SSam Ravnborg ((base - iommu->page_table) << IO_PAGE_SHIFT));
298a88b5ba8SSam Ravnborg ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
299a88b5ba8SSam Ravnborg base_paddr = __pa(oaddr & IO_PAGE_MASK);
300a88b5ba8SSam Ravnborg if (strbuf->strbuf_enabled)
301a88b5ba8SSam Ravnborg iopte_protection = IOPTE_STREAMING(ctx);
302a88b5ba8SSam Ravnborg else
303a88b5ba8SSam Ravnborg iopte_protection = IOPTE_CONSISTENT(ctx);
304a88b5ba8SSam Ravnborg if (direction != DMA_TO_DEVICE)
305a88b5ba8SSam Ravnborg iopte_protection |= IOPTE_WRITE;
306a88b5ba8SSam Ravnborg
307a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE)
308a88b5ba8SSam Ravnborg iopte_val(*base) = iopte_protection | base_paddr;
309a88b5ba8SSam Ravnborg
310a88b5ba8SSam Ravnborg return ret;
311a88b5ba8SSam Ravnborg
312a88b5ba8SSam Ravnborg bad:
313a88b5ba8SSam Ravnborg iommu_free_ctx(iommu, ctx);
314a88b5ba8SSam Ravnborg bad_no_ctx:
315a88b5ba8SSam Ravnborg if (printk_ratelimit())
316a88b5ba8SSam Ravnborg WARN_ON(1);
31706301c5eSChristoph Hellwig return DMA_MAPPING_ERROR;
318a88b5ba8SSam Ravnborg }
319a88b5ba8SSam Ravnborg
strbuf_flush(struct strbuf * strbuf,struct iommu * iommu,u32 vaddr,unsigned long ctx,unsigned long npages,enum dma_data_direction direction)320a88b5ba8SSam Ravnborg static void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu,
321a88b5ba8SSam Ravnborg u32 vaddr, unsigned long ctx, unsigned long npages,
322a88b5ba8SSam Ravnborg enum dma_data_direction direction)
323a88b5ba8SSam Ravnborg {
324a88b5ba8SSam Ravnborg int limit;
325a88b5ba8SSam Ravnborg
326a88b5ba8SSam Ravnborg if (strbuf->strbuf_ctxflush &&
327a88b5ba8SSam Ravnborg iommu->iommu_ctxflush) {
328a88b5ba8SSam Ravnborg unsigned long matchreg, flushreg;
329a88b5ba8SSam Ravnborg u64 val;
330a88b5ba8SSam Ravnborg
331a88b5ba8SSam Ravnborg flushreg = strbuf->strbuf_ctxflush;
332a88b5ba8SSam Ravnborg matchreg = STC_CTXMATCH_ADDR(strbuf, ctx);
333a88b5ba8SSam Ravnborg
334a88b5ba8SSam Ravnborg iommu_write(flushreg, ctx);
335a88b5ba8SSam Ravnborg val = iommu_read(matchreg);
336a88b5ba8SSam Ravnborg val &= 0xffff;
337a88b5ba8SSam Ravnborg if (!val)
338a88b5ba8SSam Ravnborg goto do_flush_sync;
339a88b5ba8SSam Ravnborg
340a88b5ba8SSam Ravnborg while (val) {
341a88b5ba8SSam Ravnborg if (val & 0x1)
342a88b5ba8SSam Ravnborg iommu_write(flushreg, ctx);
343a88b5ba8SSam Ravnborg val >>= 1;
344a88b5ba8SSam Ravnborg }
345a88b5ba8SSam Ravnborg val = iommu_read(matchreg);
346a88b5ba8SSam Ravnborg if (unlikely(val)) {
347a88b5ba8SSam Ravnborg printk(KERN_WARNING "strbuf_flush: ctx flush "
34890181136SSam Ravnborg "timeout matchreg[%llx] ctx[%lx]\n",
349a88b5ba8SSam Ravnborg val, ctx);
350a88b5ba8SSam Ravnborg goto do_page_flush;
351a88b5ba8SSam Ravnborg }
352a88b5ba8SSam Ravnborg } else {
353a88b5ba8SSam Ravnborg unsigned long i;
354a88b5ba8SSam Ravnborg
355a88b5ba8SSam Ravnborg do_page_flush:
356a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE)
357a88b5ba8SSam Ravnborg iommu_write(strbuf->strbuf_pflush, vaddr);
358a88b5ba8SSam Ravnborg }
359a88b5ba8SSam Ravnborg
360a88b5ba8SSam Ravnborg do_flush_sync:
361a88b5ba8SSam Ravnborg /* If the device could not have possibly put dirty data into
362a88b5ba8SSam Ravnborg * the streaming cache, no flush-flag synchronization needs
363a88b5ba8SSam Ravnborg * to be performed.
364a88b5ba8SSam Ravnborg */
365a88b5ba8SSam Ravnborg if (direction == DMA_TO_DEVICE)
366a88b5ba8SSam Ravnborg return;
367a88b5ba8SSam Ravnborg
368a88b5ba8SSam Ravnborg STC_FLUSHFLAG_INIT(strbuf);
369a88b5ba8SSam Ravnborg iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
370a88b5ba8SSam Ravnborg (void) iommu_read(iommu->write_complete_reg);
371a88b5ba8SSam Ravnborg
372a88b5ba8SSam Ravnborg limit = 100000;
373a88b5ba8SSam Ravnborg while (!STC_FLUSHFLAG_SET(strbuf)) {
374a88b5ba8SSam Ravnborg limit--;
375a88b5ba8SSam Ravnborg if (!limit)
376a88b5ba8SSam Ravnborg break;
377a88b5ba8SSam Ravnborg udelay(1);
378a88b5ba8SSam Ravnborg rmb();
379a88b5ba8SSam Ravnborg }
380a88b5ba8SSam Ravnborg if (!limit)
381a88b5ba8SSam Ravnborg printk(KERN_WARNING "strbuf_flush: flushflag timeout "
382a88b5ba8SSam Ravnborg "vaddr[%08x] ctx[%lx] npages[%ld]\n",
383a88b5ba8SSam Ravnborg vaddr, ctx, npages);
384a88b5ba8SSam Ravnborg }
385a88b5ba8SSam Ravnborg
dma_4u_unmap_page(struct device * dev,dma_addr_t bus_addr,size_t sz,enum dma_data_direction direction,unsigned long attrs)386797a7568SFUJITA Tomonori static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr,
387bc0a14f1SFUJITA Tomonori size_t sz, enum dma_data_direction direction,
38800085f1eSKrzysztof Kozlowski unsigned long attrs)
389a88b5ba8SSam Ravnborg {
390a88b5ba8SSam Ravnborg struct iommu *iommu;
391a88b5ba8SSam Ravnborg struct strbuf *strbuf;
392a88b5ba8SSam Ravnborg iopte_t *base;
393a88b5ba8SSam Ravnborg unsigned long flags, npages, ctx, i;
394a88b5ba8SSam Ravnborg
395a88b5ba8SSam Ravnborg if (unlikely(direction == DMA_NONE)) {
396a88b5ba8SSam Ravnborg if (printk_ratelimit())
397a88b5ba8SSam Ravnborg WARN_ON(1);
398a88b5ba8SSam Ravnborg return;
399a88b5ba8SSam Ravnborg }
400a88b5ba8SSam Ravnborg
401a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
402a88b5ba8SSam Ravnborg strbuf = dev->archdata.stc;
403a88b5ba8SSam Ravnborg
404a88b5ba8SSam Ravnborg npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
405a88b5ba8SSam Ravnborg npages >>= IO_PAGE_SHIFT;
406a88b5ba8SSam Ravnborg base = iommu->page_table +
407bb620c3dSSowmini Varadhan ((bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT);
408a88b5ba8SSam Ravnborg bus_addr &= IO_PAGE_MASK;
409a88b5ba8SSam Ravnborg
410a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags);
411a88b5ba8SSam Ravnborg
412a88b5ba8SSam Ravnborg /* Record the context, if any. */
413a88b5ba8SSam Ravnborg ctx = 0;
414a88b5ba8SSam Ravnborg if (iommu->iommu_ctxflush)
415a88b5ba8SSam Ravnborg ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
416a88b5ba8SSam Ravnborg
417a88b5ba8SSam Ravnborg /* Step 1: Kick data out of streaming buffers if necessary. */
41868bbc28fSAlexander Duyck if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
419a88b5ba8SSam Ravnborg strbuf_flush(strbuf, iommu, bus_addr, ctx,
420a88b5ba8SSam Ravnborg npages, direction);
421a88b5ba8SSam Ravnborg
422a88b5ba8SSam Ravnborg /* Step 2: Clear out TSB entries. */
423a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++)
424a88b5ba8SSam Ravnborg iopte_make_dummy(iommu, base + i);
425a88b5ba8SSam Ravnborg
426c12f048fSDavid S. Miller iommu_free_ctx(iommu, ctx);
427c12f048fSDavid S. Miller spin_unlock_irqrestore(&iommu->lock, flags);
428bb620c3dSSowmini Varadhan
429d618382bSDavid S. Miller iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE);
430a88b5ba8SSam Ravnborg }
431a88b5ba8SSam Ravnborg
dma_4u_map_sg(struct device * dev,struct scatterlist * sglist,int nelems,enum dma_data_direction direction,unsigned long attrs)432a88b5ba8SSam Ravnborg static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
433bc0a14f1SFUJITA Tomonori int nelems, enum dma_data_direction direction,
43400085f1eSKrzysztof Kozlowski unsigned long attrs)
435a88b5ba8SSam Ravnborg {
436a88b5ba8SSam Ravnborg struct scatterlist *s, *outs, *segstart;
437a88b5ba8SSam Ravnborg unsigned long flags, handle, prot, ctx;
438a88b5ba8SSam Ravnborg dma_addr_t dma_next = 0, dma_addr;
439a88b5ba8SSam Ravnborg unsigned int max_seg_size;
440a88b5ba8SSam Ravnborg unsigned long seg_boundary_size;
441a88b5ba8SSam Ravnborg int outcount, incount, i;
442a88b5ba8SSam Ravnborg struct strbuf *strbuf;
443a88b5ba8SSam Ravnborg struct iommu *iommu;
444a88b5ba8SSam Ravnborg unsigned long base_shift;
445a88b5ba8SSam Ravnborg
446a88b5ba8SSam Ravnborg BUG_ON(direction == DMA_NONE);
447a88b5ba8SSam Ravnborg
448a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
449a88b5ba8SSam Ravnborg strbuf = dev->archdata.stc;
450a88b5ba8SSam Ravnborg if (nelems == 0 || !iommu)
451*e02373fdSMartin Oliveira return -EINVAL;
452a88b5ba8SSam Ravnborg
453a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags);
454a88b5ba8SSam Ravnborg
455a88b5ba8SSam Ravnborg ctx = 0;
456a88b5ba8SSam Ravnborg if (iommu->iommu_ctxflush)
457a88b5ba8SSam Ravnborg ctx = iommu_alloc_ctx(iommu);
458a88b5ba8SSam Ravnborg
459a88b5ba8SSam Ravnborg if (strbuf->strbuf_enabled)
460a88b5ba8SSam Ravnborg prot = IOPTE_STREAMING(ctx);
461a88b5ba8SSam Ravnborg else
462a88b5ba8SSam Ravnborg prot = IOPTE_CONSISTENT(ctx);
463a88b5ba8SSam Ravnborg if (direction != DMA_TO_DEVICE)
464a88b5ba8SSam Ravnborg prot |= IOPTE_WRITE;
465a88b5ba8SSam Ravnborg
466a88b5ba8SSam Ravnborg outs = s = segstart = &sglist[0];
467a88b5ba8SSam Ravnborg outcount = 1;
468a88b5ba8SSam Ravnborg incount = nelems;
469a88b5ba8SSam Ravnborg handle = 0;
470a88b5ba8SSam Ravnborg
471a88b5ba8SSam Ravnborg /* Init first segment length for backout at failure */
472a88b5ba8SSam Ravnborg outs->dma_length = 0;
473a88b5ba8SSam Ravnborg
474a88b5ba8SSam Ravnborg max_seg_size = dma_get_max_seg_size(dev);
4751e9d90dbSNicolin Chen seg_boundary_size = dma_get_seg_boundary_nr_pages(dev, IO_PAGE_SHIFT);
476bb620c3dSSowmini Varadhan base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT;
477a88b5ba8SSam Ravnborg for_each_sg(sglist, s, nelems, i) {
478a88b5ba8SSam Ravnborg unsigned long paddr, npages, entry, out_entry = 0, slen;
479a88b5ba8SSam Ravnborg iopte_t *base;
480a88b5ba8SSam Ravnborg
481a88b5ba8SSam Ravnborg slen = s->length;
482a88b5ba8SSam Ravnborg /* Sanity check */
483a88b5ba8SSam Ravnborg if (slen == 0) {
484a88b5ba8SSam Ravnborg dma_next = 0;
485a88b5ba8SSam Ravnborg continue;
486a88b5ba8SSam Ravnborg }
487a88b5ba8SSam Ravnborg /* Allocate iommu entries for that segment */
488a88b5ba8SSam Ravnborg paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
489a88b5ba8SSam Ravnborg npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);
490bb620c3dSSowmini Varadhan entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages,
491bb620c3dSSowmini Varadhan &handle, (unsigned long)(-1), 0);
492a88b5ba8SSam Ravnborg
493a88b5ba8SSam Ravnborg /* Handle failure */
494d618382bSDavid S. Miller if (unlikely(entry == IOMMU_ERROR_CODE)) {
495a88b5ba8SSam Ravnborg if (printk_ratelimit())
496a88b5ba8SSam Ravnborg printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx"
497a88b5ba8SSam Ravnborg " npages %lx\n", iommu, paddr, npages);
498a88b5ba8SSam Ravnborg goto iommu_map_failed;
499a88b5ba8SSam Ravnborg }
500a88b5ba8SSam Ravnborg
501a88b5ba8SSam Ravnborg base = iommu->page_table + entry;
502a88b5ba8SSam Ravnborg
503a88b5ba8SSam Ravnborg /* Convert entry to a dma_addr_t */
504bb620c3dSSowmini Varadhan dma_addr = iommu->tbl.table_map_base +
505a88b5ba8SSam Ravnborg (entry << IO_PAGE_SHIFT);
506a88b5ba8SSam Ravnborg dma_addr |= (s->offset & ~IO_PAGE_MASK);
507a88b5ba8SSam Ravnborg
508a88b5ba8SSam Ravnborg /* Insert into HW table */
509a88b5ba8SSam Ravnborg paddr &= IO_PAGE_MASK;
510a88b5ba8SSam Ravnborg while (npages--) {
511a88b5ba8SSam Ravnborg iopte_val(*base) = prot | paddr;
512a88b5ba8SSam Ravnborg base++;
513a88b5ba8SSam Ravnborg paddr += IO_PAGE_SIZE;
514a88b5ba8SSam Ravnborg }
515a88b5ba8SSam Ravnborg
516a88b5ba8SSam Ravnborg /* If we are in an open segment, try merging */
517a88b5ba8SSam Ravnborg if (segstart != s) {
518a88b5ba8SSam Ravnborg /* We cannot merge if:
519a88b5ba8SSam Ravnborg * - allocated dma_addr isn't contiguous to previous allocation
520a88b5ba8SSam Ravnborg */
521a88b5ba8SSam Ravnborg if ((dma_addr != dma_next) ||
522a88b5ba8SSam Ravnborg (outs->dma_length + s->length > max_seg_size) ||
523a88b5ba8SSam Ravnborg (is_span_boundary(out_entry, base_shift,
524a88b5ba8SSam Ravnborg seg_boundary_size, outs, s))) {
525a88b5ba8SSam Ravnborg /* Can't merge: create a new segment */
526a88b5ba8SSam Ravnborg segstart = s;
527a88b5ba8SSam Ravnborg outcount++;
528a88b5ba8SSam Ravnborg outs = sg_next(outs);
529a88b5ba8SSam Ravnborg } else {
530a88b5ba8SSam Ravnborg outs->dma_length += s->length;
531a88b5ba8SSam Ravnborg }
532a88b5ba8SSam Ravnborg }
533a88b5ba8SSam Ravnborg
534a88b5ba8SSam Ravnborg if (segstart == s) {
535a88b5ba8SSam Ravnborg /* This is a new segment, fill entries */
536a88b5ba8SSam Ravnborg outs->dma_address = dma_addr;
537a88b5ba8SSam Ravnborg outs->dma_length = slen;
538a88b5ba8SSam Ravnborg out_entry = entry;
539a88b5ba8SSam Ravnborg }
540a88b5ba8SSam Ravnborg
541a88b5ba8SSam Ravnborg /* Calculate next page pointer for contiguous check */
542a88b5ba8SSam Ravnborg dma_next = dma_addr + slen;
543a88b5ba8SSam Ravnborg }
544a88b5ba8SSam Ravnborg
545a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
546a88b5ba8SSam Ravnborg
547a88b5ba8SSam Ravnborg if (outcount < incount) {
548a88b5ba8SSam Ravnborg outs = sg_next(outs);
549a88b5ba8SSam Ravnborg outs->dma_length = 0;
550a88b5ba8SSam Ravnborg }
551a88b5ba8SSam Ravnborg
552a88b5ba8SSam Ravnborg return outcount;
553a88b5ba8SSam Ravnborg
554a88b5ba8SSam Ravnborg iommu_map_failed:
555a88b5ba8SSam Ravnborg for_each_sg(sglist, s, nelems, i) {
556a88b5ba8SSam Ravnborg if (s->dma_length != 0) {
557a88b5ba8SSam Ravnborg unsigned long vaddr, npages, entry, j;
558a88b5ba8SSam Ravnborg iopte_t *base;
559a88b5ba8SSam Ravnborg
560a88b5ba8SSam Ravnborg vaddr = s->dma_address & IO_PAGE_MASK;
561a88b5ba8SSam Ravnborg npages = iommu_num_pages(s->dma_address, s->dma_length,
562a88b5ba8SSam Ravnborg IO_PAGE_SIZE);
563a88b5ba8SSam Ravnborg
564bb620c3dSSowmini Varadhan entry = (vaddr - iommu->tbl.table_map_base)
565a88b5ba8SSam Ravnborg >> IO_PAGE_SHIFT;
566a88b5ba8SSam Ravnborg base = iommu->page_table + entry;
567a88b5ba8SSam Ravnborg
568a88b5ba8SSam Ravnborg for (j = 0; j < npages; j++)
569a88b5ba8SSam Ravnborg iopte_make_dummy(iommu, base + j);
570a88b5ba8SSam Ravnborg
571bb620c3dSSowmini Varadhan iommu_tbl_range_free(&iommu->tbl, vaddr, npages,
572d618382bSDavid S. Miller IOMMU_ERROR_CODE);
573bb620c3dSSowmini Varadhan
574a88b5ba8SSam Ravnborg s->dma_length = 0;
575a88b5ba8SSam Ravnborg }
576a88b5ba8SSam Ravnborg if (s == outs)
577a88b5ba8SSam Ravnborg break;
578a88b5ba8SSam Ravnborg }
579a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
580a88b5ba8SSam Ravnborg
581*e02373fdSMartin Oliveira return -EINVAL;
582a88b5ba8SSam Ravnborg }
583a88b5ba8SSam Ravnborg
584a88b5ba8SSam Ravnborg /* If contexts are being used, they are the same in all of the mappings
585a88b5ba8SSam Ravnborg * we make for a particular SG.
586a88b5ba8SSam Ravnborg */
fetch_sg_ctx(struct iommu * iommu,struct scatterlist * sg)587c12f048fSDavid S. Miller static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg)
588a88b5ba8SSam Ravnborg {
589a88b5ba8SSam Ravnborg unsigned long ctx = 0;
590a88b5ba8SSam Ravnborg
591a88b5ba8SSam Ravnborg if (iommu->iommu_ctxflush) {
592a88b5ba8SSam Ravnborg iopte_t *base;
593a88b5ba8SSam Ravnborg u32 bus_addr;
594bb620c3dSSowmini Varadhan struct iommu_map_table *tbl = &iommu->tbl;
595a88b5ba8SSam Ravnborg
596a88b5ba8SSam Ravnborg bus_addr = sg->dma_address & IO_PAGE_MASK;
597a88b5ba8SSam Ravnborg base = iommu->page_table +
598bb620c3dSSowmini Varadhan ((bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT);
599a88b5ba8SSam Ravnborg
600a88b5ba8SSam Ravnborg ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
601a88b5ba8SSam Ravnborg }
602a88b5ba8SSam Ravnborg return ctx;
603a88b5ba8SSam Ravnborg }
604a88b5ba8SSam Ravnborg
dma_4u_unmap_sg(struct device * dev,struct scatterlist * sglist,int nelems,enum dma_data_direction direction,unsigned long attrs)605a88b5ba8SSam Ravnborg static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist,
606bc0a14f1SFUJITA Tomonori int nelems, enum dma_data_direction direction,
60700085f1eSKrzysztof Kozlowski unsigned long attrs)
608a88b5ba8SSam Ravnborg {
609a88b5ba8SSam Ravnborg unsigned long flags, ctx;
610a88b5ba8SSam Ravnborg struct scatterlist *sg;
611a88b5ba8SSam Ravnborg struct strbuf *strbuf;
612a88b5ba8SSam Ravnborg struct iommu *iommu;
613a88b5ba8SSam Ravnborg
614a88b5ba8SSam Ravnborg BUG_ON(direction == DMA_NONE);
615a88b5ba8SSam Ravnborg
616a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
617a88b5ba8SSam Ravnborg strbuf = dev->archdata.stc;
618a88b5ba8SSam Ravnborg
619a88b5ba8SSam Ravnborg ctx = fetch_sg_ctx(iommu, sglist);
620a88b5ba8SSam Ravnborg
621a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags);
622a88b5ba8SSam Ravnborg
623a88b5ba8SSam Ravnborg sg = sglist;
624a88b5ba8SSam Ravnborg while (nelems--) {
625a88b5ba8SSam Ravnborg dma_addr_t dma_handle = sg->dma_address;
626a88b5ba8SSam Ravnborg unsigned int len = sg->dma_length;
627a88b5ba8SSam Ravnborg unsigned long npages, entry;
628a88b5ba8SSam Ravnborg iopte_t *base;
629a88b5ba8SSam Ravnborg int i;
630a88b5ba8SSam Ravnborg
631a88b5ba8SSam Ravnborg if (!len)
632a88b5ba8SSam Ravnborg break;
633a88b5ba8SSam Ravnborg npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);
634a88b5ba8SSam Ravnborg
635bb620c3dSSowmini Varadhan entry = ((dma_handle - iommu->tbl.table_map_base)
636a88b5ba8SSam Ravnborg >> IO_PAGE_SHIFT);
637a88b5ba8SSam Ravnborg base = iommu->page_table + entry;
638a88b5ba8SSam Ravnborg
639a88b5ba8SSam Ravnborg dma_handle &= IO_PAGE_MASK;
64068bbc28fSAlexander Duyck if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
641a88b5ba8SSam Ravnborg strbuf_flush(strbuf, iommu, dma_handle, ctx,
642a88b5ba8SSam Ravnborg npages, direction);
643a88b5ba8SSam Ravnborg
644a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++)
645a88b5ba8SSam Ravnborg iopte_make_dummy(iommu, base + i);
646a88b5ba8SSam Ravnborg
647bb620c3dSSowmini Varadhan iommu_tbl_range_free(&iommu->tbl, dma_handle, npages,
648d618382bSDavid S. Miller IOMMU_ERROR_CODE);
649a88b5ba8SSam Ravnborg sg = sg_next(sg);
650a88b5ba8SSam Ravnborg }
651a88b5ba8SSam Ravnborg
652a88b5ba8SSam Ravnborg iommu_free_ctx(iommu, ctx);
653a88b5ba8SSam Ravnborg
654a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
655a88b5ba8SSam Ravnborg }
656a88b5ba8SSam Ravnborg
dma_4u_sync_single_for_cpu(struct device * dev,dma_addr_t bus_addr,size_t sz,enum dma_data_direction direction)657a88b5ba8SSam Ravnborg static void dma_4u_sync_single_for_cpu(struct device *dev,
658a88b5ba8SSam Ravnborg dma_addr_t bus_addr, size_t sz,
659a88b5ba8SSam Ravnborg enum dma_data_direction direction)
660a88b5ba8SSam Ravnborg {
661a88b5ba8SSam Ravnborg struct iommu *iommu;
662a88b5ba8SSam Ravnborg struct strbuf *strbuf;
663a88b5ba8SSam Ravnborg unsigned long flags, ctx, npages;
664a88b5ba8SSam Ravnborg
665a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
666a88b5ba8SSam Ravnborg strbuf = dev->archdata.stc;
667a88b5ba8SSam Ravnborg
668a88b5ba8SSam Ravnborg if (!strbuf->strbuf_enabled)
669a88b5ba8SSam Ravnborg return;
670a88b5ba8SSam Ravnborg
671a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags);
672a88b5ba8SSam Ravnborg
673a88b5ba8SSam Ravnborg npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
674a88b5ba8SSam Ravnborg npages >>= IO_PAGE_SHIFT;
675a88b5ba8SSam Ravnborg bus_addr &= IO_PAGE_MASK;
676a88b5ba8SSam Ravnborg
677a88b5ba8SSam Ravnborg /* Step 1: Record the context, if any. */
678a88b5ba8SSam Ravnborg ctx = 0;
679a88b5ba8SSam Ravnborg if (iommu->iommu_ctxflush &&
680a88b5ba8SSam Ravnborg strbuf->strbuf_ctxflush) {
681a88b5ba8SSam Ravnborg iopte_t *iopte;
682bb620c3dSSowmini Varadhan struct iommu_map_table *tbl = &iommu->tbl;
683a88b5ba8SSam Ravnborg
684a88b5ba8SSam Ravnborg iopte = iommu->page_table +
685bb620c3dSSowmini Varadhan ((bus_addr - tbl->table_map_base)>>IO_PAGE_SHIFT);
686a88b5ba8SSam Ravnborg ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
687a88b5ba8SSam Ravnborg }
688a88b5ba8SSam Ravnborg
689a88b5ba8SSam Ravnborg /* Step 2: Kick data out of streaming buffers. */
690a88b5ba8SSam Ravnborg strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
691a88b5ba8SSam Ravnborg
692a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
693a88b5ba8SSam Ravnborg }
694a88b5ba8SSam Ravnborg
dma_4u_sync_sg_for_cpu(struct device * dev,struct scatterlist * sglist,int nelems,enum dma_data_direction direction)695a88b5ba8SSam Ravnborg static void dma_4u_sync_sg_for_cpu(struct device *dev,
696a88b5ba8SSam Ravnborg struct scatterlist *sglist, int nelems,
697a88b5ba8SSam Ravnborg enum dma_data_direction direction)
698a88b5ba8SSam Ravnborg {
699a88b5ba8SSam Ravnborg struct iommu *iommu;
700a88b5ba8SSam Ravnborg struct strbuf *strbuf;
701a88b5ba8SSam Ravnborg unsigned long flags, ctx, npages, i;
702a88b5ba8SSam Ravnborg struct scatterlist *sg, *sgprv;
703a88b5ba8SSam Ravnborg u32 bus_addr;
704a88b5ba8SSam Ravnborg
705a88b5ba8SSam Ravnborg iommu = dev->archdata.iommu;
706a88b5ba8SSam Ravnborg strbuf = dev->archdata.stc;
707a88b5ba8SSam Ravnborg
708a88b5ba8SSam Ravnborg if (!strbuf->strbuf_enabled)
709a88b5ba8SSam Ravnborg return;
710a88b5ba8SSam Ravnborg
711a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags);
712a88b5ba8SSam Ravnborg
713a88b5ba8SSam Ravnborg /* Step 1: Record the context, if any. */
714a88b5ba8SSam Ravnborg ctx = 0;
715a88b5ba8SSam Ravnborg if (iommu->iommu_ctxflush &&
716a88b5ba8SSam Ravnborg strbuf->strbuf_ctxflush) {
717a88b5ba8SSam Ravnborg iopte_t *iopte;
718bb620c3dSSowmini Varadhan struct iommu_map_table *tbl = &iommu->tbl;
719a88b5ba8SSam Ravnborg
720bb620c3dSSowmini Varadhan iopte = iommu->page_table + ((sglist[0].dma_address -
721bb620c3dSSowmini Varadhan tbl->table_map_base) >> IO_PAGE_SHIFT);
722a88b5ba8SSam Ravnborg ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
723a88b5ba8SSam Ravnborg }
724a88b5ba8SSam Ravnborg
725a88b5ba8SSam Ravnborg /* Step 2: Kick data out of streaming buffers. */
726a88b5ba8SSam Ravnborg bus_addr = sglist[0].dma_address & IO_PAGE_MASK;
727a88b5ba8SSam Ravnborg sgprv = NULL;
728a88b5ba8SSam Ravnborg for_each_sg(sglist, sg, nelems, i) {
729a88b5ba8SSam Ravnborg if (sg->dma_length == 0)
730a88b5ba8SSam Ravnborg break;
731a88b5ba8SSam Ravnborg sgprv = sg;
732a88b5ba8SSam Ravnborg }
733a88b5ba8SSam Ravnborg
734a88b5ba8SSam Ravnborg npages = (IO_PAGE_ALIGN(sgprv->dma_address + sgprv->dma_length)
735a88b5ba8SSam Ravnborg - bus_addr) >> IO_PAGE_SHIFT;
736a88b5ba8SSam Ravnborg strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction);
737a88b5ba8SSam Ravnborg
738a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags);
739a88b5ba8SSam Ravnborg }
740a88b5ba8SSam Ravnborg
dma_4u_supported(struct device * dev,u64 device_mask)741b02c2b0bSChristoph Hellwig static int dma_4u_supported(struct device *dev, u64 device_mask)
742b02c2b0bSChristoph Hellwig {
743b02c2b0bSChristoph Hellwig struct iommu *iommu = dev->archdata.iommu;
744b02c2b0bSChristoph Hellwig
745c54fc984SChristoph Hellwig if (ali_sound_dma_hack(dev, device_mask))
746c54fc984SChristoph Hellwig return 1;
747c54fc984SChristoph Hellwig
748254ecb16SChristoph Hellwig if (device_mask < iommu->dma_addr_mask)
749b02c2b0bSChristoph Hellwig return 0;
750b02c2b0bSChristoph Hellwig return 1;
751b02c2b0bSChristoph Hellwig }
752b02c2b0bSChristoph Hellwig
7535299709dSBart Van Assche static const struct dma_map_ops sun4u_dma_ops = {
754c416258aSAndrzej Pietrasiewicz .alloc = dma_4u_alloc_coherent,
755c416258aSAndrzej Pietrasiewicz .free = dma_4u_free_coherent,
756797a7568SFUJITA Tomonori .map_page = dma_4u_map_page,
757797a7568SFUJITA Tomonori .unmap_page = dma_4u_unmap_page,
758a88b5ba8SSam Ravnborg .map_sg = dma_4u_map_sg,
759a88b5ba8SSam Ravnborg .unmap_sg = dma_4u_unmap_sg,
760a88b5ba8SSam Ravnborg .sync_single_for_cpu = dma_4u_sync_single_for_cpu,
761a88b5ba8SSam Ravnborg .sync_sg_for_cpu = dma_4u_sync_sg_for_cpu,
762b02c2b0bSChristoph Hellwig .dma_supported = dma_4u_supported,
763a88b5ba8SSam Ravnborg };
764a88b5ba8SSam Ravnborg
7655299709dSBart Van Assche const struct dma_map_ops *dma_ops = &sun4u_dma_ops;
766a88b5ba8SSam Ravnborg EXPORT_SYMBOL(dma_ops);
767