1 /**
2  * iommu_fill_pdir - Insert coalesced scatter/gather chunks into the I/O Pdir.
3  * @ioc: The I/O Controller.
4  * @startsg: The scatter/gather list of coalesced chunks.
5  * @nents: The number of entries in the scatter/gather list.
6  * @hint: The DMA Hint.
7  *
8  * This function inserts the coalesced scatter/gather list chunks into the
9  * I/O Controller's I/O Pdir.
10  */
11 static inline unsigned int
12 iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents,
13 		unsigned long hint,
14 		void (*iommu_io_pdir_entry)(u64 *, space_t, unsigned long,
15 					    unsigned long))
16 {
17 	struct scatterlist *dma_sg = startsg;	/* pointer to current DMA */
18 	unsigned int n_mappings = 0;
19 	unsigned long dma_offset = 0, dma_len = 0;
20 	u64 *pdirp = NULL;
21 
22 	/* Horrible hack.  For efficiency's sake, dma_sg starts one
23 	 * entry below the true start (it is immediately incremented
24 	 * in the loop) */
25 	 dma_sg--;
26 
27 	while (nents-- > 0) {
28 		unsigned long vaddr;
29 		long size;
30 
31 		DBG_RUN_SG(" %d : %08lx/%05x %08lx/%05x\n", nents,
32 			   (unsigned long)sg_dma_address(startsg), cnt,
33 			   sg_virt_addr(startsg), startsg->length
34 		);
35 
36 
37 		/*
38 		** Look for the start of a new DMA stream
39 		*/
40 
41 		if (sg_dma_address(startsg) & PIDE_FLAG) {
42 			u32 pide = sg_dma_address(startsg) & ~PIDE_FLAG;
43 
44 			BUG_ON(pdirp && (dma_len != sg_dma_len(dma_sg)));
45 
46 			dma_sg++;
47 
48 			dma_len = sg_dma_len(startsg);
49 			sg_dma_len(startsg) = 0;
50 			dma_offset = (unsigned long) pide & ~IOVP_MASK;
51 			n_mappings++;
52 #if defined(ZX1_SUPPORT)
53 			/* Pluto IOMMU IO Virt Address is not zero based */
54 			sg_dma_address(dma_sg) = pide | ioc->ibase;
55 #else
56 			/* SBA, ccio, and dino are zero based.
57 			 * Trying to save a few CPU cycles for most users.
58 			 */
59 			sg_dma_address(dma_sg) = pide;
60 #endif
61 			pdirp = &(ioc->pdir_base[pide >> IOVP_SHIFT]);
62 			prefetchw(pdirp);
63 		}
64 
65 		BUG_ON(pdirp == NULL);
66 
67 		vaddr = sg_virt_addr(startsg);
68 		sg_dma_len(dma_sg) += startsg->length;
69 		size = startsg->length + dma_offset;
70 		dma_offset = 0;
71 #ifdef IOMMU_MAP_STATS
72 		ioc->msg_pages += startsg->length >> IOVP_SHIFT;
73 #endif
74 		do {
75 			iommu_io_pdir_entry(pdirp, KERNEL_SPACE,
76 					    vaddr, hint);
77 			vaddr += IOVP_SIZE;
78 			size -= IOVP_SIZE;
79 			pdirp++;
80 		} while(unlikely(size > 0));
81 		startsg++;
82 	}
83 	return(n_mappings);
84 }
85 
86 
87 /*
88 ** First pass is to walk the SG list and determine where the breaks are
89 ** in the DMA stream. Allocates PDIR entries but does not fill them.
90 ** Returns the number of DMA chunks.
91 **
92 ** Doing the fill separate from the coalescing/allocation keeps the
93 ** code simpler. Future enhancement could make one pass through
94 ** the sglist do both.
95 */
96 
97 static inline unsigned int
98 iommu_coalesce_chunks(struct ioc *ioc, struct device *dev,
99 		struct scatterlist *startsg, int nents,
100 		int (*iommu_alloc_range)(struct ioc *, struct device *, size_t))
101 {
102 	struct scatterlist *contig_sg;	   /* contig chunk head */
103 	unsigned long dma_offset, dma_len; /* start/len of DMA stream */
104 	unsigned int n_mappings = 0;
105 	unsigned int max_seg_size = dma_get_max_seg_size(dev);
106 
107 	while (nents > 0) {
108 
109 		/*
110 		** Prepare for first/next DMA stream
111 		*/
112 		contig_sg = startsg;
113 		dma_len = startsg->length;
114 		dma_offset = sg_virt_addr(startsg) & ~IOVP_MASK;
115 
116 		/* PARANOID: clear entries */
117 		sg_dma_address(startsg) = 0;
118 		sg_dma_len(startsg) = 0;
119 
120 		/*
121 		** This loop terminates one iteration "early" since
122 		** it's always looking one "ahead".
123 		*/
124 		while(--nents > 0) {
125 			unsigned long prevstartsg_end, startsg_end;
126 
127 			prevstartsg_end = sg_virt_addr(startsg) +
128 				startsg->length;
129 
130 			startsg++;
131 			startsg_end = sg_virt_addr(startsg) +
132 				startsg->length;
133 
134 			/* PARANOID: clear entries */
135 			sg_dma_address(startsg) = 0;
136 			sg_dma_len(startsg) = 0;
137 
138 			/*
139 			** First make sure current dma stream won't
140 			** exceed DMA_CHUNK_SIZE if we coalesce the
141 			** next entry.
142 			*/
143 			if(unlikely(ALIGN(dma_len + dma_offset + startsg->length,
144 					    IOVP_SIZE) > DMA_CHUNK_SIZE))
145 				break;
146 
147 			if (startsg->length + dma_len > max_seg_size)
148 				break;
149 
150 			/*
151 			** Next see if we can append the next chunk (i.e.
152 			** it must end on one page and begin on another
153 			*/
154 			if (unlikely(((prevstartsg_end | sg_virt_addr(startsg)) & ~PAGE_MASK) != 0))
155 				break;
156 
157 			dma_len += startsg->length;
158 		}
159 
160 		/*
161 		** End of DMA Stream
162 		** Terminate last VCONTIG block.
163 		** Allocate space for DMA stream.
164 		*/
165 		sg_dma_len(contig_sg) = dma_len;
166 		dma_len = ALIGN(dma_len + dma_offset, IOVP_SIZE);
167 		sg_dma_address(contig_sg) =
168 			PIDE_FLAG
169 			| (iommu_alloc_range(ioc, dev, dma_len) << IOVP_SHIFT)
170 			| dma_offset;
171 		n_mappings++;
172 	}
173 
174 	return n_mappings;
175 }
176 
177