1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
4  *
5  * Rewrite, cleanup:
6  *
7  * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
8  * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
9  *
10  * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
11  */
12 
13 #include <linux/init.h>
14 #include <linux/types.h>
15 #include <linux/slab.h>
16 #include <linux/mm.h>
17 #include <linux/memblock.h>
18 #include <linux/spinlock.h>
19 #include <linux/string.h>
20 #include <linux/pci.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/crash_dump.h>
23 #include <linux/memory.h>
24 #include <linux/of.h>
25 #include <linux/iommu.h>
26 #include <linux/rculist.h>
27 #include <asm/io.h>
28 #include <asm/prom.h>
29 #include <asm/rtas.h>
30 #include <asm/iommu.h>
31 #include <asm/pci-bridge.h>
32 #include <asm/machdep.h>
33 #include <asm/firmware.h>
34 #include <asm/tce.h>
35 #include <asm/ppc-pci.h>
36 #include <asm/udbg.h>
37 #include <asm/mmzone.h>
38 #include <asm/plpar_wrappers.h>
39 
40 #include "pseries.h"
41 
42 enum {
43 	DDW_QUERY_PE_DMA_WIN  = 0,
44 	DDW_CREATE_PE_DMA_WIN = 1,
45 	DDW_REMOVE_PE_DMA_WIN = 2,
46 
47 	DDW_APPLICABLE_SIZE
48 };
49 
50 enum {
51 	DDW_EXT_SIZE = 0,
52 	DDW_EXT_RESET_DMA_WIN = 1,
53 	DDW_EXT_QUERY_OUT_SIZE = 2
54 };
55 
56 static struct iommu_table *iommu_pseries_alloc_table(int node)
57 {
58 	struct iommu_table *tbl;
59 
60 	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node);
61 	if (!tbl)
62 		return NULL;
63 
64 	INIT_LIST_HEAD_RCU(&tbl->it_group_list);
65 	kref_init(&tbl->it_kref);
66 	return tbl;
67 }
68 
69 static struct iommu_table_group *iommu_pseries_alloc_group(int node)
70 {
71 	struct iommu_table_group *table_group;
72 
73 	table_group = kzalloc_node(sizeof(*table_group), GFP_KERNEL, node);
74 	if (!table_group)
75 		return NULL;
76 
77 	table_group->tables[0] = iommu_pseries_alloc_table(node);
78 	if (table_group->tables[0])
79 		return table_group;
80 
81 	kfree(table_group);
82 	return NULL;
83 }
84 
85 static void iommu_pseries_free_group(struct iommu_table_group *table_group,
86 		const char *node_name)
87 {
88 	struct iommu_table *tbl;
89 
90 	if (!table_group)
91 		return;
92 
93 	tbl = table_group->tables[0];
94 #ifdef CONFIG_IOMMU_API
95 	if (table_group->group) {
96 		iommu_group_put(table_group->group);
97 		BUG_ON(table_group->group);
98 	}
99 #endif
100 	iommu_tce_table_put(tbl);
101 
102 	kfree(table_group);
103 }
104 
105 static int tce_build_pSeries(struct iommu_table *tbl, long index,
106 			      long npages, unsigned long uaddr,
107 			      enum dma_data_direction direction,
108 			      unsigned long attrs)
109 {
110 	u64 proto_tce;
111 	__be64 *tcep;
112 	u64 rpn;
113 	const unsigned long tceshift = tbl->it_page_shift;
114 	const unsigned long pagesize = IOMMU_PAGE_SIZE(tbl);
115 
116 	proto_tce = TCE_PCI_READ; // Read allowed
117 
118 	if (direction != DMA_TO_DEVICE)
119 		proto_tce |= TCE_PCI_WRITE;
120 
121 	tcep = ((__be64 *)tbl->it_base) + index;
122 
123 	while (npages--) {
124 		/* can't move this out since we might cross MEMBLOCK boundary */
125 		rpn = __pa(uaddr) >> tceshift;
126 		*tcep = cpu_to_be64(proto_tce | rpn << tceshift);
127 
128 		uaddr += pagesize;
129 		tcep++;
130 	}
131 	return 0;
132 }
133 
134 
135 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
136 {
137 	__be64 *tcep;
138 
139 	tcep = ((__be64 *)tbl->it_base) + index;
140 
141 	while (npages--)
142 		*(tcep++) = 0;
143 }
144 
145 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
146 {
147 	__be64 *tcep;
148 
149 	tcep = ((__be64 *)tbl->it_base) + index;
150 
151 	return be64_to_cpu(*tcep);
152 }
153 
154 static void tce_free_pSeriesLP(unsigned long liobn, long, long, long);
155 static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
156 
157 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
158 				long npages, unsigned long uaddr,
159 				enum dma_data_direction direction,
160 				unsigned long attrs)
161 {
162 	u64 rc = 0;
163 	u64 proto_tce, tce;
164 	u64 rpn;
165 	int ret = 0;
166 	long tcenum_start = tcenum, npages_start = npages;
167 
168 	rpn = __pa(uaddr) >> tceshift;
169 	proto_tce = TCE_PCI_READ;
170 	if (direction != DMA_TO_DEVICE)
171 		proto_tce |= TCE_PCI_WRITE;
172 
173 	while (npages--) {
174 		tce = proto_tce | rpn << tceshift;
175 		rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, tce);
176 
177 		if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
178 			ret = (int)rc;
179 			tce_free_pSeriesLP(liobn, tcenum_start, tceshift,
180 			                   (npages_start - (npages + 1)));
181 			break;
182 		}
183 
184 		if (rc && printk_ratelimit()) {
185 			printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
186 			printk("\tindex   = 0x%llx\n", (u64)liobn);
187 			printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
188 			printk("\ttce val = 0x%llx\n", tce );
189 			dump_stack();
190 		}
191 
192 		tcenum++;
193 		rpn++;
194 	}
195 	return ret;
196 }
197 
198 static DEFINE_PER_CPU(__be64 *, tce_page);
199 
200 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
201 				     long npages, unsigned long uaddr,
202 				     enum dma_data_direction direction,
203 				     unsigned long attrs)
204 {
205 	u64 rc = 0;
206 	u64 proto_tce;
207 	__be64 *tcep;
208 	u64 rpn;
209 	long l, limit;
210 	long tcenum_start = tcenum, npages_start = npages;
211 	int ret = 0;
212 	unsigned long flags;
213 	const unsigned long tceshift = tbl->it_page_shift;
214 
215 	if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
216 		return tce_build_pSeriesLP(tbl->it_index, tcenum,
217 					   tceshift, npages, uaddr,
218 		                           direction, attrs);
219 	}
220 
221 	local_irq_save(flags);	/* to protect tcep and the page behind it */
222 
223 	tcep = __this_cpu_read(tce_page);
224 
225 	/* This is safe to do since interrupts are off when we're called
226 	 * from iommu_alloc{,_sg}()
227 	 */
228 	if (!tcep) {
229 		tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
230 		/* If allocation fails, fall back to the loop implementation */
231 		if (!tcep) {
232 			local_irq_restore(flags);
233 			return tce_build_pSeriesLP(tbl->it_index, tcenum,
234 					tceshift,
235 					npages, uaddr, direction, attrs);
236 		}
237 		__this_cpu_write(tce_page, tcep);
238 	}
239 
240 	rpn = __pa(uaddr) >> tceshift;
241 	proto_tce = TCE_PCI_READ;
242 	if (direction != DMA_TO_DEVICE)
243 		proto_tce |= TCE_PCI_WRITE;
244 
245 	/* We can map max one pageful of TCEs at a time */
246 	do {
247 		/*
248 		 * Set up the page with TCE data, looping through and setting
249 		 * the values.
250 		 */
251 		limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
252 
253 		for (l = 0; l < limit; l++) {
254 			tcep[l] = cpu_to_be64(proto_tce | rpn << tceshift);
255 			rpn++;
256 		}
257 
258 		rc = plpar_tce_put_indirect((u64)tbl->it_index,
259 					    (u64)tcenum << tceshift,
260 					    (u64)__pa(tcep),
261 					    limit);
262 
263 		npages -= limit;
264 		tcenum += limit;
265 	} while (npages > 0 && !rc);
266 
267 	local_irq_restore(flags);
268 
269 	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
270 		ret = (int)rc;
271 		tce_freemulti_pSeriesLP(tbl, tcenum_start,
272 		                        (npages_start - (npages + limit)));
273 		return ret;
274 	}
275 
276 	if (rc && printk_ratelimit()) {
277 		printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
278 		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
279 		printk("\tnpages  = 0x%llx\n", (u64)npages);
280 		printk("\ttce[0] val = 0x%llx\n", tcep[0]);
281 		dump_stack();
282 	}
283 	return ret;
284 }
285 
286 static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
287 			       long npages)
288 {
289 	u64 rc;
290 
291 	while (npages--) {
292 		rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, 0);
293 
294 		if (rc && printk_ratelimit()) {
295 			printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
296 			printk("\tindex   = 0x%llx\n", (u64)liobn);
297 			printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
298 			dump_stack();
299 		}
300 
301 		tcenum++;
302 	}
303 }
304 
305 
306 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
307 {
308 	u64 rc;
309 
310 	if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
311 		return tce_free_pSeriesLP(tbl->it_index, tcenum,
312 					  tbl->it_page_shift, npages);
313 
314 	rc = plpar_tce_stuff((u64)tbl->it_index,
315 			     (u64)tcenum << tbl->it_page_shift, 0, npages);
316 
317 	if (rc && printk_ratelimit()) {
318 		printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
319 		printk("\trc      = %lld\n", rc);
320 		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
321 		printk("\tnpages  = 0x%llx\n", (u64)npages);
322 		dump_stack();
323 	}
324 }
325 
326 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
327 {
328 	u64 rc;
329 	unsigned long tce_ret;
330 
331 	rc = plpar_tce_get((u64)tbl->it_index,
332 			   (u64)tcenum << tbl->it_page_shift, &tce_ret);
333 
334 	if (rc && printk_ratelimit()) {
335 		printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
336 		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
337 		printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
338 		dump_stack();
339 	}
340 
341 	return tce_ret;
342 }
343 
344 /* this is compatible with cells for the device tree property */
345 struct dynamic_dma_window_prop {
346 	__be32	liobn;		/* tce table number */
347 	__be64	dma_base;	/* address hi,lo */
348 	__be32	tce_shift;	/* ilog2(tce_page_size) */
349 	__be32	window_shift;	/* ilog2(tce_window_size) */
350 };
351 
352 struct dma_win {
353 	struct device_node *device;
354 	const struct dynamic_dma_window_prop *prop;
355 	struct list_head list;
356 };
357 
358 /* Dynamic DMA Window support */
359 struct ddw_query_response {
360 	u32 windows_available;
361 	u64 largest_available_block;
362 	u32 page_size;
363 	u32 migration_capable;
364 };
365 
366 struct ddw_create_response {
367 	u32 liobn;
368 	u32 addr_hi;
369 	u32 addr_lo;
370 };
371 
372 static LIST_HEAD(dma_win_list);
373 /* prevents races between memory on/offline and window creation */
374 static DEFINE_SPINLOCK(dma_win_list_lock);
375 /* protects initializing window twice for same device */
376 static DEFINE_MUTEX(dma_win_init_mutex);
377 #define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
378 #define DMA64_PROPNAME "linux,dma64-ddr-window-info"
379 
380 static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
381 					unsigned long num_pfn, const void *arg)
382 {
383 	const struct dynamic_dma_window_prop *maprange = arg;
384 	int rc;
385 	u64 tce_size, num_tce, dma_offset, next;
386 	u32 tce_shift;
387 	long limit;
388 
389 	tce_shift = be32_to_cpu(maprange->tce_shift);
390 	tce_size = 1ULL << tce_shift;
391 	next = start_pfn << PAGE_SHIFT;
392 	num_tce = num_pfn << PAGE_SHIFT;
393 
394 	/* round back to the beginning of the tce page size */
395 	num_tce += next & (tce_size - 1);
396 	next &= ~(tce_size - 1);
397 
398 	/* covert to number of tces */
399 	num_tce |= tce_size - 1;
400 	num_tce >>= tce_shift;
401 
402 	do {
403 		/*
404 		 * Set up the page with TCE data, looping through and setting
405 		 * the values.
406 		 */
407 		limit = min_t(long, num_tce, 512);
408 		dma_offset = next + be64_to_cpu(maprange->dma_base);
409 
410 		rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
411 					     dma_offset,
412 					     0, limit);
413 		next += limit * tce_size;
414 		num_tce -= limit;
415 	} while (num_tce > 0 && !rc);
416 
417 	return rc;
418 }
419 
420 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
421 					unsigned long num_pfn, const void *arg)
422 {
423 	const struct dynamic_dma_window_prop *maprange = arg;
424 	u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn;
425 	__be64 *tcep;
426 	u32 tce_shift;
427 	u64 rc = 0;
428 	long l, limit;
429 
430 	if (!firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
431 		unsigned long tceshift = be32_to_cpu(maprange->tce_shift);
432 		unsigned long dmastart = (start_pfn << PAGE_SHIFT) +
433 				be64_to_cpu(maprange->dma_base);
434 		unsigned long tcenum = dmastart >> tceshift;
435 		unsigned long npages = num_pfn << PAGE_SHIFT >> tceshift;
436 		void *uaddr = __va(start_pfn << PAGE_SHIFT);
437 
438 		return tce_build_pSeriesLP(be32_to_cpu(maprange->liobn),
439 				tcenum, tceshift, npages, (unsigned long) uaddr,
440 				DMA_BIDIRECTIONAL, 0);
441 	}
442 
443 	local_irq_disable();	/* to protect tcep and the page behind it */
444 	tcep = __this_cpu_read(tce_page);
445 
446 	if (!tcep) {
447 		tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
448 		if (!tcep) {
449 			local_irq_enable();
450 			return -ENOMEM;
451 		}
452 		__this_cpu_write(tce_page, tcep);
453 	}
454 
455 	proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
456 
457 	liobn = (u64)be32_to_cpu(maprange->liobn);
458 	tce_shift = be32_to_cpu(maprange->tce_shift);
459 	tce_size = 1ULL << tce_shift;
460 	next = start_pfn << PAGE_SHIFT;
461 	num_tce = num_pfn << PAGE_SHIFT;
462 
463 	/* round back to the beginning of the tce page size */
464 	num_tce += next & (tce_size - 1);
465 	next &= ~(tce_size - 1);
466 
467 	/* covert to number of tces */
468 	num_tce |= tce_size - 1;
469 	num_tce >>= tce_shift;
470 
471 	/* We can map max one pageful of TCEs at a time */
472 	do {
473 		/*
474 		 * Set up the page with TCE data, looping through and setting
475 		 * the values.
476 		 */
477 		limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
478 		dma_offset = next + be64_to_cpu(maprange->dma_base);
479 
480 		for (l = 0; l < limit; l++) {
481 			tcep[l] = cpu_to_be64(proto_tce | next);
482 			next += tce_size;
483 		}
484 
485 		rc = plpar_tce_put_indirect(liobn,
486 					    dma_offset,
487 					    (u64)__pa(tcep),
488 					    limit);
489 
490 		num_tce -= limit;
491 	} while (num_tce > 0 && !rc);
492 
493 	/* error cleanup: caller will clear whole range */
494 
495 	local_irq_enable();
496 	return rc;
497 }
498 
499 static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
500 		unsigned long num_pfn, void *arg)
501 {
502 	return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
503 }
504 
505 static void iommu_table_setparms_common(struct iommu_table *tbl, unsigned long busno,
506 					unsigned long liobn, unsigned long win_addr,
507 					unsigned long window_size, unsigned long page_shift,
508 					void *base, struct iommu_table_ops *table_ops)
509 {
510 	tbl->it_busno = busno;
511 	tbl->it_index = liobn;
512 	tbl->it_offset = win_addr >> page_shift;
513 	tbl->it_size = window_size >> page_shift;
514 	tbl->it_page_shift = page_shift;
515 	tbl->it_base = (unsigned long)base;
516 	tbl->it_blocksize = 16;
517 	tbl->it_type = TCE_PCI;
518 	tbl->it_ops = table_ops;
519 }
520 
521 struct iommu_table_ops iommu_table_pseries_ops;
522 
523 static void iommu_table_setparms(struct pci_controller *phb,
524 				 struct device_node *dn,
525 				 struct iommu_table *tbl)
526 {
527 	struct device_node *node;
528 	const unsigned long *basep;
529 	const u32 *sizep;
530 
531 	/* Test if we are going over 2GB of DMA space */
532 	if (phb->dma_window_base_cur + phb->dma_window_size > SZ_2G) {
533 		udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
534 		panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
535 	}
536 
537 	node = phb->dn;
538 	basep = of_get_property(node, "linux,tce-base", NULL);
539 	sizep = of_get_property(node, "linux,tce-size", NULL);
540 	if (basep == NULL || sizep == NULL) {
541 		printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %pOF has "
542 				"missing tce entries !\n", dn);
543 		return;
544 	}
545 
546 	iommu_table_setparms_common(tbl, phb->bus->number, 0, phb->dma_window_base_cur,
547 				    phb->dma_window_size, IOMMU_PAGE_SHIFT_4K,
548 				    __va(*basep), &iommu_table_pseries_ops);
549 
550 	if (!is_kdump_kernel())
551 		memset((void *)tbl->it_base, 0, *sizep);
552 
553 	phb->dma_window_base_cur += phb->dma_window_size;
554 }
555 
556 struct iommu_table_ops iommu_table_lpar_multi_ops;
557 
558 /*
559  * iommu_table_setparms_lpar
560  *
561  * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
562  */
563 static void iommu_table_setparms_lpar(struct pci_controller *phb,
564 				      struct device_node *dn,
565 				      struct iommu_table *tbl,
566 				      struct iommu_table_group *table_group,
567 				      const __be32 *dma_window)
568 {
569 	unsigned long offset, size, liobn;
570 
571 	of_parse_dma_window(dn, dma_window, &liobn, &offset, &size);
572 
573 	iommu_table_setparms_common(tbl, phb->bus->number, liobn, offset, size, IOMMU_PAGE_SHIFT_4K, NULL,
574 				    &iommu_table_lpar_multi_ops);
575 
576 
577 	table_group->tce32_start = offset;
578 	table_group->tce32_size = size;
579 }
580 
581 struct iommu_table_ops iommu_table_pseries_ops = {
582 	.set = tce_build_pSeries,
583 	.clear = tce_free_pSeries,
584 	.get = tce_get_pseries
585 };
586 
587 static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
588 {
589 	struct device_node *dn;
590 	struct iommu_table *tbl;
591 	struct device_node *isa_dn, *isa_dn_orig;
592 	struct device_node *tmp;
593 	struct pci_dn *pci;
594 	int children;
595 
596 	dn = pci_bus_to_OF_node(bus);
597 
598 	pr_debug("pci_dma_bus_setup_pSeries: setting up bus %pOF\n", dn);
599 
600 	if (bus->self) {
601 		/* This is not a root bus, any setup will be done for the
602 		 * device-side of the bridge in iommu_dev_setup_pSeries().
603 		 */
604 		return;
605 	}
606 	pci = PCI_DN(dn);
607 
608 	/* Check if the ISA bus on the system is under
609 	 * this PHB.
610 	 */
611 	isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
612 
613 	while (isa_dn && isa_dn != dn)
614 		isa_dn = isa_dn->parent;
615 
616 	of_node_put(isa_dn_orig);
617 
618 	/* Count number of direct PCI children of the PHB. */
619 	for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
620 		children++;
621 
622 	pr_debug("Children: %d\n", children);
623 
624 	/* Calculate amount of DMA window per slot. Each window must be
625 	 * a power of two (due to pci_alloc_consistent requirements).
626 	 *
627 	 * Keep 256MB aside for PHBs with ISA.
628 	 */
629 
630 	if (!isa_dn) {
631 		/* No ISA/IDE - just set window size and return */
632 		pci->phb->dma_window_size = 0x80000000ul; /* To be divided */
633 
634 		while (pci->phb->dma_window_size * children > 0x80000000ul)
635 			pci->phb->dma_window_size >>= 1;
636 		pr_debug("No ISA/IDE, window size is 0x%llx\n",
637 			 pci->phb->dma_window_size);
638 		pci->phb->dma_window_base_cur = 0;
639 
640 		return;
641 	}
642 
643 	/* If we have ISA, then we probably have an IDE
644 	 * controller too. Allocate a 128MB table but
645 	 * skip the first 128MB to avoid stepping on ISA
646 	 * space.
647 	 */
648 	pci->phb->dma_window_size = 0x8000000ul;
649 	pci->phb->dma_window_base_cur = 0x8000000ul;
650 
651 	pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
652 	tbl = pci->table_group->tables[0];
653 
654 	iommu_table_setparms(pci->phb, dn, tbl);
655 
656 	if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
657 		panic("Failed to initialize iommu table");
658 
659 	/* Divide the rest (1.75GB) among the children */
660 	pci->phb->dma_window_size = 0x80000000ul;
661 	while (pci->phb->dma_window_size * children > 0x70000000ul)
662 		pci->phb->dma_window_size >>= 1;
663 
664 	pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
665 }
666 
667 #ifdef CONFIG_IOMMU_API
668 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned
669 				long *tce, enum dma_data_direction *direction)
670 {
671 	long rc;
672 	unsigned long ioba = (unsigned long) index << tbl->it_page_shift;
673 	unsigned long flags, oldtce = 0;
674 	u64 proto_tce = iommu_direction_to_tce_perm(*direction);
675 	unsigned long newtce = *tce | proto_tce;
676 
677 	spin_lock_irqsave(&tbl->large_pool.lock, flags);
678 
679 	rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce);
680 	if (!rc)
681 		rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce);
682 
683 	if (!rc) {
684 		*direction = iommu_tce_direction(oldtce);
685 		*tce = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
686 	}
687 
688 	spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
689 
690 	return rc;
691 }
692 #endif
693 
694 struct iommu_table_ops iommu_table_lpar_multi_ops = {
695 	.set = tce_buildmulti_pSeriesLP,
696 #ifdef CONFIG_IOMMU_API
697 	.xchg_no_kill = tce_exchange_pseries,
698 #endif
699 	.clear = tce_freemulti_pSeriesLP,
700 	.get = tce_get_pSeriesLP
701 };
702 
703 /*
704  * Find nearest ibm,dma-window (default DMA window) or direct DMA window or
705  * dynamic 64bit DMA window, walking up the device tree.
706  */
707 static struct device_node *pci_dma_find(struct device_node *dn,
708 					const __be32 **dma_window)
709 {
710 	const __be32 *dw = NULL;
711 
712 	for ( ; dn && PCI_DN(dn); dn = dn->parent) {
713 		dw = of_get_property(dn, "ibm,dma-window", NULL);
714 		if (dw) {
715 			if (dma_window)
716 				*dma_window = dw;
717 			return dn;
718 		}
719 		dw = of_get_property(dn, DIRECT64_PROPNAME, NULL);
720 		if (dw)
721 			return dn;
722 		dw = of_get_property(dn, DMA64_PROPNAME, NULL);
723 		if (dw)
724 			return dn;
725 	}
726 
727 	return NULL;
728 }
729 
730 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
731 {
732 	struct iommu_table *tbl;
733 	struct device_node *dn, *pdn;
734 	struct pci_dn *ppci;
735 	const __be32 *dma_window = NULL;
736 
737 	dn = pci_bus_to_OF_node(bus);
738 
739 	pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %pOF\n",
740 		 dn);
741 
742 	pdn = pci_dma_find(dn, &dma_window);
743 
744 	if (dma_window == NULL)
745 		pr_debug("  no ibm,dma-window property !\n");
746 
747 	ppci = PCI_DN(pdn);
748 
749 	pr_debug("  parent is %pOF, iommu_table: 0x%p\n",
750 		 pdn, ppci->table_group);
751 
752 	if (!ppci->table_group) {
753 		ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node);
754 		tbl = ppci->table_group->tables[0];
755 		if (dma_window) {
756 			iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
757 						  ppci->table_group, dma_window);
758 
759 			if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
760 				panic("Failed to initialize iommu table");
761 		}
762 		iommu_register_group(ppci->table_group,
763 				pci_domain_nr(bus), 0);
764 		pr_debug("  created table: %p\n", ppci->table_group);
765 	}
766 }
767 
768 
769 static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
770 {
771 	struct device_node *dn;
772 	struct iommu_table *tbl;
773 
774 	pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
775 
776 	dn = dev->dev.of_node;
777 
778 	/* If we're the direct child of a root bus, then we need to allocate
779 	 * an iommu table ourselves. The bus setup code should have setup
780 	 * the window sizes already.
781 	 */
782 	if (!dev->bus->self) {
783 		struct pci_controller *phb = PCI_DN(dn)->phb;
784 
785 		pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
786 		PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node);
787 		tbl = PCI_DN(dn)->table_group->tables[0];
788 		iommu_table_setparms(phb, dn, tbl);
789 
790 		if (!iommu_init_table(tbl, phb->node, 0, 0))
791 			panic("Failed to initialize iommu table");
792 
793 		set_iommu_table_base(&dev->dev, tbl);
794 		return;
795 	}
796 
797 	/* If this device is further down the bus tree, search upwards until
798 	 * an already allocated iommu table is found and use that.
799 	 */
800 
801 	while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL)
802 		dn = dn->parent;
803 
804 	if (dn && PCI_DN(dn))
805 		set_iommu_table_base(&dev->dev,
806 				PCI_DN(dn)->table_group->tables[0]);
807 	else
808 		printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
809 		       pci_name(dev));
810 }
811 
812 static int __read_mostly disable_ddw;
813 
814 static int __init disable_ddw_setup(char *str)
815 {
816 	disable_ddw = 1;
817 	printk(KERN_INFO "ppc iommu: disabling ddw.\n");
818 
819 	return 0;
820 }
821 
822 early_param("disable_ddw", disable_ddw_setup);
823 
824 static void clean_dma_window(struct device_node *np, struct dynamic_dma_window_prop *dwp)
825 {
826 	int ret;
827 
828 	ret = tce_clearrange_multi_pSeriesLP(0,
829 		1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
830 	if (ret)
831 		pr_warn("%pOF failed to clear tces in window.\n",
832 			np);
833 	else
834 		pr_debug("%pOF successfully cleared tces in window.\n",
835 			 np);
836 }
837 
838 /*
839  * Call only if DMA window is clean.
840  */
841 static void __remove_dma_window(struct device_node *np, u32 *ddw_avail, u64 liobn)
842 {
843 	int ret;
844 
845 	ret = rtas_call(ddw_avail[DDW_REMOVE_PE_DMA_WIN], 1, 1, NULL, liobn);
846 	if (ret)
847 		pr_warn("%pOF: failed to remove DMA window: rtas returned "
848 			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
849 			np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
850 	else
851 		pr_debug("%pOF: successfully removed DMA window: rtas returned "
852 			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
853 			np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
854 }
855 
856 static void remove_dma_window(struct device_node *np, u32 *ddw_avail,
857 			      struct property *win)
858 {
859 	struct dynamic_dma_window_prop *dwp;
860 	u64 liobn;
861 
862 	dwp = win->value;
863 	liobn = (u64)be32_to_cpu(dwp->liobn);
864 
865 	clean_dma_window(np, dwp);
866 	__remove_dma_window(np, ddw_avail, liobn);
867 }
868 
869 static int remove_ddw(struct device_node *np, bool remove_prop, const char *win_name)
870 {
871 	struct property *win;
872 	u32 ddw_avail[DDW_APPLICABLE_SIZE];
873 	int ret = 0;
874 
875 	win = of_find_property(np, win_name, NULL);
876 	if (!win)
877 		return -EINVAL;
878 
879 	ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
880 					 &ddw_avail[0], DDW_APPLICABLE_SIZE);
881 	if (ret)
882 		return 0;
883 
884 
885 	if (win->length >= sizeof(struct dynamic_dma_window_prop))
886 		remove_dma_window(np, ddw_avail, win);
887 
888 	if (!remove_prop)
889 		return 0;
890 
891 	ret = of_remove_property(np, win);
892 	if (ret)
893 		pr_warn("%pOF: failed to remove DMA window property: %d\n",
894 			np, ret);
895 	return 0;
896 }
897 
898 static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift)
899 {
900 	struct dma_win *window;
901 	const struct dynamic_dma_window_prop *dma64;
902 	bool found = false;
903 
904 	spin_lock(&dma_win_list_lock);
905 	/* check if we already created a window and dupe that config if so */
906 	list_for_each_entry(window, &dma_win_list, list) {
907 		if (window->device == pdn) {
908 			dma64 = window->prop;
909 			*dma_addr = be64_to_cpu(dma64->dma_base);
910 			*window_shift = be32_to_cpu(dma64->window_shift);
911 			found = true;
912 			break;
913 		}
914 	}
915 	spin_unlock(&dma_win_list_lock);
916 
917 	return found;
918 }
919 
920 static struct dma_win *ddw_list_new_entry(struct device_node *pdn,
921 					  const struct dynamic_dma_window_prop *dma64)
922 {
923 	struct dma_win *window;
924 
925 	window = kzalloc(sizeof(*window), GFP_KERNEL);
926 	if (!window)
927 		return NULL;
928 
929 	window->device = pdn;
930 	window->prop = dma64;
931 
932 	return window;
933 }
934 
935 static void find_existing_ddw_windows_named(const char *name)
936 {
937 	int len;
938 	struct device_node *pdn;
939 	struct dma_win *window;
940 	const struct dynamic_dma_window_prop *dma64;
941 
942 	for_each_node_with_property(pdn, name) {
943 		dma64 = of_get_property(pdn, name, &len);
944 		if (!dma64 || len < sizeof(*dma64)) {
945 			remove_ddw(pdn, true, name);
946 			continue;
947 		}
948 
949 		window = ddw_list_new_entry(pdn, dma64);
950 		if (!window) {
951 			of_node_put(pdn);
952 			break;
953 		}
954 
955 		spin_lock(&dma_win_list_lock);
956 		list_add(&window->list, &dma_win_list);
957 		spin_unlock(&dma_win_list_lock);
958 	}
959 }
960 
961 static int find_existing_ddw_windows(void)
962 {
963 	if (!firmware_has_feature(FW_FEATURE_LPAR))
964 		return 0;
965 
966 	find_existing_ddw_windows_named(DIRECT64_PROPNAME);
967 	find_existing_ddw_windows_named(DMA64_PROPNAME);
968 
969 	return 0;
970 }
971 machine_arch_initcall(pseries, find_existing_ddw_windows);
972 
973 /**
974  * ddw_read_ext - Get the value of an DDW extension
975  * @np:		device node from which the extension value is to be read.
976  * @extnum:	index number of the extension.
977  * @value:	pointer to return value, modified when extension is available.
978  *
979  * Checks if "ibm,ddw-extensions" exists for this node, and get the value
980  * on index 'extnum'.
981  * It can be used only to check if a property exists, passing value == NULL.
982  *
983  * Returns:
984  *	0 if extension successfully read
985  *	-EINVAL if the "ibm,ddw-extensions" does not exist,
986  *	-ENODATA if "ibm,ddw-extensions" does not have a value, and
987  *	-EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
988  */
989 static inline int ddw_read_ext(const struct device_node *np, int extnum,
990 			       u32 *value)
991 {
992 	static const char propname[] = "ibm,ddw-extensions";
993 	u32 count;
994 	int ret;
995 
996 	ret = of_property_read_u32_index(np, propname, DDW_EXT_SIZE, &count);
997 	if (ret)
998 		return ret;
999 
1000 	if (count < extnum)
1001 		return -EOVERFLOW;
1002 
1003 	if (!value)
1004 		value = &count;
1005 
1006 	return of_property_read_u32_index(np, propname, extnum, value);
1007 }
1008 
1009 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1010 		     struct ddw_query_response *query,
1011 		     struct device_node *parent)
1012 {
1013 	struct device_node *dn;
1014 	struct pci_dn *pdn;
1015 	u32 cfg_addr, ext_query, query_out[5];
1016 	u64 buid;
1017 	int ret, out_sz;
1018 
1019 	/*
1020 	 * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many
1021 	 * output parameters ibm,query-pe-dma-windows will have, ranging from
1022 	 * 5 to 6.
1023 	 */
1024 	ret = ddw_read_ext(parent, DDW_EXT_QUERY_OUT_SIZE, &ext_query);
1025 	if (!ret && ext_query == 1)
1026 		out_sz = 6;
1027 	else
1028 		out_sz = 5;
1029 
1030 	/*
1031 	 * Get the config address and phb buid of the PE window.
1032 	 * Rely on eeh to retrieve this for us.
1033 	 * Retrieve them from the pci device, not the node with the
1034 	 * dma-window property
1035 	 */
1036 	dn = pci_device_to_OF_node(dev);
1037 	pdn = PCI_DN(dn);
1038 	buid = pdn->phb->buid;
1039 	cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1040 
1041 	ret = rtas_call(ddw_avail[DDW_QUERY_PE_DMA_WIN], 3, out_sz, query_out,
1042 			cfg_addr, BUID_HI(buid), BUID_LO(buid));
1043 
1044 	switch (out_sz) {
1045 	case 5:
1046 		query->windows_available = query_out[0];
1047 		query->largest_available_block = query_out[1];
1048 		query->page_size = query_out[2];
1049 		query->migration_capable = query_out[3];
1050 		break;
1051 	case 6:
1052 		query->windows_available = query_out[0];
1053 		query->largest_available_block = ((u64)query_out[1] << 32) |
1054 						 query_out[2];
1055 		query->page_size = query_out[3];
1056 		query->migration_capable = query_out[4];
1057 		break;
1058 	}
1059 
1060 	dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d, lb=%llx ps=%x wn=%d\n",
1061 		 ddw_avail[DDW_QUERY_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1062 		 BUID_LO(buid), ret, query->largest_available_block,
1063 		 query->page_size, query->windows_available);
1064 
1065 	return ret;
1066 }
1067 
1068 static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1069 			struct ddw_create_response *create, int page_shift,
1070 			int window_shift)
1071 {
1072 	struct device_node *dn;
1073 	struct pci_dn *pdn;
1074 	u32 cfg_addr;
1075 	u64 buid;
1076 	int ret;
1077 
1078 	/*
1079 	 * Get the config address and phb buid of the PE window.
1080 	 * Rely on eeh to retrieve this for us.
1081 	 * Retrieve them from the pci device, not the node with the
1082 	 * dma-window property
1083 	 */
1084 	dn = pci_device_to_OF_node(dev);
1085 	pdn = PCI_DN(dn);
1086 	buid = pdn->phb->buid;
1087 	cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1088 
1089 	do {
1090 		/* extra outputs are LIOBN and dma-addr (hi, lo) */
1091 		ret = rtas_call(ddw_avail[DDW_CREATE_PE_DMA_WIN], 5, 4,
1092 				(u32 *)create, cfg_addr, BUID_HI(buid),
1093 				BUID_LO(buid), page_shift, window_shift);
1094 	} while (rtas_busy_delay(ret));
1095 	dev_info(&dev->dev,
1096 		"ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
1097 		"(liobn = 0x%x starting addr = %x %x)\n",
1098 		 ddw_avail[DDW_CREATE_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1099 		 BUID_LO(buid), page_shift, window_shift, ret, create->liobn,
1100 		 create->addr_hi, create->addr_lo);
1101 
1102 	return ret;
1103 }
1104 
1105 struct failed_ddw_pdn {
1106 	struct device_node *pdn;
1107 	struct list_head list;
1108 };
1109 
1110 static LIST_HEAD(failed_ddw_pdn_list);
1111 
1112 static phys_addr_t ddw_memory_hotplug_max(void)
1113 {
1114 	phys_addr_t max_addr = memory_hotplug_max();
1115 	struct device_node *memory;
1116 
1117 	for_each_node_by_type(memory, "memory") {
1118 		unsigned long start, size;
1119 		int n_mem_addr_cells, n_mem_size_cells, len;
1120 		const __be32 *memcell_buf;
1121 
1122 		memcell_buf = of_get_property(memory, "reg", &len);
1123 		if (!memcell_buf || len <= 0)
1124 			continue;
1125 
1126 		n_mem_addr_cells = of_n_addr_cells(memory);
1127 		n_mem_size_cells = of_n_size_cells(memory);
1128 
1129 		start = of_read_number(memcell_buf, n_mem_addr_cells);
1130 		memcell_buf += n_mem_addr_cells;
1131 		size = of_read_number(memcell_buf, n_mem_size_cells);
1132 		memcell_buf += n_mem_size_cells;
1133 
1134 		max_addr = max_t(phys_addr_t, max_addr, start + size);
1135 	}
1136 
1137 	return max_addr;
1138 }
1139 
1140 /*
1141  * Platforms supporting the DDW option starting with LoPAR level 2.7 implement
1142  * ibm,ddw-extensions, which carries the rtas token for
1143  * ibm,reset-pe-dma-windows.
1144  * That rtas-call can be used to restore the default DMA window for the device.
1145  */
1146 static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
1147 {
1148 	int ret;
1149 	u32 cfg_addr, reset_dma_win;
1150 	u64 buid;
1151 	struct device_node *dn;
1152 	struct pci_dn *pdn;
1153 
1154 	ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win);
1155 	if (ret)
1156 		return;
1157 
1158 	dn = pci_device_to_OF_node(dev);
1159 	pdn = PCI_DN(dn);
1160 	buid = pdn->phb->buid;
1161 	cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8);
1162 
1163 	ret = rtas_call(reset_dma_win, 3, 1, NULL, cfg_addr, BUID_HI(buid),
1164 			BUID_LO(buid));
1165 	if (ret)
1166 		dev_info(&dev->dev,
1167 			 "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ",
1168 			 reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid),
1169 			 ret);
1170 }
1171 
1172 /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
1173 static int iommu_get_page_shift(u32 query_page_size)
1174 {
1175 	/* Supported IO page-sizes according to LoPAR, note that 2M is out of order */
1176 	const int shift[] = {
1177 		__builtin_ctzll(SZ_4K),   __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
1178 		__builtin_ctzll(SZ_32M),  __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
1179 		__builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G), __builtin_ctzll(SZ_2M)
1180 	};
1181 
1182 	int i = ARRAY_SIZE(shift) - 1;
1183 	int ret = 0;
1184 
1185 	/*
1186 	 * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
1187 	 * - bit 31 means 4k pages are supported,
1188 	 * - bit 30 means 64k pages are supported, and so on.
1189 	 * Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
1190 	 */
1191 	for (; i >= 0 ; i--) {
1192 		if (query_page_size & (1 << i))
1193 			ret = max(ret, shift[i]);
1194 	}
1195 
1196 	return ret;
1197 }
1198 
1199 static struct property *ddw_property_create(const char *propname, u32 liobn, u64 dma_addr,
1200 					    u32 page_shift, u32 window_shift)
1201 {
1202 	struct dynamic_dma_window_prop *ddwprop;
1203 	struct property *win64;
1204 
1205 	win64 = kzalloc(sizeof(*win64), GFP_KERNEL);
1206 	if (!win64)
1207 		return NULL;
1208 
1209 	win64->name = kstrdup(propname, GFP_KERNEL);
1210 	ddwprop = kzalloc(sizeof(*ddwprop), GFP_KERNEL);
1211 	win64->value = ddwprop;
1212 	win64->length = sizeof(*ddwprop);
1213 	if (!win64->name || !win64->value) {
1214 		kfree(win64->name);
1215 		kfree(win64->value);
1216 		kfree(win64);
1217 		return NULL;
1218 	}
1219 
1220 	ddwprop->liobn = cpu_to_be32(liobn);
1221 	ddwprop->dma_base = cpu_to_be64(dma_addr);
1222 	ddwprop->tce_shift = cpu_to_be32(page_shift);
1223 	ddwprop->window_shift = cpu_to_be32(window_shift);
1224 
1225 	return win64;
1226 }
1227 
1228 /*
1229  * If the PE supports dynamic dma windows, and there is space for a table
1230  * that can map all pages in a linear offset, then setup such a table,
1231  * and record the dma-offset in the struct device.
1232  *
1233  * dev: the pci device we are checking
1234  * pdn: the parent pe node with the ibm,dma_window property
1235  * Future: also check if we can remap the base window for our base page size
1236  *
1237  * returns true if can map all pages (direct mapping), false otherwise..
1238  */
1239 static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn)
1240 {
1241 	int len = 0, ret;
1242 	int max_ram_len = order_base_2(ddw_memory_hotplug_max());
1243 	struct ddw_query_response query;
1244 	struct ddw_create_response create;
1245 	int page_shift;
1246 	u64 win_addr;
1247 	const char *win_name;
1248 	struct device_node *dn;
1249 	u32 ddw_avail[DDW_APPLICABLE_SIZE];
1250 	struct dma_win *window;
1251 	struct property *win64;
1252 	struct failed_ddw_pdn *fpdn;
1253 	bool default_win_removed = false, direct_mapping = false;
1254 	bool pmem_present;
1255 	struct pci_dn *pci = PCI_DN(pdn);
1256 	struct property *default_win = NULL;
1257 
1258 	dn = of_find_node_by_type(NULL, "ibm,pmemory");
1259 	pmem_present = dn != NULL;
1260 	of_node_put(dn);
1261 
1262 	mutex_lock(&dma_win_init_mutex);
1263 
1264 	if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len)) {
1265 		direct_mapping = (len >= max_ram_len);
1266 		goto out_unlock;
1267 	}
1268 
1269 	/*
1270 	 * If we already went through this for a previous function of
1271 	 * the same device and failed, we don't want to muck with the
1272 	 * DMA window again, as it will race with in-flight operations
1273 	 * and can lead to EEHs. The above mutex protects access to the
1274 	 * list.
1275 	 */
1276 	list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) {
1277 		if (fpdn->pdn == pdn)
1278 			goto out_unlock;
1279 	}
1280 
1281 	/*
1282 	 * the ibm,ddw-applicable property holds the tokens for:
1283 	 * ibm,query-pe-dma-window
1284 	 * ibm,create-pe-dma-window
1285 	 * ibm,remove-pe-dma-window
1286 	 * for the given node in that order.
1287 	 * the property is actually in the parent, not the PE
1288 	 */
1289 	ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable",
1290 					 &ddw_avail[0], DDW_APPLICABLE_SIZE);
1291 	if (ret)
1292 		goto out_failed;
1293 
1294        /*
1295 	 * Query if there is a second window of size to map the
1296 	 * whole partition.  Query returns number of windows, largest
1297 	 * block assigned to PE (partition endpoint), and two bitmasks
1298 	 * of page sizes: supported and supported for migrate-dma.
1299 	 */
1300 	dn = pci_device_to_OF_node(dev);
1301 	ret = query_ddw(dev, ddw_avail, &query, pdn);
1302 	if (ret != 0)
1303 		goto out_failed;
1304 
1305 	/*
1306 	 * If there is no window available, remove the default DMA window,
1307 	 * if it's present. This will make all the resources available to the
1308 	 * new DDW window.
1309 	 * If anything fails after this, we need to restore it, so also check
1310 	 * for extensions presence.
1311 	 */
1312 	if (query.windows_available == 0) {
1313 		int reset_win_ext;
1314 
1315 		/* DDW + IOMMU on single window may fail if there is any allocation */
1316 		if (iommu_table_in_use(pci->table_group->tables[0])) {
1317 			dev_warn(&dev->dev, "current IOMMU table in use, can't be replaced.\n");
1318 			goto out_failed;
1319 		}
1320 
1321 		default_win = of_find_property(pdn, "ibm,dma-window", NULL);
1322 		if (!default_win)
1323 			goto out_failed;
1324 
1325 		reset_win_ext = ddw_read_ext(pdn, DDW_EXT_RESET_DMA_WIN, NULL);
1326 		if (reset_win_ext)
1327 			goto out_failed;
1328 
1329 		remove_dma_window(pdn, ddw_avail, default_win);
1330 		default_win_removed = true;
1331 
1332 		/* Query again, to check if the window is available */
1333 		ret = query_ddw(dev, ddw_avail, &query, pdn);
1334 		if (ret != 0)
1335 			goto out_failed;
1336 
1337 		if (query.windows_available == 0) {
1338 			/* no windows are available for this device. */
1339 			dev_dbg(&dev->dev, "no free dynamic windows");
1340 			goto out_failed;
1341 		}
1342 	}
1343 
1344 	page_shift = iommu_get_page_shift(query.page_size);
1345 	if (!page_shift) {
1346 		dev_dbg(&dev->dev, "no supported page size in mask %x",
1347 			query.page_size);
1348 		goto out_failed;
1349 	}
1350 
1351 
1352 	/*
1353 	 * The "ibm,pmemory" can appear anywhere in the address space.
1354 	 * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS
1355 	 * for the upper limit and fallback to max RAM otherwise but this
1356 	 * disables device::dma_ops_bypass.
1357 	 */
1358 	len = max_ram_len;
1359 	if (pmem_present) {
1360 		if (query.largest_available_block >=
1361 		    (1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1362 			len = MAX_PHYSMEM_BITS;
1363 		else
1364 			dev_info(&dev->dev, "Skipping ibm,pmemory");
1365 	}
1366 
1367 	/* check if the available block * number of ptes will map everything */
1368 	if (query.largest_available_block < (1ULL << (len - page_shift))) {
1369 		dev_dbg(&dev->dev,
1370 			"can't map partition max 0x%llx with %llu %llu-sized pages\n",
1371 			1ULL << len,
1372 			query.largest_available_block,
1373 			1ULL << page_shift);
1374 
1375 		len = order_base_2(query.largest_available_block << page_shift);
1376 		win_name = DMA64_PROPNAME;
1377 	} else {
1378 		direct_mapping = !default_win_removed ||
1379 			(len == MAX_PHYSMEM_BITS) ||
1380 			(!pmem_present && (len == max_ram_len));
1381 		win_name = direct_mapping ? DIRECT64_PROPNAME : DMA64_PROPNAME;
1382 	}
1383 
1384 	ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
1385 	if (ret != 0)
1386 		goto out_failed;
1387 
1388 	dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n",
1389 		  create.liobn, dn);
1390 
1391 	win_addr = ((u64)create.addr_hi << 32) | create.addr_lo;
1392 	win64 = ddw_property_create(win_name, create.liobn, win_addr, page_shift, len);
1393 
1394 	if (!win64) {
1395 		dev_info(&dev->dev,
1396 			 "couldn't allocate property, property name, or value\n");
1397 		goto out_remove_win;
1398 	}
1399 
1400 	ret = of_add_property(pdn, win64);
1401 	if (ret) {
1402 		dev_err(&dev->dev, "unable to add DMA window property for %pOF: %d",
1403 			pdn, ret);
1404 		goto out_free_prop;
1405 	}
1406 
1407 	window = ddw_list_new_entry(pdn, win64->value);
1408 	if (!window)
1409 		goto out_del_prop;
1410 
1411 	if (direct_mapping) {
1412 		/* DDW maps the whole partition, so enable direct DMA mapping */
1413 		ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
1414 					    win64->value, tce_setrange_multi_pSeriesLP_walk);
1415 		if (ret) {
1416 			dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n",
1417 				 dn, ret);
1418 
1419 			/* Make sure to clean DDW if any TCE was set*/
1420 			clean_dma_window(pdn, win64->value);
1421 			goto out_del_list;
1422 		}
1423 	} else {
1424 		struct iommu_table *newtbl;
1425 		int i;
1426 		unsigned long start = 0, end = 0;
1427 
1428 		for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) {
1429 			const unsigned long mask = IORESOURCE_MEM_64 | IORESOURCE_MEM;
1430 
1431 			/* Look for MMIO32 */
1432 			if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) {
1433 				start = pci->phb->mem_resources[i].start;
1434 				end = pci->phb->mem_resources[i].end;
1435 				break;
1436 			}
1437 		}
1438 
1439 		/* New table for using DDW instead of the default DMA window */
1440 		newtbl = iommu_pseries_alloc_table(pci->phb->node);
1441 		if (!newtbl) {
1442 			dev_dbg(&dev->dev, "couldn't create new IOMMU table\n");
1443 			goto out_del_list;
1444 		}
1445 
1446 		iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, win_addr,
1447 					    1UL << len, page_shift, NULL, &iommu_table_lpar_multi_ops);
1448 		iommu_init_table(newtbl, pci->phb->node, start, end);
1449 
1450 		pci->table_group->tables[1] = newtbl;
1451 
1452 		set_iommu_table_base(&dev->dev, newtbl);
1453 	}
1454 
1455 	if (default_win_removed) {
1456 		iommu_tce_table_put(pci->table_group->tables[0]);
1457 		pci->table_group->tables[0] = NULL;
1458 
1459 		/* default_win is valid here because default_win_removed == true */
1460 		of_remove_property(pdn, default_win);
1461 		dev_info(&dev->dev, "Removed default DMA window for %pOF\n", pdn);
1462 	}
1463 
1464 	spin_lock(&dma_win_list_lock);
1465 	list_add(&window->list, &dma_win_list);
1466 	spin_unlock(&dma_win_list_lock);
1467 
1468 	dev->dev.archdata.dma_offset = win_addr;
1469 	goto out_unlock;
1470 
1471 out_del_list:
1472 	kfree(window);
1473 
1474 out_del_prop:
1475 	of_remove_property(pdn, win64);
1476 
1477 out_free_prop:
1478 	kfree(win64->name);
1479 	kfree(win64->value);
1480 	kfree(win64);
1481 
1482 out_remove_win:
1483 	/* DDW is clean, so it's ok to call this directly. */
1484 	__remove_dma_window(pdn, ddw_avail, create.liobn);
1485 
1486 out_failed:
1487 	if (default_win_removed)
1488 		reset_dma_window(dev, pdn);
1489 
1490 	fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
1491 	if (!fpdn)
1492 		goto out_unlock;
1493 	fpdn->pdn = pdn;
1494 	list_add(&fpdn->list, &failed_ddw_pdn_list);
1495 
1496 out_unlock:
1497 	mutex_unlock(&dma_win_init_mutex);
1498 
1499 	/*
1500 	 * If we have persistent memory and the window size is only as big
1501 	 * as RAM, then we failed to create a window to cover persistent
1502 	 * memory and need to set the DMA limit.
1503 	 */
1504 	if (pmem_present && direct_mapping && len == max_ram_len)
1505 		dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << len);
1506 
1507 	return direct_mapping;
1508 }
1509 
1510 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
1511 {
1512 	struct device_node *pdn, *dn;
1513 	struct iommu_table *tbl;
1514 	const __be32 *dma_window = NULL;
1515 	struct pci_dn *pci;
1516 
1517 	pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1518 
1519 	/* dev setup for LPAR is a little tricky, since the device tree might
1520 	 * contain the dma-window properties per-device and not necessarily
1521 	 * for the bus. So we need to search upwards in the tree until we
1522 	 * either hit a dma-window property, OR find a parent with a table
1523 	 * already allocated.
1524 	 */
1525 	dn = pci_device_to_OF_node(dev);
1526 	pr_debug("  node is %pOF\n", dn);
1527 
1528 	pdn = pci_dma_find(dn, &dma_window);
1529 	if (!pdn || !PCI_DN(pdn)) {
1530 		printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1531 		       "no DMA window found for pci dev=%s dn=%pOF\n",
1532 				 pci_name(dev), dn);
1533 		return;
1534 	}
1535 	pr_debug("  parent is %pOF\n", pdn);
1536 
1537 	pci = PCI_DN(pdn);
1538 	if (!pci->table_group) {
1539 		pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
1540 		tbl = pci->table_group->tables[0];
1541 		iommu_table_setparms_lpar(pci->phb, pdn, tbl,
1542 				pci->table_group, dma_window);
1543 
1544 		iommu_init_table(tbl, pci->phb->node, 0, 0);
1545 		iommu_register_group(pci->table_group,
1546 				pci_domain_nr(pci->phb->bus), 0);
1547 		pr_debug("  created table: %p\n", pci->table_group);
1548 	} else {
1549 		pr_debug("  found DMA window, table: %p\n", pci->table_group);
1550 	}
1551 
1552 	set_iommu_table_base(&dev->dev, pci->table_group->tables[0]);
1553 	iommu_add_device(pci->table_group, &dev->dev);
1554 }
1555 
1556 static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask)
1557 {
1558 	struct device_node *dn = pci_device_to_OF_node(pdev), *pdn;
1559 
1560 	/* only attempt to use a new window if 64-bit DMA is requested */
1561 	if (dma_mask < DMA_BIT_MASK(64))
1562 		return false;
1563 
1564 	dev_dbg(&pdev->dev, "node is %pOF\n", dn);
1565 
1566 	/*
1567 	 * the device tree might contain the dma-window properties
1568 	 * per-device and not necessarily for the bus. So we need to
1569 	 * search upwards in the tree until we either hit a dma-window
1570 	 * property, OR find a parent with a table already allocated.
1571 	 */
1572 	pdn = pci_dma_find(dn, NULL);
1573 	if (pdn && PCI_DN(pdn))
1574 		return enable_ddw(pdev, pdn);
1575 
1576 	return false;
1577 }
1578 
1579 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1580 		void *data)
1581 {
1582 	struct dma_win *window;
1583 	struct memory_notify *arg = data;
1584 	int ret = 0;
1585 
1586 	switch (action) {
1587 	case MEM_GOING_ONLINE:
1588 		spin_lock(&dma_win_list_lock);
1589 		list_for_each_entry(window, &dma_win_list, list) {
1590 			ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1591 					arg->nr_pages, window->prop);
1592 			/* XXX log error */
1593 		}
1594 		spin_unlock(&dma_win_list_lock);
1595 		break;
1596 	case MEM_CANCEL_ONLINE:
1597 	case MEM_OFFLINE:
1598 		spin_lock(&dma_win_list_lock);
1599 		list_for_each_entry(window, &dma_win_list, list) {
1600 			ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1601 					arg->nr_pages, window->prop);
1602 			/* XXX log error */
1603 		}
1604 		spin_unlock(&dma_win_list_lock);
1605 		break;
1606 	default:
1607 		break;
1608 	}
1609 	if (ret && action != MEM_CANCEL_ONLINE)
1610 		return NOTIFY_BAD;
1611 
1612 	return NOTIFY_OK;
1613 }
1614 
1615 static struct notifier_block iommu_mem_nb = {
1616 	.notifier_call = iommu_mem_notifier,
1617 };
1618 
1619 static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data)
1620 {
1621 	int err = NOTIFY_OK;
1622 	struct of_reconfig_data *rd = data;
1623 	struct device_node *np = rd->dn;
1624 	struct pci_dn *pci = PCI_DN(np);
1625 	struct dma_win *window;
1626 
1627 	switch (action) {
1628 	case OF_RECONFIG_DETACH_NODE:
1629 		/*
1630 		 * Removing the property will invoke the reconfig
1631 		 * notifier again, which causes dead-lock on the
1632 		 * read-write semaphore of the notifier chain. So
1633 		 * we have to remove the property when releasing
1634 		 * the device node.
1635 		 */
1636 		if (remove_ddw(np, false, DIRECT64_PROPNAME))
1637 			remove_ddw(np, false, DMA64_PROPNAME);
1638 
1639 		if (pci && pci->table_group)
1640 			iommu_pseries_free_group(pci->table_group,
1641 					np->full_name);
1642 
1643 		spin_lock(&dma_win_list_lock);
1644 		list_for_each_entry(window, &dma_win_list, list) {
1645 			if (window->device == np) {
1646 				list_del(&window->list);
1647 				kfree(window);
1648 				break;
1649 			}
1650 		}
1651 		spin_unlock(&dma_win_list_lock);
1652 		break;
1653 	default:
1654 		err = NOTIFY_DONE;
1655 		break;
1656 	}
1657 	return err;
1658 }
1659 
1660 static struct notifier_block iommu_reconfig_nb = {
1661 	.notifier_call = iommu_reconfig_notifier,
1662 };
1663 
1664 /* These are called very early. */
1665 void __init iommu_init_early_pSeries(void)
1666 {
1667 	if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1668 		return;
1669 
1670 	if (firmware_has_feature(FW_FEATURE_LPAR)) {
1671 		pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1672 		pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1673 		if (!disable_ddw)
1674 			pseries_pci_controller_ops.iommu_bypass_supported =
1675 				iommu_bypass_supported_pSeriesLP;
1676 	} else {
1677 		pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries;
1678 		pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries;
1679 	}
1680 
1681 
1682 	of_reconfig_notifier_register(&iommu_reconfig_nb);
1683 	register_memory_notifier(&iommu_mem_nb);
1684 
1685 	set_pci_dma_ops(&dma_iommu_ops);
1686 }
1687 
1688 static int __init disable_multitce(char *str)
1689 {
1690 	if (strcmp(str, "off") == 0 &&
1691 	    firmware_has_feature(FW_FEATURE_LPAR) &&
1692 	    (firmware_has_feature(FW_FEATURE_PUT_TCE_IND) ||
1693 	     firmware_has_feature(FW_FEATURE_STUFF_TCE))) {
1694 		printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1695 		powerpc_firmware_features &=
1696 			~(FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE);
1697 	}
1698 	return 1;
1699 }
1700 
1701 __setup("multitce=", disable_multitce);
1702 
1703 static int tce_iommu_bus_notifier(struct notifier_block *nb,
1704 		unsigned long action, void *data)
1705 {
1706 	struct device *dev = data;
1707 
1708 	switch (action) {
1709 	case BUS_NOTIFY_DEL_DEVICE:
1710 		iommu_del_device(dev);
1711 		return 0;
1712 	default:
1713 		return 0;
1714 	}
1715 }
1716 
1717 static struct notifier_block tce_iommu_bus_nb = {
1718 	.notifier_call = tce_iommu_bus_notifier,
1719 };
1720 
1721 static int __init tce_iommu_bus_notifier_init(void)
1722 {
1723 	bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1724 	return 0;
1725 }
1726 machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init);
1727