1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Support PCI/PCIe on PowerNV platforms
4  *
5  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
6  */
7 
8 #undef DEBUG
9 
10 #include <linux/kernel.h>
11 #include <linux/pci.h>
12 #include <linux/crash_dump.h>
13 #include <linux/delay.h>
14 #include <linux/string.h>
15 #include <linux/init.h>
16 #include <linux/memblock.h>
17 #include <linux/irq.h>
18 #include <linux/io.h>
19 #include <linux/msi.h>
20 #include <linux/iommu.h>
21 #include <linux/rculist.h>
22 #include <linux/sizes.h>
23 #include <linux/debugfs.h>
24 #include <linux/of_address.h>
25 #include <linux/of_irq.h>
26 
27 #include <asm/sections.h>
28 #include <asm/io.h>
29 #include <asm/pci-bridge.h>
30 #include <asm/machdep.h>
31 #include <asm/msi_bitmap.h>
32 #include <asm/ppc-pci.h>
33 #include <asm/opal.h>
34 #include <asm/iommu.h>
35 #include <asm/tce.h>
36 #include <asm/xics.h>
37 #include <asm/firmware.h>
38 #include <asm/pnv-pci.h>
39 #include <asm/mmzone.h>
40 #include <asm/xive.h>
41 
42 #include <misc/cxl-base.h>
43 
44 #include "powernv.h"
45 #include "pci.h"
46 #include "../../../../drivers/pci/pci.h"
47 
48 #define PNV_IODA1_M64_NUM	16	/* Number of M64 BARs	*/
49 #define PNV_IODA1_M64_SEGS	8	/* Segments per M64 BAR	*/
50 #define PNV_IODA1_DMA32_SEGSIZE	0x10000000
51 
52 static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_OCAPI" };
53 
54 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
55 static void pnv_pci_configure_bus(struct pci_bus *bus);
56 
57 void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
58 			    const char *fmt, ...)
59 {
60 	struct va_format vaf;
61 	va_list args;
62 	char pfix[32];
63 
64 	va_start(args, fmt);
65 
66 	vaf.fmt = fmt;
67 	vaf.va = &args;
68 
69 	if (pe->flags & PNV_IODA_PE_DEV)
70 		strscpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
71 	else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
72 		sprintf(pfix, "%04x:%02x     ",
73 			pci_domain_nr(pe->pbus), pe->pbus->number);
74 #ifdef CONFIG_PCI_IOV
75 	else if (pe->flags & PNV_IODA_PE_VF)
76 		sprintf(pfix, "%04x:%02x:%2x.%d",
77 			pci_domain_nr(pe->parent_dev->bus),
78 			(pe->rid & 0xff00) >> 8,
79 			PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
80 #endif /* CONFIG_PCI_IOV*/
81 
82 	printk("%spci %s: [PE# %.2x] %pV",
83 	       level, pfix, pe->pe_number, &vaf);
84 
85 	va_end(args);
86 }
87 
88 static bool pnv_iommu_bypass_disabled __read_mostly;
89 static bool pci_reset_phbs __read_mostly;
90 
91 static int __init iommu_setup(char *str)
92 {
93 	if (!str)
94 		return -EINVAL;
95 
96 	while (*str) {
97 		if (!strncmp(str, "nobypass", 8)) {
98 			pnv_iommu_bypass_disabled = true;
99 			pr_info("PowerNV: IOMMU bypass window disabled.\n");
100 			break;
101 		}
102 		str += strcspn(str, ",");
103 		if (*str == ',')
104 			str++;
105 	}
106 
107 	return 0;
108 }
109 early_param("iommu", iommu_setup);
110 
111 static int __init pci_reset_phbs_setup(char *str)
112 {
113 	pci_reset_phbs = true;
114 	return 0;
115 }
116 
117 early_param("ppc_pci_reset_phbs", pci_reset_phbs_setup);
118 
119 static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
120 {
121 	s64 rc;
122 
123 	phb->ioda.pe_array[pe_no].phb = phb;
124 	phb->ioda.pe_array[pe_no].pe_number = pe_no;
125 	phb->ioda.pe_array[pe_no].dma_setup_done = false;
126 
127 	/*
128 	 * Clear the PE frozen state as it might be put into frozen state
129 	 * in the last PCI remove path. It's not harmful to do so when the
130 	 * PE is already in unfrozen state.
131 	 */
132 	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
133 				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
134 	if (rc != OPAL_SUCCESS && rc != OPAL_UNSUPPORTED)
135 		pr_warn("%s: Error %lld unfreezing PHB#%x-PE#%x\n",
136 			__func__, rc, phb->hose->global_number, pe_no);
137 
138 	return &phb->ioda.pe_array[pe_no];
139 }
140 
141 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
142 {
143 	if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe_num)) {
144 		pr_warn("%s: Invalid PE %x on PHB#%x\n",
145 			__func__, pe_no, phb->hose->global_number);
146 		return;
147 	}
148 
149 	mutex_lock(&phb->ioda.pe_alloc_mutex);
150 	if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
151 		pr_debug("%s: PE %x was reserved on PHB#%x\n",
152 			 __func__, pe_no, phb->hose->global_number);
153 	mutex_unlock(&phb->ioda.pe_alloc_mutex);
154 
155 	pnv_ioda_init_pe(phb, pe_no);
156 }
157 
158 struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb, int count)
159 {
160 	struct pnv_ioda_pe *ret = NULL;
161 	int run = 0, pe, i;
162 
163 	mutex_lock(&phb->ioda.pe_alloc_mutex);
164 
165 	/* scan backwards for a run of @count cleared bits */
166 	for (pe = phb->ioda.total_pe_num - 1; pe >= 0; pe--) {
167 		if (test_bit(pe, phb->ioda.pe_alloc)) {
168 			run = 0;
169 			continue;
170 		}
171 
172 		run++;
173 		if (run == count)
174 			break;
175 	}
176 	if (run != count)
177 		goto out;
178 
179 	for (i = pe; i < pe + count; i++) {
180 		set_bit(i, phb->ioda.pe_alloc);
181 		pnv_ioda_init_pe(phb, i);
182 	}
183 	ret = &phb->ioda.pe_array[pe];
184 
185 out:
186 	mutex_unlock(&phb->ioda.pe_alloc_mutex);
187 	return ret;
188 }
189 
190 void pnv_ioda_free_pe(struct pnv_ioda_pe *pe)
191 {
192 	struct pnv_phb *phb = pe->phb;
193 	unsigned int pe_num = pe->pe_number;
194 
195 	WARN_ON(pe->pdev);
196 	memset(pe, 0, sizeof(struct pnv_ioda_pe));
197 
198 	mutex_lock(&phb->ioda.pe_alloc_mutex);
199 	clear_bit(pe_num, phb->ioda.pe_alloc);
200 	mutex_unlock(&phb->ioda.pe_alloc_mutex);
201 }
202 
203 /* The default M64 BAR is shared by all PEs */
204 static int pnv_ioda2_init_m64(struct pnv_phb *phb)
205 {
206 	const char *desc;
207 	struct resource *r;
208 	s64 rc;
209 
210 	/* Configure the default M64 BAR */
211 	rc = opal_pci_set_phb_mem_window(phb->opal_id,
212 					 OPAL_M64_WINDOW_TYPE,
213 					 phb->ioda.m64_bar_idx,
214 					 phb->ioda.m64_base,
215 					 0, /* unused */
216 					 phb->ioda.m64_size);
217 	if (rc != OPAL_SUCCESS) {
218 		desc = "configuring";
219 		goto fail;
220 	}
221 
222 	/* Enable the default M64 BAR */
223 	rc = opal_pci_phb_mmio_enable(phb->opal_id,
224 				      OPAL_M64_WINDOW_TYPE,
225 				      phb->ioda.m64_bar_idx,
226 				      OPAL_ENABLE_M64_SPLIT);
227 	if (rc != OPAL_SUCCESS) {
228 		desc = "enabling";
229 		goto fail;
230 	}
231 
232 	/*
233 	 * Exclude the segments for reserved and root bus PE, which
234 	 * are first or last two PEs.
235 	 */
236 	r = &phb->hose->mem_resources[1];
237 	if (phb->ioda.reserved_pe_idx == 0)
238 		r->start += (2 * phb->ioda.m64_segsize);
239 	else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
240 		r->end -= (2 * phb->ioda.m64_segsize);
241 	else
242 		pr_warn("  Cannot strip M64 segment for reserved PE#%x\n",
243 			phb->ioda.reserved_pe_idx);
244 
245 	return 0;
246 
247 fail:
248 	pr_warn("  Failure %lld %s M64 BAR#%d\n",
249 		rc, desc, phb->ioda.m64_bar_idx);
250 	opal_pci_phb_mmio_enable(phb->opal_id,
251 				 OPAL_M64_WINDOW_TYPE,
252 				 phb->ioda.m64_bar_idx,
253 				 OPAL_DISABLE_M64);
254 	return -EIO;
255 }
256 
257 static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
258 					 unsigned long *pe_bitmap)
259 {
260 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
261 	struct resource *r;
262 	resource_size_t base, sgsz, start, end;
263 	int segno, i;
264 
265 	base = phb->ioda.m64_base;
266 	sgsz = phb->ioda.m64_segsize;
267 	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
268 		r = &pdev->resource[i];
269 		if (!r->parent || !pnv_pci_is_m64(phb, r))
270 			continue;
271 
272 		start = ALIGN_DOWN(r->start - base, sgsz);
273 		end = ALIGN(r->end - base, sgsz);
274 		for (segno = start / sgsz; segno < end / sgsz; segno++) {
275 			if (pe_bitmap)
276 				set_bit(segno, pe_bitmap);
277 			else
278 				pnv_ioda_reserve_pe(phb, segno);
279 		}
280 	}
281 }
282 
283 static int pnv_ioda1_init_m64(struct pnv_phb *phb)
284 {
285 	struct resource *r;
286 	int index;
287 
288 	/*
289 	 * There are 16 M64 BARs, each of which has 8 segments. So
290 	 * there are as many M64 segments as the maximum number of
291 	 * PEs, which is 128.
292 	 */
293 	for (index = 0; index < PNV_IODA1_M64_NUM; index++) {
294 		unsigned long base, segsz = phb->ioda.m64_segsize;
295 		int64_t rc;
296 
297 		base = phb->ioda.m64_base +
298 		       index * PNV_IODA1_M64_SEGS * segsz;
299 		rc = opal_pci_set_phb_mem_window(phb->opal_id,
300 				OPAL_M64_WINDOW_TYPE, index, base, 0,
301 				PNV_IODA1_M64_SEGS * segsz);
302 		if (rc != OPAL_SUCCESS) {
303 			pr_warn("  Error %lld setting M64 PHB#%x-BAR#%d\n",
304 				rc, phb->hose->global_number, index);
305 			goto fail;
306 		}
307 
308 		rc = opal_pci_phb_mmio_enable(phb->opal_id,
309 				OPAL_M64_WINDOW_TYPE, index,
310 				OPAL_ENABLE_M64_SPLIT);
311 		if (rc != OPAL_SUCCESS) {
312 			pr_warn("  Error %lld enabling M64 PHB#%x-BAR#%d\n",
313 				rc, phb->hose->global_number, index);
314 			goto fail;
315 		}
316 	}
317 
318 	for (index = 0; index < phb->ioda.total_pe_num; index++) {
319 		int64_t rc;
320 
321 		/*
322 		 * P7IOC supports M64DT, which helps mapping M64 segment
323 		 * to one particular PE#. However, PHB3 has fixed mapping
324 		 * between M64 segment and PE#. In order to have same logic
325 		 * for P7IOC and PHB3, we enforce fixed mapping between M64
326 		 * segment and PE# on P7IOC.
327 		 */
328 		rc = opal_pci_map_pe_mmio_window(phb->opal_id,
329 				index, OPAL_M64_WINDOW_TYPE,
330 				index / PNV_IODA1_M64_SEGS,
331 				index % PNV_IODA1_M64_SEGS);
332 		if (rc != OPAL_SUCCESS) {
333 			pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n",
334 				__func__, rc, phb->hose->global_number,
335 				index);
336 			goto fail;
337 		}
338 	}
339 
340 	/*
341 	 * Exclude the segments for reserved and root bus PE, which
342 	 * are first or last two PEs.
343 	 */
344 	r = &phb->hose->mem_resources[1];
345 	if (phb->ioda.reserved_pe_idx == 0)
346 		r->start += (2 * phb->ioda.m64_segsize);
347 	else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
348 		r->end -= (2 * phb->ioda.m64_segsize);
349 	else
350 		WARN(1, "Wrong reserved PE#%x on PHB#%x\n",
351 		     phb->ioda.reserved_pe_idx, phb->hose->global_number);
352 
353 	return 0;
354 
355 fail:
356 	for ( ; index >= 0; index--)
357 		opal_pci_phb_mmio_enable(phb->opal_id,
358 			OPAL_M64_WINDOW_TYPE, index, OPAL_DISABLE_M64);
359 
360 	return -EIO;
361 }
362 
363 static void pnv_ioda_reserve_m64_pe(struct pci_bus *bus,
364 				    unsigned long *pe_bitmap,
365 				    bool all)
366 {
367 	struct pci_dev *pdev;
368 
369 	list_for_each_entry(pdev, &bus->devices, bus_list) {
370 		pnv_ioda_reserve_dev_m64_pe(pdev, pe_bitmap);
371 
372 		if (all && pdev->subordinate)
373 			pnv_ioda_reserve_m64_pe(pdev->subordinate,
374 						pe_bitmap, all);
375 	}
376 }
377 
378 static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all)
379 {
380 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
381 	struct pnv_ioda_pe *master_pe, *pe;
382 	unsigned long size, *pe_alloc;
383 	int i;
384 
385 	/* Root bus shouldn't use M64 */
386 	if (pci_is_root_bus(bus))
387 		return NULL;
388 
389 	/* Allocate bitmap */
390 	size = ALIGN(phb->ioda.total_pe_num / 8, sizeof(unsigned long));
391 	pe_alloc = kzalloc(size, GFP_KERNEL);
392 	if (!pe_alloc) {
393 		pr_warn("%s: Out of memory !\n",
394 			__func__);
395 		return NULL;
396 	}
397 
398 	/* Figure out reserved PE numbers by the PE */
399 	pnv_ioda_reserve_m64_pe(bus, pe_alloc, all);
400 
401 	/*
402 	 * the current bus might not own M64 window and that's all
403 	 * contributed by its child buses. For the case, we needn't
404 	 * pick M64 dependent PE#.
405 	 */
406 	if (bitmap_empty(pe_alloc, phb->ioda.total_pe_num)) {
407 		kfree(pe_alloc);
408 		return NULL;
409 	}
410 
411 	/*
412 	 * Figure out the master PE and put all slave PEs to master
413 	 * PE's list to form compound PE.
414 	 */
415 	master_pe = NULL;
416 	i = -1;
417 	while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe_num, i + 1)) <
418 		phb->ioda.total_pe_num) {
419 		pe = &phb->ioda.pe_array[i];
420 
421 		phb->ioda.m64_segmap[pe->pe_number] = pe->pe_number;
422 		if (!master_pe) {
423 			pe->flags |= PNV_IODA_PE_MASTER;
424 			INIT_LIST_HEAD(&pe->slaves);
425 			master_pe = pe;
426 		} else {
427 			pe->flags |= PNV_IODA_PE_SLAVE;
428 			pe->master = master_pe;
429 			list_add_tail(&pe->list, &master_pe->slaves);
430 		}
431 	}
432 
433 	kfree(pe_alloc);
434 	return master_pe;
435 }
436 
437 static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
438 {
439 	struct pci_controller *hose = phb->hose;
440 	struct device_node *dn = hose->dn;
441 	struct resource *res;
442 	u32 m64_range[2], i;
443 	const __be32 *r;
444 	u64 pci_addr;
445 
446 	if (phb->type != PNV_PHB_IODA1 && phb->type != PNV_PHB_IODA2) {
447 		pr_info("  Not support M64 window\n");
448 		return;
449 	}
450 
451 	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
452 		pr_info("  Firmware too old to support M64 window\n");
453 		return;
454 	}
455 
456 	r = of_get_property(dn, "ibm,opal-m64-window", NULL);
457 	if (!r) {
458 		pr_info("  No <ibm,opal-m64-window> on %pOF\n",
459 			dn);
460 		return;
461 	}
462 
463 	/*
464 	 * Find the available M64 BAR range and pickup the last one for
465 	 * covering the whole 64-bits space. We support only one range.
466 	 */
467 	if (of_property_read_u32_array(dn, "ibm,opal-available-m64-ranges",
468 				       m64_range, 2)) {
469 		/* In absence of the property, assume 0..15 */
470 		m64_range[0] = 0;
471 		m64_range[1] = 16;
472 	}
473 	/* We only support 64 bits in our allocator */
474 	if (m64_range[1] > 63) {
475 		pr_warn("%s: Limiting M64 range to 63 (from %d) on PHB#%x\n",
476 			__func__, m64_range[1], phb->hose->global_number);
477 		m64_range[1] = 63;
478 	}
479 	/* Empty range, no m64 */
480 	if (m64_range[1] <= m64_range[0]) {
481 		pr_warn("%s: M64 empty, disabling M64 usage on PHB#%x\n",
482 			__func__, phb->hose->global_number);
483 		return;
484 	}
485 
486 	/* Configure M64 informations */
487 	res = &hose->mem_resources[1];
488 	res->name = dn->full_name;
489 	res->start = of_translate_address(dn, r + 2);
490 	res->end = res->start + of_read_number(r + 4, 2) - 1;
491 	res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
492 	pci_addr = of_read_number(r, 2);
493 	hose->mem_offset[1] = res->start - pci_addr;
494 
495 	phb->ioda.m64_size = resource_size(res);
496 	phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe_num;
497 	phb->ioda.m64_base = pci_addr;
498 
499 	/* This lines up nicely with the display from processing OF ranges */
500 	pr_info(" MEM 0x%016llx..0x%016llx -> 0x%016llx (M64 #%d..%d)\n",
501 		res->start, res->end, pci_addr, m64_range[0],
502 		m64_range[0] + m64_range[1] - 1);
503 
504 	/* Mark all M64 used up by default */
505 	phb->ioda.m64_bar_alloc = (unsigned long)-1;
506 
507 	/* Use last M64 BAR to cover M64 window */
508 	m64_range[1]--;
509 	phb->ioda.m64_bar_idx = m64_range[0] + m64_range[1];
510 
511 	pr_info(" Using M64 #%d as default window\n", phb->ioda.m64_bar_idx);
512 
513 	/* Mark remaining ones free */
514 	for (i = m64_range[0]; i < m64_range[1]; i++)
515 		clear_bit(i, &phb->ioda.m64_bar_alloc);
516 
517 	/*
518 	 * Setup init functions for M64 based on IODA version, IODA3 uses
519 	 * the IODA2 code.
520 	 */
521 	if (phb->type == PNV_PHB_IODA1)
522 		phb->init_m64 = pnv_ioda1_init_m64;
523 	else
524 		phb->init_m64 = pnv_ioda2_init_m64;
525 }
526 
527 static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
528 {
529 	struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
530 	struct pnv_ioda_pe *slave;
531 	s64 rc;
532 
533 	/* Fetch master PE */
534 	if (pe->flags & PNV_IODA_PE_SLAVE) {
535 		pe = pe->master;
536 		if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
537 			return;
538 
539 		pe_no = pe->pe_number;
540 	}
541 
542 	/* Freeze master PE */
543 	rc = opal_pci_eeh_freeze_set(phb->opal_id,
544 				     pe_no,
545 				     OPAL_EEH_ACTION_SET_FREEZE_ALL);
546 	if (rc != OPAL_SUCCESS) {
547 		pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
548 			__func__, rc, phb->hose->global_number, pe_no);
549 		return;
550 	}
551 
552 	/* Freeze slave PEs */
553 	if (!(pe->flags & PNV_IODA_PE_MASTER))
554 		return;
555 
556 	list_for_each_entry(slave, &pe->slaves, list) {
557 		rc = opal_pci_eeh_freeze_set(phb->opal_id,
558 					     slave->pe_number,
559 					     OPAL_EEH_ACTION_SET_FREEZE_ALL);
560 		if (rc != OPAL_SUCCESS)
561 			pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
562 				__func__, rc, phb->hose->global_number,
563 				slave->pe_number);
564 	}
565 }
566 
567 static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
568 {
569 	struct pnv_ioda_pe *pe, *slave;
570 	s64 rc;
571 
572 	/* Find master PE */
573 	pe = &phb->ioda.pe_array[pe_no];
574 	if (pe->flags & PNV_IODA_PE_SLAVE) {
575 		pe = pe->master;
576 		WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
577 		pe_no = pe->pe_number;
578 	}
579 
580 	/* Clear frozen state for master PE */
581 	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
582 	if (rc != OPAL_SUCCESS) {
583 		pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
584 			__func__, rc, opt, phb->hose->global_number, pe_no);
585 		return -EIO;
586 	}
587 
588 	if (!(pe->flags & PNV_IODA_PE_MASTER))
589 		return 0;
590 
591 	/* Clear frozen state for slave PEs */
592 	list_for_each_entry(slave, &pe->slaves, list) {
593 		rc = opal_pci_eeh_freeze_clear(phb->opal_id,
594 					     slave->pe_number,
595 					     opt);
596 		if (rc != OPAL_SUCCESS) {
597 			pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
598 				__func__, rc, opt, phb->hose->global_number,
599 				slave->pe_number);
600 			return -EIO;
601 		}
602 	}
603 
604 	return 0;
605 }
606 
607 static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
608 {
609 	struct pnv_ioda_pe *slave, *pe;
610 	u8 fstate = 0, state;
611 	__be16 pcierr = 0;
612 	s64 rc;
613 
614 	/* Sanity check on PE number */
615 	if (pe_no < 0 || pe_no >= phb->ioda.total_pe_num)
616 		return OPAL_EEH_STOPPED_PERM_UNAVAIL;
617 
618 	/*
619 	 * Fetch the master PE and the PE instance might be
620 	 * not initialized yet.
621 	 */
622 	pe = &phb->ioda.pe_array[pe_no];
623 	if (pe->flags & PNV_IODA_PE_SLAVE) {
624 		pe = pe->master;
625 		WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
626 		pe_no = pe->pe_number;
627 	}
628 
629 	/* Check the master PE */
630 	rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
631 					&state, &pcierr, NULL);
632 	if (rc != OPAL_SUCCESS) {
633 		pr_warn("%s: Failure %lld getting "
634 			"PHB#%x-PE#%x state\n",
635 			__func__, rc,
636 			phb->hose->global_number, pe_no);
637 		return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
638 	}
639 
640 	/* Check the slave PE */
641 	if (!(pe->flags & PNV_IODA_PE_MASTER))
642 		return state;
643 
644 	list_for_each_entry(slave, &pe->slaves, list) {
645 		rc = opal_pci_eeh_freeze_status(phb->opal_id,
646 						slave->pe_number,
647 						&fstate,
648 						&pcierr,
649 						NULL);
650 		if (rc != OPAL_SUCCESS) {
651 			pr_warn("%s: Failure %lld getting "
652 				"PHB#%x-PE#%x state\n",
653 				__func__, rc,
654 				phb->hose->global_number, slave->pe_number);
655 			return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
656 		}
657 
658 		/*
659 		 * Override the result based on the ascending
660 		 * priority.
661 		 */
662 		if (fstate > state)
663 			state = fstate;
664 	}
665 
666 	return state;
667 }
668 
669 struct pnv_ioda_pe *pnv_pci_bdfn_to_pe(struct pnv_phb *phb, u16 bdfn)
670 {
671 	int pe_number = phb->ioda.pe_rmap[bdfn];
672 
673 	if (pe_number == IODA_INVALID_PE)
674 		return NULL;
675 
676 	return &phb->ioda.pe_array[pe_number];
677 }
678 
679 struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
680 {
681 	struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
682 	struct pci_dn *pdn = pci_get_pdn(dev);
683 
684 	if (!pdn)
685 		return NULL;
686 	if (pdn->pe_number == IODA_INVALID_PE)
687 		return NULL;
688 	return &phb->ioda.pe_array[pdn->pe_number];
689 }
690 
691 static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
692 				  struct pnv_ioda_pe *parent,
693 				  struct pnv_ioda_pe *child,
694 				  bool is_add)
695 {
696 	const char *desc = is_add ? "adding" : "removing";
697 	uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
698 			      OPAL_REMOVE_PE_FROM_DOMAIN;
699 	struct pnv_ioda_pe *slave;
700 	long rc;
701 
702 	/* Parent PE affects child PE */
703 	rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
704 				child->pe_number, op);
705 	if (rc != OPAL_SUCCESS) {
706 		pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
707 			rc, desc);
708 		return -ENXIO;
709 	}
710 
711 	if (!(child->flags & PNV_IODA_PE_MASTER))
712 		return 0;
713 
714 	/* Compound case: parent PE affects slave PEs */
715 	list_for_each_entry(slave, &child->slaves, list) {
716 		rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
717 					slave->pe_number, op);
718 		if (rc != OPAL_SUCCESS) {
719 			pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
720 				rc, desc);
721 			return -ENXIO;
722 		}
723 	}
724 
725 	return 0;
726 }
727 
728 static int pnv_ioda_set_peltv(struct pnv_phb *phb,
729 			      struct pnv_ioda_pe *pe,
730 			      bool is_add)
731 {
732 	struct pnv_ioda_pe *slave;
733 	struct pci_dev *pdev = NULL;
734 	int ret;
735 
736 	/*
737 	 * Clear PE frozen state. If it's master PE, we need
738 	 * clear slave PE frozen state as well.
739 	 */
740 	if (is_add) {
741 		opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
742 					  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
743 		if (pe->flags & PNV_IODA_PE_MASTER) {
744 			list_for_each_entry(slave, &pe->slaves, list)
745 				opal_pci_eeh_freeze_clear(phb->opal_id,
746 							  slave->pe_number,
747 							  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
748 		}
749 	}
750 
751 	/*
752 	 * Associate PE in PELT. We need add the PE into the
753 	 * corresponding PELT-V as well. Otherwise, the error
754 	 * originated from the PE might contribute to other
755 	 * PEs.
756 	 */
757 	ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
758 	if (ret)
759 		return ret;
760 
761 	/* For compound PEs, any one affects all of them */
762 	if (pe->flags & PNV_IODA_PE_MASTER) {
763 		list_for_each_entry(slave, &pe->slaves, list) {
764 			ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
765 			if (ret)
766 				return ret;
767 		}
768 	}
769 
770 	if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
771 		pdev = pe->pbus->self;
772 	else if (pe->flags & PNV_IODA_PE_DEV)
773 		pdev = pe->pdev->bus->self;
774 #ifdef CONFIG_PCI_IOV
775 	else if (pe->flags & PNV_IODA_PE_VF)
776 		pdev = pe->parent_dev;
777 #endif /* CONFIG_PCI_IOV */
778 	while (pdev) {
779 		struct pci_dn *pdn = pci_get_pdn(pdev);
780 		struct pnv_ioda_pe *parent;
781 
782 		if (pdn && pdn->pe_number != IODA_INVALID_PE) {
783 			parent = &phb->ioda.pe_array[pdn->pe_number];
784 			ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
785 			if (ret)
786 				return ret;
787 		}
788 
789 		pdev = pdev->bus->self;
790 	}
791 
792 	return 0;
793 }
794 
795 static void pnv_ioda_unset_peltv(struct pnv_phb *phb,
796 				 struct pnv_ioda_pe *pe,
797 				 struct pci_dev *parent)
798 {
799 	int64_t rc;
800 
801 	while (parent) {
802 		struct pci_dn *pdn = pci_get_pdn(parent);
803 
804 		if (pdn && pdn->pe_number != IODA_INVALID_PE) {
805 			rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
806 						pe->pe_number,
807 						OPAL_REMOVE_PE_FROM_DOMAIN);
808 			/* XXX What to do in case of error ? */
809 		}
810 		parent = parent->bus->self;
811 	}
812 
813 	opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
814 				  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
815 
816 	/* Disassociate PE in PELT */
817 	rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
818 				pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
819 	if (rc)
820 		pe_warn(pe, "OPAL error %lld remove self from PELTV\n", rc);
821 }
822 
823 int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
824 {
825 	struct pci_dev *parent;
826 	uint8_t bcomp, dcomp, fcomp;
827 	int64_t rc;
828 	long rid_end, rid;
829 
830 	/* Currently, we just deconfigure VF PE. Bus PE will always there.*/
831 	if (pe->pbus) {
832 		int count;
833 
834 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
835 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
836 		parent = pe->pbus->self;
837 		if (pe->flags & PNV_IODA_PE_BUS_ALL)
838 			count = resource_size(&pe->pbus->busn_res);
839 		else
840 			count = 1;
841 
842 		switch(count) {
843 		case  1: bcomp = OpalPciBusAll;         break;
844 		case  2: bcomp = OpalPciBus7Bits;       break;
845 		case  4: bcomp = OpalPciBus6Bits;       break;
846 		case  8: bcomp = OpalPciBus5Bits;       break;
847 		case 16: bcomp = OpalPciBus4Bits;       break;
848 		case 32: bcomp = OpalPciBus3Bits;       break;
849 		default:
850 			dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
851 			        count);
852 			/* Do an exact match only */
853 			bcomp = OpalPciBusAll;
854 		}
855 		rid_end = pe->rid + (count << 8);
856 	} else {
857 #ifdef CONFIG_PCI_IOV
858 		if (pe->flags & PNV_IODA_PE_VF)
859 			parent = pe->parent_dev;
860 		else
861 #endif
862 			parent = pe->pdev->bus->self;
863 		bcomp = OpalPciBusAll;
864 		dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
865 		fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
866 		rid_end = pe->rid + 1;
867 	}
868 
869 	/* Clear the reverse map */
870 	for (rid = pe->rid; rid < rid_end; rid++)
871 		phb->ioda.pe_rmap[rid] = IODA_INVALID_PE;
872 
873 	/*
874 	 * Release from all parents PELT-V. NPUs don't have a PELTV
875 	 * table
876 	 */
877 	if (phb->type != PNV_PHB_NPU_OCAPI)
878 		pnv_ioda_unset_peltv(phb, pe, parent);
879 
880 	rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
881 			     bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
882 	if (rc)
883 		pe_err(pe, "OPAL error %lld trying to setup PELT table\n", rc);
884 
885 	pe->pbus = NULL;
886 	pe->pdev = NULL;
887 #ifdef CONFIG_PCI_IOV
888 	pe->parent_dev = NULL;
889 #endif
890 
891 	return 0;
892 }
893 
894 int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
895 {
896 	uint8_t bcomp, dcomp, fcomp;
897 	long rc, rid_end, rid;
898 
899 	/* Bus validation ? */
900 	if (pe->pbus) {
901 		int count;
902 
903 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
904 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
905 		if (pe->flags & PNV_IODA_PE_BUS_ALL)
906 			count = resource_size(&pe->pbus->busn_res);
907 		else
908 			count = 1;
909 
910 		switch(count) {
911 		case  1: bcomp = OpalPciBusAll;		break;
912 		case  2: bcomp = OpalPciBus7Bits;	break;
913 		case  4: bcomp = OpalPciBus6Bits;	break;
914 		case  8: bcomp = OpalPciBus5Bits;	break;
915 		case 16: bcomp = OpalPciBus4Bits;	break;
916 		case 32: bcomp = OpalPciBus3Bits;	break;
917 		default:
918 			dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
919 			        count);
920 			/* Do an exact match only */
921 			bcomp = OpalPciBusAll;
922 		}
923 		rid_end = pe->rid + (count << 8);
924 	} else {
925 		bcomp = OpalPciBusAll;
926 		dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
927 		fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
928 		rid_end = pe->rid + 1;
929 	}
930 
931 	/*
932 	 * Associate PE in PELT. We need add the PE into the
933 	 * corresponding PELT-V as well. Otherwise, the error
934 	 * originated from the PE might contribute to other
935 	 * PEs.
936 	 */
937 	rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
938 			     bcomp, dcomp, fcomp, OPAL_MAP_PE);
939 	if (rc) {
940 		pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
941 		return -ENXIO;
942 	}
943 
944 	/*
945 	 * Configure PELTV. NPUs don't have a PELTV table so skip
946 	 * configuration on them.
947 	 */
948 	if (phb->type != PNV_PHB_NPU_OCAPI)
949 		pnv_ioda_set_peltv(phb, pe, true);
950 
951 	/* Setup reverse map */
952 	for (rid = pe->rid; rid < rid_end; rid++)
953 		phb->ioda.pe_rmap[rid] = pe->pe_number;
954 
955 	/* Setup one MVTs on IODA1 */
956 	if (phb->type != PNV_PHB_IODA1) {
957 		pe->mve_number = 0;
958 		goto out;
959 	}
960 
961 	pe->mve_number = pe->pe_number;
962 	rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
963 	if (rc != OPAL_SUCCESS) {
964 		pe_err(pe, "OPAL error %ld setting up MVE %x\n",
965 		       rc, pe->mve_number);
966 		pe->mve_number = -1;
967 	} else {
968 		rc = opal_pci_set_mve_enable(phb->opal_id,
969 					     pe->mve_number, OPAL_ENABLE_MVE);
970 		if (rc) {
971 			pe_err(pe, "OPAL error %ld enabling MVE %x\n",
972 			       rc, pe->mve_number);
973 			pe->mve_number = -1;
974 		}
975 	}
976 
977 out:
978 	return 0;
979 }
980 
981 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
982 {
983 	struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
984 	struct pci_dn *pdn = pci_get_pdn(dev);
985 	struct pnv_ioda_pe *pe;
986 
987 	if (!pdn) {
988 		pr_err("%s: Device tree node not associated properly\n",
989 			   pci_name(dev));
990 		return NULL;
991 	}
992 	if (pdn->pe_number != IODA_INVALID_PE)
993 		return NULL;
994 
995 	pe = pnv_ioda_alloc_pe(phb, 1);
996 	if (!pe) {
997 		pr_warn("%s: Not enough PE# available, disabling device\n",
998 			pci_name(dev));
999 		return NULL;
1000 	}
1001 
1002 	/* NOTE: We don't get a reference for the pointer in the PE
1003 	 * data structure, both the device and PE structures should be
1004 	 * destroyed at the same time.
1005 	 *
1006 	 * At some point we want to remove the PDN completely anyways
1007 	 */
1008 	pdn->pe_number = pe->pe_number;
1009 	pe->flags = PNV_IODA_PE_DEV;
1010 	pe->pdev = dev;
1011 	pe->pbus = NULL;
1012 	pe->mve_number = -1;
1013 	pe->rid = dev->bus->number << 8 | pdn->devfn;
1014 	pe->device_count++;
1015 
1016 	pe_info(pe, "Associated device to PE\n");
1017 
1018 	if (pnv_ioda_configure_pe(phb, pe)) {
1019 		/* XXX What do we do here ? */
1020 		pnv_ioda_free_pe(pe);
1021 		pdn->pe_number = IODA_INVALID_PE;
1022 		pe->pdev = NULL;
1023 		return NULL;
1024 	}
1025 
1026 	/* Put PE to the list */
1027 	mutex_lock(&phb->ioda.pe_list_mutex);
1028 	list_add_tail(&pe->list, &phb->ioda.pe_list);
1029 	mutex_unlock(&phb->ioda.pe_list_mutex);
1030 	return pe;
1031 }
1032 
1033 /*
1034  * There're 2 types of PCI bus sensitive PEs: One that is compromised of
1035  * single PCI bus. Another one that contains the primary PCI bus and its
1036  * subordinate PCI devices and buses. The second type of PE is normally
1037  * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
1038  */
1039 static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
1040 {
1041 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
1042 	struct pnv_ioda_pe *pe = NULL;
1043 	unsigned int pe_num;
1044 
1045 	/*
1046 	 * In partial hotplug case, the PE instance might be still alive.
1047 	 * We should reuse it instead of allocating a new one.
1048 	 */
1049 	pe_num = phb->ioda.pe_rmap[bus->number << 8];
1050 	if (WARN_ON(pe_num != IODA_INVALID_PE)) {
1051 		pe = &phb->ioda.pe_array[pe_num];
1052 		return NULL;
1053 	}
1054 
1055 	/* PE number for root bus should have been reserved */
1056 	if (pci_is_root_bus(bus))
1057 		pe = &phb->ioda.pe_array[phb->ioda.root_pe_idx];
1058 
1059 	/* Check if PE is determined by M64 */
1060 	if (!pe)
1061 		pe = pnv_ioda_pick_m64_pe(bus, all);
1062 
1063 	/* The PE number isn't pinned by M64 */
1064 	if (!pe)
1065 		pe = pnv_ioda_alloc_pe(phb, 1);
1066 
1067 	if (!pe) {
1068 		pr_warn("%s: Not enough PE# available for PCI bus %04x:%02x\n",
1069 			__func__, pci_domain_nr(bus), bus->number);
1070 		return NULL;
1071 	}
1072 
1073 	pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
1074 	pe->pbus = bus;
1075 	pe->pdev = NULL;
1076 	pe->mve_number = -1;
1077 	pe->rid = bus->busn_res.start << 8;
1078 
1079 	if (all)
1080 		pe_info(pe, "Secondary bus %pad..%pad associated with PE#%x\n",
1081 			&bus->busn_res.start, &bus->busn_res.end,
1082 			pe->pe_number);
1083 	else
1084 		pe_info(pe, "Secondary bus %pad associated with PE#%x\n",
1085 			&bus->busn_res.start, pe->pe_number);
1086 
1087 	if (pnv_ioda_configure_pe(phb, pe)) {
1088 		/* XXX What do we do here ? */
1089 		pnv_ioda_free_pe(pe);
1090 		pe->pbus = NULL;
1091 		return NULL;
1092 	}
1093 
1094 	/* Put PE to the list */
1095 	list_add_tail(&pe->list, &phb->ioda.pe_list);
1096 
1097 	return pe;
1098 }
1099 
1100 static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb,
1101 				       struct pnv_ioda_pe *pe);
1102 
1103 static void pnv_pci_ioda_dma_dev_setup(struct pci_dev *pdev)
1104 {
1105 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
1106 	struct pci_dn *pdn = pci_get_pdn(pdev);
1107 	struct pnv_ioda_pe *pe;
1108 
1109 	/* Check if the BDFN for this device is associated with a PE yet */
1110 	pe = pnv_pci_bdfn_to_pe(phb, pdev->devfn | (pdev->bus->number << 8));
1111 	if (!pe) {
1112 		/* VF PEs should be pre-configured in pnv_pci_sriov_enable() */
1113 		if (WARN_ON(pdev->is_virtfn))
1114 			return;
1115 
1116 		pnv_pci_configure_bus(pdev->bus);
1117 		pe = pnv_pci_bdfn_to_pe(phb, pdev->devfn | (pdev->bus->number << 8));
1118 		pci_info(pdev, "Configured PE#%x\n", pe ? pe->pe_number : 0xfffff);
1119 
1120 
1121 		/*
1122 		 * If we can't setup the IODA PE something has gone horribly
1123 		 * wrong and we can't enable DMA for the device.
1124 		 */
1125 		if (WARN_ON(!pe))
1126 			return;
1127 	} else {
1128 		pci_info(pdev, "Added to existing PE#%x\n", pe->pe_number);
1129 	}
1130 
1131 	/*
1132 	 * We assume that bridges *probably* don't need to do any DMA so we can
1133 	 * skip allocating a TCE table, etc unless we get a non-bridge device.
1134 	 */
1135 	if (!pe->dma_setup_done && !pci_is_bridge(pdev)) {
1136 		switch (phb->type) {
1137 		case PNV_PHB_IODA1:
1138 			pnv_pci_ioda1_setup_dma_pe(phb, pe);
1139 			break;
1140 		case PNV_PHB_IODA2:
1141 			pnv_pci_ioda2_setup_dma_pe(phb, pe);
1142 			break;
1143 		default:
1144 			pr_warn("%s: No DMA for PHB#%x (type %d)\n",
1145 				__func__, phb->hose->global_number, phb->type);
1146 		}
1147 	}
1148 
1149 	if (pdn)
1150 		pdn->pe_number = pe->pe_number;
1151 	pe->device_count++;
1152 
1153 	WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1154 	pdev->dev.archdata.dma_offset = pe->tce_bypass_base;
1155 	set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1156 
1157 	/* PEs with a DMA weight of zero won't have a group */
1158 	if (pe->table_group.group)
1159 		iommu_add_device(&pe->table_group, &pdev->dev);
1160 }
1161 
1162 /*
1163  * Reconfigure TVE#0 to be usable as 64-bit DMA space.
1164  *
1165  * The first 4GB of virtual memory for a PE is reserved for 32-bit accesses.
1166  * Devices can only access more than that if bit 59 of the PCI address is set
1167  * by hardware, which indicates TVE#1 should be used instead of TVE#0.
1168  * Many PCI devices are not capable of addressing that many bits, and as a
1169  * result are limited to the 4GB of virtual memory made available to 32-bit
1170  * devices in TVE#0.
1171  *
1172  * In order to work around this, reconfigure TVE#0 to be suitable for 64-bit
1173  * devices by configuring the virtual memory past the first 4GB inaccessible
1174  * by 64-bit DMAs.  This should only be used by devices that want more than
1175  * 4GB, and only on PEs that have no 32-bit devices.
1176  *
1177  * Currently this will only work on PHB3 (POWER8).
1178  */
1179 static int pnv_pci_ioda_dma_64bit_bypass(struct pnv_ioda_pe *pe)
1180 {
1181 	u64 window_size, table_size, tce_count, addr;
1182 	struct page *table_pages;
1183 	u64 tce_order = 28; /* 256MB TCEs */
1184 	__be64 *tces;
1185 	s64 rc;
1186 
1187 	/*
1188 	 * Window size needs to be a power of two, but needs to account for
1189 	 * shifting memory by the 4GB offset required to skip 32bit space.
1190 	 */
1191 	window_size = roundup_pow_of_two(memory_hotplug_max() + (1ULL << 32));
1192 	tce_count = window_size >> tce_order;
1193 	table_size = tce_count << 3;
1194 
1195 	if (table_size < PAGE_SIZE)
1196 		table_size = PAGE_SIZE;
1197 
1198 	table_pages = alloc_pages_node(pe->phb->hose->node, GFP_KERNEL,
1199 				       get_order(table_size));
1200 	if (!table_pages)
1201 		goto err;
1202 
1203 	tces = page_address(table_pages);
1204 	if (!tces)
1205 		goto err;
1206 
1207 	memset(tces, 0, table_size);
1208 
1209 	for (addr = 0; addr < memory_hotplug_max(); addr += (1 << tce_order)) {
1210 		tces[(addr + (1ULL << 32)) >> tce_order] =
1211 			cpu_to_be64(addr | TCE_PCI_READ | TCE_PCI_WRITE);
1212 	}
1213 
1214 	rc = opal_pci_map_pe_dma_window(pe->phb->opal_id,
1215 					pe->pe_number,
1216 					/* reconfigure window 0 */
1217 					(pe->pe_number << 1) + 0,
1218 					1,
1219 					__pa(tces),
1220 					table_size,
1221 					1 << tce_order);
1222 	if (rc == OPAL_SUCCESS) {
1223 		pe_info(pe, "Using 64-bit DMA iommu bypass (through TVE#0)\n");
1224 		return 0;
1225 	}
1226 err:
1227 	pe_err(pe, "Error configuring 64-bit DMA bypass\n");
1228 	return -EIO;
1229 }
1230 
1231 static bool pnv_pci_ioda_iommu_bypass_supported(struct pci_dev *pdev,
1232 		u64 dma_mask)
1233 {
1234 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
1235 	struct pci_dn *pdn = pci_get_pdn(pdev);
1236 	struct pnv_ioda_pe *pe;
1237 
1238 	if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1239 		return false;
1240 
1241 	pe = &phb->ioda.pe_array[pdn->pe_number];
1242 	if (pe->tce_bypass_enabled) {
1243 		u64 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1244 		if (dma_mask >= top)
1245 			return true;
1246 	}
1247 
1248 	/*
1249 	 * If the device can't set the TCE bypass bit but still wants
1250 	 * to access 4GB or more, on PHB3 we can reconfigure TVE#0 to
1251 	 * bypass the 32-bit region and be usable for 64-bit DMAs.
1252 	 * The device needs to be able to address all of this space.
1253 	 */
1254 	if (dma_mask >> 32 &&
1255 	    dma_mask > (memory_hotplug_max() + (1ULL << 32)) &&
1256 	    /* pe->pdev should be set if it's a single device, pe->pbus if not */
1257 	    (pe->device_count == 1 || !pe->pbus) &&
1258 	    phb->model == PNV_PHB_MODEL_PHB3) {
1259 		/* Configure the bypass mode */
1260 		s64 rc = pnv_pci_ioda_dma_64bit_bypass(pe);
1261 		if (rc)
1262 			return false;
1263 		/* 4GB offset bypasses 32-bit space */
1264 		pdev->dev.archdata.dma_offset = (1ULL << 32);
1265 		return true;
1266 	}
1267 
1268 	return false;
1269 }
1270 
1271 static inline __be64 __iomem *pnv_ioda_get_inval_reg(struct pnv_phb *phb)
1272 {
1273 	return phb->regs + 0x210;
1274 }
1275 
1276 static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl,
1277 		unsigned long index, unsigned long npages)
1278 {
1279 	struct iommu_table_group_link *tgl = list_first_entry_or_null(
1280 			&tbl->it_group_list, struct iommu_table_group_link,
1281 			next);
1282 	struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1283 			struct pnv_ioda_pe, table_group);
1284 	__be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb);
1285 	unsigned long start, end, inc;
1286 
1287 	start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
1288 	end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
1289 			npages - 1);
1290 
1291 	/* p7ioc-style invalidation, 2 TCEs per write */
1292 	start |= (1ull << 63);
1293 	end |= (1ull << 63);
1294 	inc = 16;
1295         end |= inc - 1;	/* round up end to be different than start */
1296 
1297         mb(); /* Ensure above stores are visible */
1298         while (start <= end) {
1299 		__raw_writeq_be(start, invalidate);
1300                 start += inc;
1301         }
1302 
1303 	/*
1304 	 * The iommu layer will do another mb() for us on build()
1305 	 * and we don't care on free()
1306 	 */
1307 }
1308 
1309 static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
1310 		long npages, unsigned long uaddr,
1311 		enum dma_data_direction direction,
1312 		unsigned long attrs)
1313 {
1314 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1315 			attrs);
1316 
1317 	if (!ret)
1318 		pnv_pci_p7ioc_tce_invalidate(tbl, index, npages);
1319 
1320 	return ret;
1321 }
1322 
1323 #ifdef CONFIG_IOMMU_API
1324 /* Common for IODA1 and IODA2 */
1325 static int pnv_ioda_tce_xchg_no_kill(struct iommu_table *tbl, long index,
1326 		unsigned long *hpa, enum dma_data_direction *direction)
1327 {
1328 	return pnv_tce_xchg(tbl, index, hpa, direction);
1329 }
1330 #endif
1331 
1332 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
1333 		long npages)
1334 {
1335 	pnv_tce_free(tbl, index, npages);
1336 
1337 	pnv_pci_p7ioc_tce_invalidate(tbl, index, npages);
1338 }
1339 
1340 static struct iommu_table_ops pnv_ioda1_iommu_ops = {
1341 	.set = pnv_ioda1_tce_build,
1342 #ifdef CONFIG_IOMMU_API
1343 	.xchg_no_kill = pnv_ioda_tce_xchg_no_kill,
1344 	.tce_kill = pnv_pci_p7ioc_tce_invalidate,
1345 	.useraddrptr = pnv_tce_useraddrptr,
1346 #endif
1347 	.clear = pnv_ioda1_tce_free,
1348 	.get = pnv_tce_get,
1349 };
1350 
1351 #define PHB3_TCE_KILL_INVAL_ALL		PPC_BIT(0)
1352 #define PHB3_TCE_KILL_INVAL_PE		PPC_BIT(1)
1353 #define PHB3_TCE_KILL_INVAL_ONE		PPC_BIT(2)
1354 
1355 static inline void pnv_pci_phb3_tce_invalidate_pe(struct pnv_ioda_pe *pe)
1356 {
1357 	/* 01xb - invalidate TCEs that match the specified PE# */
1358 	__be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb);
1359 	unsigned long val = PHB3_TCE_KILL_INVAL_PE | (pe->pe_number & 0xFF);
1360 
1361 	mb(); /* Ensure above stores are visible */
1362 	__raw_writeq_be(val, invalidate);
1363 }
1364 
1365 static void pnv_pci_phb3_tce_invalidate(struct pnv_ioda_pe *pe,
1366 					unsigned shift, unsigned long index,
1367 					unsigned long npages)
1368 {
1369 	__be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb);
1370 	unsigned long start, end, inc;
1371 
1372 	/* We'll invalidate DMA address in PE scope */
1373 	start = PHB3_TCE_KILL_INVAL_ONE;
1374 	start |= (pe->pe_number & 0xFF);
1375 	end = start;
1376 
1377 	/* Figure out the start, end and step */
1378 	start |= (index << shift);
1379 	end |= ((index + npages - 1) << shift);
1380 	inc = (0x1ull << shift);
1381 	mb();
1382 
1383 	while (start <= end) {
1384 		__raw_writeq_be(start, invalidate);
1385 		start += inc;
1386 	}
1387 }
1388 
1389 static inline void pnv_pci_ioda2_tce_invalidate_pe(struct pnv_ioda_pe *pe)
1390 {
1391 	struct pnv_phb *phb = pe->phb;
1392 
1393 	if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
1394 		pnv_pci_phb3_tce_invalidate_pe(pe);
1395 	else
1396 		opal_pci_tce_kill(phb->opal_id, OPAL_PCI_TCE_KILL_PE,
1397 				  pe->pe_number, 0, 0, 0);
1398 }
1399 
1400 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1401 		unsigned long index, unsigned long npages)
1402 {
1403 	struct iommu_table_group_link *tgl;
1404 
1405 	list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {
1406 		struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1407 				struct pnv_ioda_pe, table_group);
1408 		struct pnv_phb *phb = pe->phb;
1409 		unsigned int shift = tbl->it_page_shift;
1410 
1411 		if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
1412 			pnv_pci_phb3_tce_invalidate(pe, shift,
1413 						    index, npages);
1414 		else
1415 			opal_pci_tce_kill(phb->opal_id,
1416 					  OPAL_PCI_TCE_KILL_PAGES,
1417 					  pe->pe_number, 1u << shift,
1418 					  index << shift, npages);
1419 	}
1420 }
1421 
1422 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1423 		long npages, unsigned long uaddr,
1424 		enum dma_data_direction direction,
1425 		unsigned long attrs)
1426 {
1427 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1428 			attrs);
1429 
1430 	if (!ret)
1431 		pnv_pci_ioda2_tce_invalidate(tbl, index, npages);
1432 
1433 	return ret;
1434 }
1435 
1436 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1437 		long npages)
1438 {
1439 	pnv_tce_free(tbl, index, npages);
1440 
1441 	pnv_pci_ioda2_tce_invalidate(tbl, index, npages);
1442 }
1443 
1444 static struct iommu_table_ops pnv_ioda2_iommu_ops = {
1445 	.set = pnv_ioda2_tce_build,
1446 #ifdef CONFIG_IOMMU_API
1447 	.xchg_no_kill = pnv_ioda_tce_xchg_no_kill,
1448 	.tce_kill = pnv_pci_ioda2_tce_invalidate,
1449 	.useraddrptr = pnv_tce_useraddrptr,
1450 #endif
1451 	.clear = pnv_ioda2_tce_free,
1452 	.get = pnv_tce_get,
1453 	.free = pnv_pci_ioda2_table_free_pages,
1454 };
1455 
1456 static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data)
1457 {
1458 	unsigned int *weight = (unsigned int *)data;
1459 
1460 	/* This is quite simplistic. The "base" weight of a device
1461 	 * is 10. 0 means no DMA is to be accounted for it.
1462 	 */
1463 	if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
1464 		return 0;
1465 
1466 	if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
1467 	    dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
1468 	    dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1469 		*weight += 3;
1470 	else if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
1471 		*weight += 15;
1472 	else
1473 		*weight += 10;
1474 
1475 	return 0;
1476 }
1477 
1478 static unsigned int pnv_pci_ioda_pe_dma_weight(struct pnv_ioda_pe *pe)
1479 {
1480 	unsigned int weight = 0;
1481 
1482 	/* SRIOV VF has same DMA32 weight as its PF */
1483 #ifdef CONFIG_PCI_IOV
1484 	if ((pe->flags & PNV_IODA_PE_VF) && pe->parent_dev) {
1485 		pnv_pci_ioda_dev_dma_weight(pe->parent_dev, &weight);
1486 		return weight;
1487 	}
1488 #endif
1489 
1490 	if ((pe->flags & PNV_IODA_PE_DEV) && pe->pdev) {
1491 		pnv_pci_ioda_dev_dma_weight(pe->pdev, &weight);
1492 	} else if ((pe->flags & PNV_IODA_PE_BUS) && pe->pbus) {
1493 		struct pci_dev *pdev;
1494 
1495 		list_for_each_entry(pdev, &pe->pbus->devices, bus_list)
1496 			pnv_pci_ioda_dev_dma_weight(pdev, &weight);
1497 	} else if ((pe->flags & PNV_IODA_PE_BUS_ALL) && pe->pbus) {
1498 		pci_walk_bus(pe->pbus, pnv_pci_ioda_dev_dma_weight, &weight);
1499 	}
1500 
1501 	return weight;
1502 }
1503 
1504 static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb,
1505 				       struct pnv_ioda_pe *pe)
1506 {
1507 
1508 	struct page *tce_mem = NULL;
1509 	struct iommu_table *tbl;
1510 	unsigned int weight, total_weight = 0;
1511 	unsigned int tce32_segsz, base, segs, avail, i;
1512 	int64_t rc;
1513 	void *addr;
1514 
1515 	/* XXX FIXME: Handle 64-bit only DMA devices */
1516 	/* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
1517 	/* XXX FIXME: Allocate multi-level tables on PHB3 */
1518 	weight = pnv_pci_ioda_pe_dma_weight(pe);
1519 	if (!weight)
1520 		return;
1521 
1522 	pci_walk_bus(phb->hose->bus, pnv_pci_ioda_dev_dma_weight,
1523 		     &total_weight);
1524 	segs = (weight * phb->ioda.dma32_count) / total_weight;
1525 	if (!segs)
1526 		segs = 1;
1527 
1528 	/*
1529 	 * Allocate contiguous DMA32 segments. We begin with the expected
1530 	 * number of segments. With one more attempt, the number of DMA32
1531 	 * segments to be allocated is decreased by one until one segment
1532 	 * is allocated successfully.
1533 	 */
1534 	do {
1535 		for (base = 0; base <= phb->ioda.dma32_count - segs; base++) {
1536 			for (avail = 0, i = base; i < base + segs; i++) {
1537 				if (phb->ioda.dma32_segmap[i] ==
1538 				    IODA_INVALID_PE)
1539 					avail++;
1540 			}
1541 
1542 			if (avail == segs)
1543 				goto found;
1544 		}
1545 	} while (--segs);
1546 
1547 	if (!segs) {
1548 		pe_warn(pe, "No available DMA32 segments\n");
1549 		return;
1550 	}
1551 
1552 found:
1553 	tbl = pnv_pci_table_alloc(phb->hose->node);
1554 	if (WARN_ON(!tbl))
1555 		return;
1556 
1557 	iommu_register_group(&pe->table_group, phb->hose->global_number,
1558 			pe->pe_number);
1559 	pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
1560 
1561 	/* Grab a 32-bit TCE table */
1562 	pe_info(pe, "DMA weight %d (%d), assigned (%d) %d DMA32 segments\n",
1563 		weight, total_weight, base, segs);
1564 	pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
1565 		base * PNV_IODA1_DMA32_SEGSIZE,
1566 		(base + segs) * PNV_IODA1_DMA32_SEGSIZE - 1);
1567 
1568 	/* XXX Currently, we allocate one big contiguous table for the
1569 	 * TCEs. We only really need one chunk per 256M of TCE space
1570 	 * (ie per segment) but that's an optimization for later, it
1571 	 * requires some added smarts with our get/put_tce implementation
1572 	 *
1573 	 * Each TCE page is 4KB in size and each TCE entry occupies 8
1574 	 * bytes
1575 	 */
1576 	tce32_segsz = PNV_IODA1_DMA32_SEGSIZE >> (IOMMU_PAGE_SHIFT_4K - 3);
1577 	tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
1578 				   get_order(tce32_segsz * segs));
1579 	if (!tce_mem) {
1580 		pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
1581 		goto fail;
1582 	}
1583 	addr = page_address(tce_mem);
1584 	memset(addr, 0, tce32_segsz * segs);
1585 
1586 	/* Configure HW */
1587 	for (i = 0; i < segs; i++) {
1588 		rc = opal_pci_map_pe_dma_window(phb->opal_id,
1589 					      pe->pe_number,
1590 					      base + i, 1,
1591 					      __pa(addr) + tce32_segsz * i,
1592 					      tce32_segsz, IOMMU_PAGE_SIZE_4K);
1593 		if (rc) {
1594 			pe_err(pe, " Failed to configure 32-bit TCE table, err %lld\n",
1595 			       rc);
1596 			goto fail;
1597 		}
1598 	}
1599 
1600 	/* Setup DMA32 segment mapping */
1601 	for (i = base; i < base + segs; i++)
1602 		phb->ioda.dma32_segmap[i] = pe->pe_number;
1603 
1604 	/* Setup linux iommu table */
1605 	pnv_pci_setup_iommu_table(tbl, addr, tce32_segsz * segs,
1606 				  base * PNV_IODA1_DMA32_SEGSIZE,
1607 				  IOMMU_PAGE_SHIFT_4K);
1608 
1609 	tbl->it_ops = &pnv_ioda1_iommu_ops;
1610 	pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
1611 	pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
1612 	tbl->it_index = (phb->hose->global_number << 16) | pe->pe_number;
1613 	if (!iommu_init_table(tbl, phb->hose->node, 0, 0))
1614 		panic("Failed to initialize iommu table");
1615 
1616 	pe->dma_setup_done = true;
1617 	return;
1618  fail:
1619 	/* XXX Failure: Try to fallback to 64-bit only ? */
1620 	if (tce_mem)
1621 		__free_pages(tce_mem, get_order(tce32_segsz * segs));
1622 	if (tbl) {
1623 		pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
1624 		iommu_tce_table_put(tbl);
1625 	}
1626 }
1627 
1628 static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1629 		int num, struct iommu_table *tbl)
1630 {
1631 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1632 			table_group);
1633 	struct pnv_phb *phb = pe->phb;
1634 	int64_t rc;
1635 	const unsigned long size = tbl->it_indirect_levels ?
1636 			tbl->it_level_size : tbl->it_size;
1637 	const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1638 	const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1639 
1640 	pe_info(pe, "Setting up window#%d %llx..%llx pg=%lx\n",
1641 		num, start_addr, start_addr + win_size - 1,
1642 		IOMMU_PAGE_SIZE(tbl));
1643 
1644 	/*
1645 	 * Map TCE table through TVT. The TVE index is the PE number
1646 	 * shifted by 1 bit for 32-bits DMA space.
1647 	 */
1648 	rc = opal_pci_map_pe_dma_window(phb->opal_id,
1649 			pe->pe_number,
1650 			(pe->pe_number << 1) + num,
1651 			tbl->it_indirect_levels + 1,
1652 			__pa(tbl->it_base),
1653 			size << 3,
1654 			IOMMU_PAGE_SIZE(tbl));
1655 	if (rc) {
1656 		pe_err(pe, "Failed to configure TCE table, err %lld\n", rc);
1657 		return rc;
1658 	}
1659 
1660 	pnv_pci_link_table_and_group(phb->hose->node, num,
1661 			tbl, &pe->table_group);
1662 	pnv_pci_ioda2_tce_invalidate_pe(pe);
1663 
1664 	return 0;
1665 }
1666 
1667 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
1668 {
1669 	uint16_t window_id = (pe->pe_number << 1 ) + 1;
1670 	int64_t rc;
1671 
1672 	pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
1673 	if (enable) {
1674 		phys_addr_t top = memblock_end_of_DRAM();
1675 
1676 		top = roundup_pow_of_two(top);
1677 		rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1678 						     pe->pe_number,
1679 						     window_id,
1680 						     pe->tce_bypass_base,
1681 						     top);
1682 	} else {
1683 		rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1684 						     pe->pe_number,
1685 						     window_id,
1686 						     pe->tce_bypass_base,
1687 						     0);
1688 	}
1689 	if (rc)
1690 		pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
1691 	else
1692 		pe->tce_bypass_enabled = enable;
1693 }
1694 
1695 static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
1696 		int num, __u32 page_shift, __u64 window_size, __u32 levels,
1697 		bool alloc_userspace_copy, struct iommu_table **ptbl)
1698 {
1699 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1700 			table_group);
1701 	int nid = pe->phb->hose->node;
1702 	__u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
1703 	long ret;
1704 	struct iommu_table *tbl;
1705 
1706 	tbl = pnv_pci_table_alloc(nid);
1707 	if (!tbl)
1708 		return -ENOMEM;
1709 
1710 	tbl->it_ops = &pnv_ioda2_iommu_ops;
1711 
1712 	ret = pnv_pci_ioda2_table_alloc_pages(nid,
1713 			bus_offset, page_shift, window_size,
1714 			levels, alloc_userspace_copy, tbl);
1715 	if (ret) {
1716 		iommu_tce_table_put(tbl);
1717 		return ret;
1718 	}
1719 
1720 	*ptbl = tbl;
1721 
1722 	return 0;
1723 }
1724 
1725 static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
1726 {
1727 	struct iommu_table *tbl = NULL;
1728 	long rc;
1729 	unsigned long res_start, res_end;
1730 
1731 	/*
1732 	 * crashkernel= specifies the kdump kernel's maximum memory at
1733 	 * some offset and there is no guaranteed the result is a power
1734 	 * of 2, which will cause errors later.
1735 	 */
1736 	const u64 max_memory = __rounddown_pow_of_two(memory_hotplug_max());
1737 
1738 	/*
1739 	 * In memory constrained environments, e.g. kdump kernel, the
1740 	 * DMA window can be larger than available memory, which will
1741 	 * cause errors later.
1742 	 */
1743 	const u64 maxblock = 1UL << (PAGE_SHIFT + MAX_ORDER - 1);
1744 
1745 	/*
1746 	 * We create the default window as big as we can. The constraint is
1747 	 * the max order of allocation possible. The TCE table is likely to
1748 	 * end up being multilevel and with on-demand allocation in place,
1749 	 * the initial use is not going to be huge as the default window aims
1750 	 * to support crippled devices (i.e. not fully 64bit DMAble) only.
1751 	 */
1752 	/* iommu_table::it_map uses 1 bit per IOMMU page, hence 8 */
1753 	const u64 window_size = min((maxblock * 8) << PAGE_SHIFT, max_memory);
1754 	/* Each TCE level cannot exceed maxblock so go multilevel if needed */
1755 	unsigned long tces_order = ilog2(window_size >> PAGE_SHIFT);
1756 	unsigned long tcelevel_order = ilog2(maxblock >> 3);
1757 	unsigned int levels = tces_order / tcelevel_order;
1758 
1759 	if (tces_order % tcelevel_order)
1760 		levels += 1;
1761 	/*
1762 	 * We try to stick to default levels (which is >1 at the moment) in
1763 	 * order to save memory by relying on on-demain TCE level allocation.
1764 	 */
1765 	levels = max_t(unsigned int, levels, POWERNV_IOMMU_DEFAULT_LEVELS);
1766 
1767 	rc = pnv_pci_ioda2_create_table(&pe->table_group, 0, PAGE_SHIFT,
1768 			window_size, levels, false, &tbl);
1769 	if (rc) {
1770 		pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
1771 				rc);
1772 		return rc;
1773 	}
1774 
1775 	/* We use top part of 32bit space for MMIO so exclude it from DMA */
1776 	res_start = 0;
1777 	res_end = 0;
1778 	if (window_size > pe->phb->ioda.m32_pci_base) {
1779 		res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift;
1780 		res_end = min(window_size, SZ_4G) >> tbl->it_page_shift;
1781 	}
1782 
1783 	tbl->it_index = (pe->phb->hose->global_number << 16) | pe->pe_number;
1784 	if (iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end))
1785 		rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
1786 	else
1787 		rc = -ENOMEM;
1788 	if (rc) {
1789 		pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n", rc);
1790 		iommu_tce_table_put(tbl);
1791 		tbl = NULL; /* This clears iommu_table_base below */
1792 	}
1793 	if (!pnv_iommu_bypass_disabled)
1794 		pnv_pci_ioda2_set_bypass(pe, true);
1795 
1796 	/*
1797 	 * Set table base for the case of IOMMU DMA use. Usually this is done
1798 	 * from dma_dev_setup() which is not called when a device is returned
1799 	 * from VFIO so do it here.
1800 	 */
1801 	if (pe->pdev)
1802 		set_iommu_table_base(&pe->pdev->dev, tbl);
1803 
1804 	return 0;
1805 }
1806 
1807 static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1808 		int num)
1809 {
1810 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1811 			table_group);
1812 	struct pnv_phb *phb = pe->phb;
1813 	long ret;
1814 
1815 	pe_info(pe, "Removing DMA window #%d\n", num);
1816 
1817 	ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1818 			(pe->pe_number << 1) + num,
1819 			0/* levels */, 0/* table address */,
1820 			0/* table size */, 0/* page size */);
1821 	if (ret)
1822 		pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
1823 	else
1824 		pnv_pci_ioda2_tce_invalidate_pe(pe);
1825 
1826 	pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
1827 
1828 	return ret;
1829 }
1830 
1831 #ifdef CONFIG_IOMMU_API
1832 unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
1833 		__u64 window_size, __u32 levels)
1834 {
1835 	unsigned long bytes = 0;
1836 	const unsigned window_shift = ilog2(window_size);
1837 	unsigned entries_shift = window_shift - page_shift;
1838 	unsigned table_shift = entries_shift + 3;
1839 	unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
1840 	unsigned long direct_table_size;
1841 
1842 	if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
1843 			!is_power_of_2(window_size))
1844 		return 0;
1845 
1846 	/* Calculate a direct table size from window_size and levels */
1847 	entries_shift = (entries_shift + levels - 1) / levels;
1848 	table_shift = entries_shift + 3;
1849 	table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
1850 	direct_table_size =  1UL << table_shift;
1851 
1852 	for ( ; levels; --levels) {
1853 		bytes += ALIGN(tce_table_size, direct_table_size);
1854 
1855 		tce_table_size /= direct_table_size;
1856 		tce_table_size <<= 3;
1857 		tce_table_size = max_t(unsigned long,
1858 				tce_table_size, direct_table_size);
1859 	}
1860 
1861 	return bytes + bytes; /* one for HW table, one for userspace copy */
1862 }
1863 
1864 static long pnv_pci_ioda2_create_table_userspace(
1865 		struct iommu_table_group *table_group,
1866 		int num, __u32 page_shift, __u64 window_size, __u32 levels,
1867 		struct iommu_table **ptbl)
1868 {
1869 	long ret = pnv_pci_ioda2_create_table(table_group,
1870 			num, page_shift, window_size, levels, true, ptbl);
1871 
1872 	if (!ret)
1873 		(*ptbl)->it_allocated_size = pnv_pci_ioda2_get_table_size(
1874 				page_shift, window_size, levels);
1875 	return ret;
1876 }
1877 
1878 static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
1879 {
1880 	struct pci_dev *dev;
1881 
1882 	list_for_each_entry(dev, &bus->devices, bus_list) {
1883 		set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
1884 		dev->dev.archdata.dma_offset = pe->tce_bypass_base;
1885 
1886 		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1887 			pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1888 	}
1889 }
1890 
1891 static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
1892 {
1893 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1894 						table_group);
1895 	/* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
1896 	struct iommu_table *tbl = pe->table_group.tables[0];
1897 
1898 	pnv_pci_ioda2_set_bypass(pe, false);
1899 	pnv_pci_ioda2_unset_window(&pe->table_group, 0);
1900 	if (pe->pbus)
1901 		pnv_ioda_setup_bus_dma(pe, pe->pbus);
1902 	else if (pe->pdev)
1903 		set_iommu_table_base(&pe->pdev->dev, NULL);
1904 	iommu_tce_table_put(tbl);
1905 }
1906 
1907 static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
1908 {
1909 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1910 						table_group);
1911 
1912 	pnv_pci_ioda2_setup_default_config(pe);
1913 	if (pe->pbus)
1914 		pnv_ioda_setup_bus_dma(pe, pe->pbus);
1915 }
1916 
1917 static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
1918 	.get_table_size = pnv_pci_ioda2_get_table_size,
1919 	.create_table = pnv_pci_ioda2_create_table_userspace,
1920 	.set_window = pnv_pci_ioda2_set_window,
1921 	.unset_window = pnv_pci_ioda2_unset_window,
1922 	.take_ownership = pnv_ioda2_take_ownership,
1923 	.release_ownership = pnv_ioda2_release_ownership,
1924 };
1925 #endif
1926 
1927 void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1928 				struct pnv_ioda_pe *pe)
1929 {
1930 	int64_t rc;
1931 
1932 	/* TVE #1 is selected by PCI address bit 59 */
1933 	pe->tce_bypass_base = 1ull << 59;
1934 
1935 	/* The PE will reserve all possible 32-bits space */
1936 	pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
1937 		phb->ioda.m32_pci_base);
1938 
1939 	/* Setup linux iommu table */
1940 	pe->table_group.tce32_start = 0;
1941 	pe->table_group.tce32_size = phb->ioda.m32_pci_base;
1942 	pe->table_group.max_dynamic_windows_supported =
1943 			IOMMU_TABLE_GROUP_MAX_TABLES;
1944 	pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
1945 	pe->table_group.pgsizes = pnv_ioda_parse_tce_sizes(phb);
1946 
1947 	rc = pnv_pci_ioda2_setup_default_config(pe);
1948 	if (rc)
1949 		return;
1950 
1951 #ifdef CONFIG_IOMMU_API
1952 	pe->table_group.ops = &pnv_pci_ioda2_ops;
1953 	iommu_register_group(&pe->table_group, phb->hose->global_number,
1954 			     pe->pe_number);
1955 #endif
1956 	pe->dma_setup_done = true;
1957 }
1958 
1959 /*
1960  * Called from KVM in real mode to EOI passthru interrupts. The ICP
1961  * EOI is handled directly in KVM in kvmppc_deliver_irq_passthru().
1962  *
1963  * The IRQ data is mapped in the PCI-MSI domain and the EOI OPAL call
1964  * needs an HW IRQ number mapped in the XICS IRQ domain. The HW IRQ
1965  * numbers of the in-the-middle MSI domain are vector numbers and it's
1966  * good enough for OPAL. Use that.
1967  */
1968 int64_t pnv_opal_pci_msi_eoi(struct irq_data *d)
1969 {
1970 	struct pci_controller *hose = irq_data_get_irq_chip_data(d->parent_data);
1971 	struct pnv_phb *phb = hose->private_data;
1972 
1973 	return opal_pci_msi_eoi(phb->opal_id, d->parent_data->hwirq);
1974 }
1975 
1976 /*
1977  * The IRQ data is mapped in the XICS domain, with OPAL HW IRQ numbers
1978  */
1979 static void pnv_ioda2_msi_eoi(struct irq_data *d)
1980 {
1981 	int64_t rc;
1982 	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
1983 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
1984 	struct pnv_phb *phb = hose->private_data;
1985 
1986 	rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
1987 	WARN_ON_ONCE(rc);
1988 
1989 	icp_native_eoi(d);
1990 }
1991 
1992 /* P8/CXL only */
1993 void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
1994 {
1995 	struct irq_data *idata;
1996 	struct irq_chip *ichip;
1997 
1998 	/* The MSI EOI OPAL call is only needed on PHB3 */
1999 	if (phb->model != PNV_PHB_MODEL_PHB3)
2000 		return;
2001 
2002 	if (!phb->ioda.irq_chip_init) {
2003 		/*
2004 		 * First time we setup an MSI IRQ, we need to setup the
2005 		 * corresponding IRQ chip to route correctly.
2006 		 */
2007 		idata = irq_get_irq_data(virq);
2008 		ichip = irq_data_get_irq_chip(idata);
2009 		phb->ioda.irq_chip_init = 1;
2010 		phb->ioda.irq_chip = *ichip;
2011 		phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
2012 	}
2013 	irq_set_chip(virq, &phb->ioda.irq_chip);
2014 	irq_set_chip_data(virq, phb->hose);
2015 }
2016 
2017 static struct irq_chip pnv_pci_msi_irq_chip;
2018 
2019 /*
2020  * Returns true iff chip is something that we could call
2021  * pnv_opal_pci_msi_eoi for.
2022  */
2023 bool is_pnv_opal_msi(struct irq_chip *chip)
2024 {
2025 	return chip == &pnv_pci_msi_irq_chip;
2026 }
2027 EXPORT_SYMBOL_GPL(is_pnv_opal_msi);
2028 
2029 static int __pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
2030 				    unsigned int xive_num,
2031 				    unsigned int is_64, struct msi_msg *msg)
2032 {
2033 	struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
2034 	__be32 data;
2035 	int rc;
2036 
2037 	dev_dbg(&dev->dev, "%s: setup %s-bit MSI for vector #%d\n", __func__,
2038 		is_64 ? "64" : "32", xive_num);
2039 
2040 	/* No PE assigned ? bail out ... no MSI for you ! */
2041 	if (pe == NULL)
2042 		return -ENXIO;
2043 
2044 	/* Check if we have an MVE */
2045 	if (pe->mve_number < 0)
2046 		return -ENXIO;
2047 
2048 	/* Force 32-bit MSI on some broken devices */
2049 	if (dev->no_64bit_msi)
2050 		is_64 = 0;
2051 
2052 	/* Assign XIVE to PE */
2053 	rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
2054 	if (rc) {
2055 		pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
2056 			pci_name(dev), rc, xive_num);
2057 		return -EIO;
2058 	}
2059 
2060 	if (is_64) {
2061 		__be64 addr64;
2062 
2063 		rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
2064 				     &addr64, &data);
2065 		if (rc) {
2066 			pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
2067 				pci_name(dev), rc);
2068 			return -EIO;
2069 		}
2070 		msg->address_hi = be64_to_cpu(addr64) >> 32;
2071 		msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
2072 	} else {
2073 		__be32 addr32;
2074 
2075 		rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
2076 				     &addr32, &data);
2077 		if (rc) {
2078 			pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
2079 				pci_name(dev), rc);
2080 			return -EIO;
2081 		}
2082 		msg->address_hi = 0;
2083 		msg->address_lo = be32_to_cpu(addr32);
2084 	}
2085 	msg->data = be32_to_cpu(data);
2086 
2087 	return 0;
2088 }
2089 
2090 /*
2091  * The msi_free() op is called before irq_domain_free_irqs_top() when
2092  * the handler data is still available. Use that to clear the XIVE
2093  * controller.
2094  */
2095 static void pnv_msi_ops_msi_free(struct irq_domain *domain,
2096 				 struct msi_domain_info *info,
2097 				 unsigned int irq)
2098 {
2099 	if (xive_enabled())
2100 		xive_irq_free_data(irq);
2101 }
2102 
2103 static struct msi_domain_ops pnv_pci_msi_domain_ops = {
2104 	.msi_free	= pnv_msi_ops_msi_free,
2105 };
2106 
2107 static void pnv_msi_shutdown(struct irq_data *d)
2108 {
2109 	d = d->parent_data;
2110 	if (d->chip->irq_shutdown)
2111 		d->chip->irq_shutdown(d);
2112 }
2113 
2114 static void pnv_msi_mask(struct irq_data *d)
2115 {
2116 	pci_msi_mask_irq(d);
2117 	irq_chip_mask_parent(d);
2118 }
2119 
2120 static void pnv_msi_unmask(struct irq_data *d)
2121 {
2122 	pci_msi_unmask_irq(d);
2123 	irq_chip_unmask_parent(d);
2124 }
2125 
2126 static struct irq_chip pnv_pci_msi_irq_chip = {
2127 	.name		= "PNV-PCI-MSI",
2128 	.irq_shutdown	= pnv_msi_shutdown,
2129 	.irq_mask	= pnv_msi_mask,
2130 	.irq_unmask	= pnv_msi_unmask,
2131 	.irq_eoi	= irq_chip_eoi_parent,
2132 };
2133 
2134 static struct msi_domain_info pnv_msi_domain_info = {
2135 	.flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
2136 		  MSI_FLAG_MULTI_PCI_MSI  | MSI_FLAG_PCI_MSIX),
2137 	.ops   = &pnv_pci_msi_domain_ops,
2138 	.chip  = &pnv_pci_msi_irq_chip,
2139 };
2140 
2141 static void pnv_msi_compose_msg(struct irq_data *d, struct msi_msg *msg)
2142 {
2143 	struct msi_desc *entry = irq_data_get_msi_desc(d);
2144 	struct pci_dev *pdev = msi_desc_to_pci_dev(entry);
2145 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
2146 	struct pnv_phb *phb = hose->private_data;
2147 	int rc;
2148 
2149 	rc = __pnv_pci_ioda_msi_setup(phb, pdev, d->hwirq,
2150 				      entry->pci.msi_attrib.is_64, msg);
2151 	if (rc)
2152 		dev_err(&pdev->dev, "Failed to setup %s-bit MSI #%ld : %d\n",
2153 			entry->pci.msi_attrib.is_64 ? "64" : "32", d->hwirq, rc);
2154 }
2155 
2156 /*
2157  * The IRQ data is mapped in the MSI domain in which HW IRQ numbers
2158  * correspond to vector numbers.
2159  */
2160 static void pnv_msi_eoi(struct irq_data *d)
2161 {
2162 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
2163 	struct pnv_phb *phb = hose->private_data;
2164 
2165 	if (phb->model == PNV_PHB_MODEL_PHB3) {
2166 		/*
2167 		 * The EOI OPAL call takes an OPAL HW IRQ number but
2168 		 * since it is translated into a vector number in
2169 		 * OPAL, use that directly.
2170 		 */
2171 		WARN_ON_ONCE(opal_pci_msi_eoi(phb->opal_id, d->hwirq));
2172 	}
2173 
2174 	irq_chip_eoi_parent(d);
2175 }
2176 
2177 static struct irq_chip pnv_msi_irq_chip = {
2178 	.name			= "PNV-MSI",
2179 	.irq_shutdown		= pnv_msi_shutdown,
2180 	.irq_mask		= irq_chip_mask_parent,
2181 	.irq_unmask		= irq_chip_unmask_parent,
2182 	.irq_eoi		= pnv_msi_eoi,
2183 	.irq_set_affinity	= irq_chip_set_affinity_parent,
2184 	.irq_compose_msi_msg	= pnv_msi_compose_msg,
2185 };
2186 
2187 static int pnv_irq_parent_domain_alloc(struct irq_domain *domain,
2188 				       unsigned int virq, int hwirq)
2189 {
2190 	struct irq_fwspec parent_fwspec;
2191 	int ret;
2192 
2193 	parent_fwspec.fwnode = domain->parent->fwnode;
2194 	parent_fwspec.param_count = 2;
2195 	parent_fwspec.param[0] = hwirq;
2196 	parent_fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
2197 
2198 	ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &parent_fwspec);
2199 	if (ret)
2200 		return ret;
2201 
2202 	return 0;
2203 }
2204 
2205 static int pnv_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
2206 				unsigned int nr_irqs, void *arg)
2207 {
2208 	struct pci_controller *hose = domain->host_data;
2209 	struct pnv_phb *phb = hose->private_data;
2210 	msi_alloc_info_t *info = arg;
2211 	struct pci_dev *pdev = msi_desc_to_pci_dev(info->desc);
2212 	int hwirq;
2213 	int i, ret;
2214 
2215 	hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, nr_irqs);
2216 	if (hwirq < 0) {
2217 		dev_warn(&pdev->dev, "failed to find a free MSI\n");
2218 		return -ENOSPC;
2219 	}
2220 
2221 	dev_dbg(&pdev->dev, "%s bridge %pOF %d/%x #%d\n", __func__,
2222 		hose->dn, virq, hwirq, nr_irqs);
2223 
2224 	for (i = 0; i < nr_irqs; i++) {
2225 		ret = pnv_irq_parent_domain_alloc(domain, virq + i,
2226 						  phb->msi_base + hwirq + i);
2227 		if (ret)
2228 			goto out;
2229 
2230 		irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
2231 					      &pnv_msi_irq_chip, hose);
2232 	}
2233 
2234 	return 0;
2235 
2236 out:
2237 	irq_domain_free_irqs_parent(domain, virq, i - 1);
2238 	msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, nr_irqs);
2239 	return ret;
2240 }
2241 
2242 static void pnv_irq_domain_free(struct irq_domain *domain, unsigned int virq,
2243 				unsigned int nr_irqs)
2244 {
2245 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
2246 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
2247 	struct pnv_phb *phb = hose->private_data;
2248 
2249 	pr_debug("%s bridge %pOF %d/%lx #%d\n", __func__, hose->dn,
2250 		 virq, d->hwirq, nr_irqs);
2251 
2252 	msi_bitmap_free_hwirqs(&phb->msi_bmp, d->hwirq, nr_irqs);
2253 	/* XIVE domain is cleared through ->msi_free() */
2254 }
2255 
2256 static const struct irq_domain_ops pnv_irq_domain_ops = {
2257 	.alloc  = pnv_irq_domain_alloc,
2258 	.free   = pnv_irq_domain_free,
2259 };
2260 
2261 static int __init pnv_msi_allocate_domains(struct pci_controller *hose, unsigned int count)
2262 {
2263 	struct pnv_phb *phb = hose->private_data;
2264 	struct irq_domain *parent = irq_get_default_host();
2265 
2266 	hose->fwnode = irq_domain_alloc_named_id_fwnode("PNV-MSI", phb->opal_id);
2267 	if (!hose->fwnode)
2268 		return -ENOMEM;
2269 
2270 	hose->dev_domain = irq_domain_create_hierarchy(parent, 0, count,
2271 						       hose->fwnode,
2272 						       &pnv_irq_domain_ops, hose);
2273 	if (!hose->dev_domain) {
2274 		pr_err("PCI: failed to create IRQ domain bridge %pOF (domain %d)\n",
2275 		       hose->dn, hose->global_number);
2276 		irq_domain_free_fwnode(hose->fwnode);
2277 		return -ENOMEM;
2278 	}
2279 
2280 	hose->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(hose->dn),
2281 						     &pnv_msi_domain_info,
2282 						     hose->dev_domain);
2283 	if (!hose->msi_domain) {
2284 		pr_err("PCI: failed to create MSI IRQ domain bridge %pOF (domain %d)\n",
2285 		       hose->dn, hose->global_number);
2286 		irq_domain_free_fwnode(hose->fwnode);
2287 		irq_domain_remove(hose->dev_domain);
2288 		return -ENOMEM;
2289 	}
2290 
2291 	return 0;
2292 }
2293 
2294 static void __init pnv_pci_init_ioda_msis(struct pnv_phb *phb)
2295 {
2296 	unsigned int count;
2297 	const __be32 *prop = of_get_property(phb->hose->dn,
2298 					     "ibm,opal-msi-ranges", NULL);
2299 	if (!prop) {
2300 		/* BML Fallback */
2301 		prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
2302 	}
2303 	if (!prop)
2304 		return;
2305 
2306 	phb->msi_base = be32_to_cpup(prop);
2307 	count = be32_to_cpup(prop + 1);
2308 	if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
2309 		pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
2310 		       phb->hose->global_number);
2311 		return;
2312 	}
2313 
2314 	pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
2315 		count, phb->msi_base);
2316 
2317 	pnv_msi_allocate_domains(phb->hose, count);
2318 }
2319 
2320 static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
2321 				  struct resource *res)
2322 {
2323 	struct pnv_phb *phb = pe->phb;
2324 	struct pci_bus_region region;
2325 	int index;
2326 	int64_t rc;
2327 
2328 	if (!res || !res->flags || res->start > res->end)
2329 		return;
2330 
2331 	if (res->flags & IORESOURCE_IO) {
2332 		region.start = res->start - phb->ioda.io_pci_base;
2333 		region.end   = res->end - phb->ioda.io_pci_base;
2334 		index = region.start / phb->ioda.io_segsize;
2335 
2336 		while (index < phb->ioda.total_pe_num &&
2337 		       region.start <= region.end) {
2338 			phb->ioda.io_segmap[index] = pe->pe_number;
2339 			rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2340 				pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2341 			if (rc != OPAL_SUCCESS) {
2342 				pr_err("%s: Error %lld mapping IO segment#%d to PE#%x\n",
2343 				       __func__, rc, index, pe->pe_number);
2344 				break;
2345 			}
2346 
2347 			region.start += phb->ioda.io_segsize;
2348 			index++;
2349 		}
2350 	} else if ((res->flags & IORESOURCE_MEM) &&
2351 		   !pnv_pci_is_m64(phb, res)) {
2352 		region.start = res->start -
2353 			       phb->hose->mem_offset[0] -
2354 			       phb->ioda.m32_pci_base;
2355 		region.end   = res->end -
2356 			       phb->hose->mem_offset[0] -
2357 			       phb->ioda.m32_pci_base;
2358 		index = region.start / phb->ioda.m32_segsize;
2359 
2360 		while (index < phb->ioda.total_pe_num &&
2361 		       region.start <= region.end) {
2362 			phb->ioda.m32_segmap[index] = pe->pe_number;
2363 			rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2364 				pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2365 			if (rc != OPAL_SUCCESS) {
2366 				pr_err("%s: Error %lld mapping M32 segment#%d to PE#%x",
2367 				       __func__, rc, index, pe->pe_number);
2368 				break;
2369 			}
2370 
2371 			region.start += phb->ioda.m32_segsize;
2372 			index++;
2373 		}
2374 	}
2375 }
2376 
2377 /*
2378  * This function is supposed to be called on basis of PE from top
2379  * to bottom style. So the I/O or MMIO segment assigned to
2380  * parent PE could be overridden by its child PEs if necessary.
2381  */
2382 static void pnv_ioda_setup_pe_seg(struct pnv_ioda_pe *pe)
2383 {
2384 	struct pci_dev *pdev;
2385 	int i;
2386 
2387 	/*
2388 	 * NOTE: We only care PCI bus based PE for now. For PCI
2389 	 * device based PE, for example SRIOV sensitive VF should
2390 	 * be figured out later.
2391 	 */
2392 	BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2393 
2394 	list_for_each_entry(pdev, &pe->pbus->devices, bus_list) {
2395 		for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2396 			pnv_ioda_setup_pe_res(pe, &pdev->resource[i]);
2397 
2398 		/*
2399 		 * If the PE contains all subordinate PCI buses, the
2400 		 * windows of the child bridges should be mapped to
2401 		 * the PE as well.
2402 		 */
2403 		if (!(pe->flags & PNV_IODA_PE_BUS_ALL) || !pci_is_bridge(pdev))
2404 			continue;
2405 		for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
2406 			pnv_ioda_setup_pe_res(pe,
2407 				&pdev->resource[PCI_BRIDGE_RESOURCES + i]);
2408 	}
2409 }
2410 
2411 #ifdef CONFIG_DEBUG_FS
2412 static int pnv_pci_diag_data_set(void *data, u64 val)
2413 {
2414 	struct pnv_phb *phb = data;
2415 	s64 ret;
2416 
2417 	/* Retrieve the diag data from firmware */
2418 	ret = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag_data,
2419 					  phb->diag_data_size);
2420 	if (ret != OPAL_SUCCESS)
2421 		return -EIO;
2422 
2423 	/* Print the diag data to the kernel log */
2424 	pnv_pci_dump_phb_diag_data(phb->hose, phb->diag_data);
2425 	return 0;
2426 }
2427 
2428 DEFINE_DEBUGFS_ATTRIBUTE(pnv_pci_diag_data_fops, NULL, pnv_pci_diag_data_set,
2429 			 "%llu\n");
2430 
2431 static int pnv_pci_ioda_pe_dump(void *data, u64 val)
2432 {
2433 	struct pnv_phb *phb = data;
2434 	int pe_num;
2435 
2436 	for (pe_num = 0; pe_num < phb->ioda.total_pe_num; pe_num++) {
2437 		struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_num];
2438 
2439 		if (!test_bit(pe_num, phb->ioda.pe_alloc))
2440 			continue;
2441 
2442 		pe_warn(pe, "rid: %04x dev count: %2d flags: %s%s%s%s%s%s\n",
2443 			pe->rid, pe->device_count,
2444 			(pe->flags & PNV_IODA_PE_DEV) ? "dev " : "",
2445 			(pe->flags & PNV_IODA_PE_BUS) ? "bus " : "",
2446 			(pe->flags & PNV_IODA_PE_BUS_ALL) ? "all " : "",
2447 			(pe->flags & PNV_IODA_PE_MASTER) ? "master " : "",
2448 			(pe->flags & PNV_IODA_PE_SLAVE) ? "slave " : "",
2449 			(pe->flags & PNV_IODA_PE_VF) ? "vf " : "");
2450 	}
2451 
2452 	return 0;
2453 }
2454 
2455 DEFINE_DEBUGFS_ATTRIBUTE(pnv_pci_ioda_pe_dump_fops, NULL,
2456 			 pnv_pci_ioda_pe_dump, "%llu\n");
2457 
2458 #endif /* CONFIG_DEBUG_FS */
2459 
2460 static void pnv_pci_ioda_create_dbgfs(void)
2461 {
2462 #ifdef CONFIG_DEBUG_FS
2463 	struct pci_controller *hose, *tmp;
2464 	struct pnv_phb *phb;
2465 	char name[16];
2466 
2467 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2468 		phb = hose->private_data;
2469 
2470 		sprintf(name, "PCI%04x", hose->global_number);
2471 		phb->dbgfs = debugfs_create_dir(name, arch_debugfs_dir);
2472 
2473 		debugfs_create_file_unsafe("dump_diag_regs", 0200, phb->dbgfs,
2474 					   phb, &pnv_pci_diag_data_fops);
2475 		debugfs_create_file_unsafe("dump_ioda_pe_state", 0200, phb->dbgfs,
2476 					   phb, &pnv_pci_ioda_pe_dump_fops);
2477 	}
2478 #endif /* CONFIG_DEBUG_FS */
2479 }
2480 
2481 static void pnv_pci_enable_bridge(struct pci_bus *bus)
2482 {
2483 	struct pci_dev *dev = bus->self;
2484 	struct pci_bus *child;
2485 
2486 	/* Empty bus ? bail */
2487 	if (list_empty(&bus->devices))
2488 		return;
2489 
2490 	/*
2491 	 * If there's a bridge associated with that bus enable it. This works
2492 	 * around races in the generic code if the enabling is done during
2493 	 * parallel probing. This can be removed once those races have been
2494 	 * fixed.
2495 	 */
2496 	if (dev) {
2497 		int rc = pci_enable_device(dev);
2498 		if (rc)
2499 			pci_err(dev, "Error enabling bridge (%d)\n", rc);
2500 		pci_set_master(dev);
2501 	}
2502 
2503 	/* Perform the same to child busses */
2504 	list_for_each_entry(child, &bus->children, node)
2505 		pnv_pci_enable_bridge(child);
2506 }
2507 
2508 static void pnv_pci_enable_bridges(void)
2509 {
2510 	struct pci_controller *hose;
2511 
2512 	list_for_each_entry(hose, &hose_list, list_node)
2513 		pnv_pci_enable_bridge(hose->bus);
2514 }
2515 
2516 static void pnv_pci_ioda_fixup(void)
2517 {
2518 	pnv_pci_ioda_create_dbgfs();
2519 
2520 	pnv_pci_enable_bridges();
2521 
2522 #ifdef CONFIG_EEH
2523 	pnv_eeh_post_init();
2524 #endif
2525 }
2526 
2527 /*
2528  * Returns the alignment for I/O or memory windows for P2P
2529  * bridges. That actually depends on how PEs are segmented.
2530  * For now, we return I/O or M32 segment size for PE sensitive
2531  * P2P bridges. Otherwise, the default values (4KiB for I/O,
2532  * 1MiB for memory) will be returned.
2533  *
2534  * The current PCI bus might be put into one PE, which was
2535  * create against the parent PCI bridge. For that case, we
2536  * needn't enlarge the alignment so that we can save some
2537  * resources.
2538  */
2539 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2540 						unsigned long type)
2541 {
2542 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
2543 	int num_pci_bridges = 0;
2544 	struct pci_dev *bridge;
2545 
2546 	bridge = bus->self;
2547 	while (bridge) {
2548 		if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2549 			num_pci_bridges++;
2550 			if (num_pci_bridges >= 2)
2551 				return 1;
2552 		}
2553 
2554 		bridge = bridge->bus->self;
2555 	}
2556 
2557 	/*
2558 	 * We fall back to M32 if M64 isn't supported. We enforce the M64
2559 	 * alignment for any 64-bit resource, PCIe doesn't care and
2560 	 * bridges only do 64-bit prefetchable anyway.
2561 	 */
2562 	if (phb->ioda.m64_segsize && pnv_pci_is_m64_flags(type))
2563 		return phb->ioda.m64_segsize;
2564 	if (type & IORESOURCE_MEM)
2565 		return phb->ioda.m32_segsize;
2566 
2567 	return phb->ioda.io_segsize;
2568 }
2569 
2570 /*
2571  * We are updating root port or the upstream port of the
2572  * bridge behind the root port with PHB's windows in order
2573  * to accommodate the changes on required resources during
2574  * PCI (slot) hotplug, which is connected to either root
2575  * port or the downstream ports of PCIe switch behind the
2576  * root port.
2577  */
2578 static void pnv_pci_fixup_bridge_resources(struct pci_bus *bus,
2579 					   unsigned long type)
2580 {
2581 	struct pci_controller *hose = pci_bus_to_host(bus);
2582 	struct pnv_phb *phb = hose->private_data;
2583 	struct pci_dev *bridge = bus->self;
2584 	struct resource *r, *w;
2585 	bool msi_region = false;
2586 	int i;
2587 
2588 	/* Check if we need apply fixup to the bridge's windows */
2589 	if (!pci_is_root_bus(bridge->bus) &&
2590 	    !pci_is_root_bus(bridge->bus->self->bus))
2591 		return;
2592 
2593 	/* Fixup the resources */
2594 	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
2595 		r = &bridge->resource[PCI_BRIDGE_RESOURCES + i];
2596 		if (!r->flags || !r->parent)
2597 			continue;
2598 
2599 		w = NULL;
2600 		if (r->flags & type & IORESOURCE_IO)
2601 			w = &hose->io_resource;
2602 		else if (pnv_pci_is_m64(phb, r) &&
2603 			 (type & IORESOURCE_PREFETCH) &&
2604 			 phb->ioda.m64_segsize)
2605 			w = &hose->mem_resources[1];
2606 		else if (r->flags & type & IORESOURCE_MEM) {
2607 			w = &hose->mem_resources[0];
2608 			msi_region = true;
2609 		}
2610 
2611 		r->start = w->start;
2612 		r->end = w->end;
2613 
2614 		/* The 64KB 32-bits MSI region shouldn't be included in
2615 		 * the 32-bits bridge window. Otherwise, we can see strange
2616 		 * issues. One of them is EEH error observed on Garrison.
2617 		 *
2618 		 * Exclude top 1MB region which is the minimal alignment of
2619 		 * 32-bits bridge window.
2620 		 */
2621 		if (msi_region) {
2622 			r->end += 0x10000;
2623 			r->end -= 0x100000;
2624 		}
2625 	}
2626 }
2627 
2628 static void pnv_pci_configure_bus(struct pci_bus *bus)
2629 {
2630 	struct pci_dev *bridge = bus->self;
2631 	struct pnv_ioda_pe *pe;
2632 	bool all = (bridge && pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE);
2633 
2634 	dev_info(&bus->dev, "Configuring PE for bus\n");
2635 
2636 	/* Don't assign PE to PCI bus, which doesn't have subordinate devices */
2637 	if (WARN_ON(list_empty(&bus->devices)))
2638 		return;
2639 
2640 	/* Reserve PEs according to used M64 resources */
2641 	pnv_ioda_reserve_m64_pe(bus, NULL, all);
2642 
2643 	/*
2644 	 * Assign PE. We might run here because of partial hotplug.
2645 	 * For the case, we just pick up the existing PE and should
2646 	 * not allocate resources again.
2647 	 */
2648 	pe = pnv_ioda_setup_bus_PE(bus, all);
2649 	if (!pe)
2650 		return;
2651 
2652 	pnv_ioda_setup_pe_seg(pe);
2653 }
2654 
2655 static resource_size_t pnv_pci_default_alignment(void)
2656 {
2657 	return PAGE_SIZE;
2658 }
2659 
2660 /* Prevent enabling devices for which we couldn't properly
2661  * assign a PE
2662  */
2663 static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
2664 {
2665 	struct pci_dn *pdn;
2666 
2667 	pdn = pci_get_pdn(dev);
2668 	if (!pdn || pdn->pe_number == IODA_INVALID_PE) {
2669 		pci_err(dev, "pci_enable_device() blocked, no PE assigned.\n");
2670 		return false;
2671 	}
2672 
2673 	return true;
2674 }
2675 
2676 static bool pnv_ocapi_enable_device_hook(struct pci_dev *dev)
2677 {
2678 	struct pci_dn *pdn;
2679 	struct pnv_ioda_pe *pe;
2680 
2681 	pdn = pci_get_pdn(dev);
2682 	if (!pdn)
2683 		return false;
2684 
2685 	if (pdn->pe_number == IODA_INVALID_PE) {
2686 		pe = pnv_ioda_setup_dev_PE(dev);
2687 		if (!pe)
2688 			return false;
2689 	}
2690 	return true;
2691 }
2692 
2693 static long pnv_pci_ioda1_unset_window(struct iommu_table_group *table_group,
2694 				       int num)
2695 {
2696 	struct pnv_ioda_pe *pe = container_of(table_group,
2697 					      struct pnv_ioda_pe, table_group);
2698 	struct pnv_phb *phb = pe->phb;
2699 	unsigned int idx;
2700 	long rc;
2701 
2702 	pe_info(pe, "Removing DMA window #%d\n", num);
2703 	for (idx = 0; idx < phb->ioda.dma32_count; idx++) {
2704 		if (phb->ioda.dma32_segmap[idx] != pe->pe_number)
2705 			continue;
2706 
2707 		rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
2708 						idx, 0, 0ul, 0ul, 0ul);
2709 		if (rc != OPAL_SUCCESS) {
2710 			pe_warn(pe, "Failure %ld unmapping DMA32 segment#%d\n",
2711 				rc, idx);
2712 			return rc;
2713 		}
2714 
2715 		phb->ioda.dma32_segmap[idx] = IODA_INVALID_PE;
2716 	}
2717 
2718 	pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
2719 	return OPAL_SUCCESS;
2720 }
2721 
2722 static void pnv_pci_ioda1_release_pe_dma(struct pnv_ioda_pe *pe)
2723 {
2724 	struct iommu_table *tbl = pe->table_group.tables[0];
2725 	int64_t rc;
2726 
2727 	if (!pe->dma_setup_done)
2728 		return;
2729 
2730 	rc = pnv_pci_ioda1_unset_window(&pe->table_group, 0);
2731 	if (rc != OPAL_SUCCESS)
2732 		return;
2733 
2734 	pnv_pci_p7ioc_tce_invalidate(tbl, tbl->it_offset, tbl->it_size);
2735 	if (pe->table_group.group) {
2736 		iommu_group_put(pe->table_group.group);
2737 		WARN_ON(pe->table_group.group);
2738 	}
2739 
2740 	free_pages(tbl->it_base, get_order(tbl->it_size << 3));
2741 	iommu_tce_table_put(tbl);
2742 }
2743 
2744 void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
2745 {
2746 	struct iommu_table *tbl = pe->table_group.tables[0];
2747 	int64_t rc;
2748 
2749 	if (!pe->dma_setup_done)
2750 		return;
2751 
2752 	rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
2753 	if (rc)
2754 		pe_warn(pe, "OPAL error %lld release DMA window\n", rc);
2755 
2756 	pnv_pci_ioda2_set_bypass(pe, false);
2757 	if (pe->table_group.group) {
2758 		iommu_group_put(pe->table_group.group);
2759 		WARN_ON(pe->table_group.group);
2760 	}
2761 
2762 	iommu_tce_table_put(tbl);
2763 }
2764 
2765 static void pnv_ioda_free_pe_seg(struct pnv_ioda_pe *pe,
2766 				 unsigned short win,
2767 				 unsigned int *map)
2768 {
2769 	struct pnv_phb *phb = pe->phb;
2770 	int idx;
2771 	int64_t rc;
2772 
2773 	for (idx = 0; idx < phb->ioda.total_pe_num; idx++) {
2774 		if (map[idx] != pe->pe_number)
2775 			continue;
2776 
2777 		rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2778 				phb->ioda.reserved_pe_idx, win, 0, idx);
2779 
2780 		if (rc != OPAL_SUCCESS)
2781 			pe_warn(pe, "Error %lld unmapping (%d) segment#%d\n",
2782 				rc, win, idx);
2783 
2784 		map[idx] = IODA_INVALID_PE;
2785 	}
2786 }
2787 
2788 static void pnv_ioda_release_pe_seg(struct pnv_ioda_pe *pe)
2789 {
2790 	struct pnv_phb *phb = pe->phb;
2791 
2792 	if (phb->type == PNV_PHB_IODA1) {
2793 		pnv_ioda_free_pe_seg(pe, OPAL_IO_WINDOW_TYPE,
2794 				     phb->ioda.io_segmap);
2795 		pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
2796 				     phb->ioda.m32_segmap);
2797 		/* M64 is pre-configured by pnv_ioda1_init_m64() */
2798 	} else if (phb->type == PNV_PHB_IODA2) {
2799 		pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
2800 				     phb->ioda.m32_segmap);
2801 	}
2802 }
2803 
2804 static void pnv_ioda_release_pe(struct pnv_ioda_pe *pe)
2805 {
2806 	struct pnv_phb *phb = pe->phb;
2807 	struct pnv_ioda_pe *slave, *tmp;
2808 
2809 	pe_info(pe, "Releasing PE\n");
2810 
2811 	mutex_lock(&phb->ioda.pe_list_mutex);
2812 	list_del(&pe->list);
2813 	mutex_unlock(&phb->ioda.pe_list_mutex);
2814 
2815 	switch (phb->type) {
2816 	case PNV_PHB_IODA1:
2817 		pnv_pci_ioda1_release_pe_dma(pe);
2818 		break;
2819 	case PNV_PHB_IODA2:
2820 		pnv_pci_ioda2_release_pe_dma(pe);
2821 		break;
2822 	case PNV_PHB_NPU_OCAPI:
2823 		break;
2824 	default:
2825 		WARN_ON(1);
2826 	}
2827 
2828 	pnv_ioda_release_pe_seg(pe);
2829 	pnv_ioda_deconfigure_pe(pe->phb, pe);
2830 
2831 	/* Release slave PEs in the compound PE */
2832 	if (pe->flags & PNV_IODA_PE_MASTER) {
2833 		list_for_each_entry_safe(slave, tmp, &pe->slaves, list) {
2834 			list_del(&slave->list);
2835 			pnv_ioda_free_pe(slave);
2836 		}
2837 	}
2838 
2839 	/*
2840 	 * The PE for root bus can be removed because of hotplug in EEH
2841 	 * recovery for fenced PHB error. We need to mark the PE dead so
2842 	 * that it can be populated again in PCI hot add path. The PE
2843 	 * shouldn't be destroyed as it's the global reserved resource.
2844 	 */
2845 	if (phb->ioda.root_pe_idx == pe->pe_number)
2846 		return;
2847 
2848 	pnv_ioda_free_pe(pe);
2849 }
2850 
2851 static void pnv_pci_release_device(struct pci_dev *pdev)
2852 {
2853 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
2854 	struct pci_dn *pdn = pci_get_pdn(pdev);
2855 	struct pnv_ioda_pe *pe;
2856 
2857 	/* The VF PE state is torn down when sriov_disable() is called */
2858 	if (pdev->is_virtfn)
2859 		return;
2860 
2861 	if (!pdn || pdn->pe_number == IODA_INVALID_PE)
2862 		return;
2863 
2864 #ifdef CONFIG_PCI_IOV
2865 	/*
2866 	 * FIXME: Try move this to sriov_disable(). It's here since we allocate
2867 	 * the iov state at probe time since we need to fiddle with the IOV
2868 	 * resources.
2869 	 */
2870 	if (pdev->is_physfn)
2871 		kfree(pdev->dev.archdata.iov_data);
2872 #endif
2873 
2874 	/*
2875 	 * PCI hotplug can happen as part of EEH error recovery. The @pdn
2876 	 * isn't removed and added afterwards in this scenario. We should
2877 	 * set the PE number in @pdn to an invalid one. Otherwise, the PE's
2878 	 * device count is decreased on removing devices while failing to
2879 	 * be increased on adding devices. It leads to unbalanced PE's device
2880 	 * count and eventually make normal PCI hotplug path broken.
2881 	 */
2882 	pe = &phb->ioda.pe_array[pdn->pe_number];
2883 	pdn->pe_number = IODA_INVALID_PE;
2884 
2885 	WARN_ON(--pe->device_count < 0);
2886 	if (pe->device_count == 0)
2887 		pnv_ioda_release_pe(pe);
2888 }
2889 
2890 static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
2891 {
2892 	struct pnv_phb *phb = hose->private_data;
2893 
2894 	opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
2895 		       OPAL_ASSERT_RESET);
2896 }
2897 
2898 static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus)
2899 {
2900 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
2901 	struct pnv_ioda_pe *pe;
2902 
2903 	list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2904 		if (!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)))
2905 			continue;
2906 
2907 		if (!pe->pbus)
2908 			continue;
2909 
2910 		if (bus->number == ((pe->rid >> 8) & 0xFF)) {
2911 			pe->pbus = bus;
2912 			break;
2913 		}
2914 	}
2915 }
2916 
2917 static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2918 	.dma_dev_setup		= pnv_pci_ioda_dma_dev_setup,
2919 	.dma_bus_setup		= pnv_pci_ioda_dma_bus_setup,
2920 	.iommu_bypass_supported	= pnv_pci_ioda_iommu_bypass_supported,
2921 	.enable_device_hook	= pnv_pci_enable_device_hook,
2922 	.release_device		= pnv_pci_release_device,
2923 	.window_alignment	= pnv_pci_window_alignment,
2924 	.setup_bridge		= pnv_pci_fixup_bridge_resources,
2925 	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
2926 	.shutdown		= pnv_pci_ioda_shutdown,
2927 };
2928 
2929 static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
2930 	.enable_device_hook	= pnv_ocapi_enable_device_hook,
2931 	.release_device		= pnv_pci_release_device,
2932 	.window_alignment	= pnv_pci_window_alignment,
2933 	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
2934 	.shutdown		= pnv_pci_ioda_shutdown,
2935 };
2936 
2937 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2938 					 u64 hub_id, int ioda_type)
2939 {
2940 	struct pci_controller *hose;
2941 	struct pnv_phb *phb;
2942 	unsigned long size, m64map_off, m32map_off, pemap_off;
2943 	unsigned long iomap_off = 0, dma32map_off = 0;
2944 	struct pnv_ioda_pe *root_pe;
2945 	struct resource r;
2946 	const __be64 *prop64;
2947 	const __be32 *prop32;
2948 	int len;
2949 	unsigned int segno;
2950 	u64 phb_id;
2951 	void *aux;
2952 	long rc;
2953 
2954 	if (!of_device_is_available(np))
2955 		return;
2956 
2957 	pr_info("Initializing %s PHB (%pOF)\n",	pnv_phb_names[ioda_type], np);
2958 
2959 	prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2960 	if (!prop64) {
2961 		pr_err("  Missing \"ibm,opal-phbid\" property !\n");
2962 		return;
2963 	}
2964 	phb_id = be64_to_cpup(prop64);
2965 	pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
2966 
2967 	phb = kzalloc(sizeof(*phb), GFP_KERNEL);
2968 	if (!phb)
2969 		panic("%s: Failed to allocate %zu bytes\n", __func__,
2970 		      sizeof(*phb));
2971 
2972 	/* Allocate PCI controller */
2973 	phb->hose = hose = pcibios_alloc_controller(np);
2974 	if (!phb->hose) {
2975 		pr_err("  Can't allocate PCI controller for %pOF\n",
2976 		       np);
2977 		memblock_free(phb, sizeof(struct pnv_phb));
2978 		return;
2979 	}
2980 
2981 	spin_lock_init(&phb->lock);
2982 	prop32 = of_get_property(np, "bus-range", &len);
2983 	if (prop32 && len == 8) {
2984 		hose->first_busno = be32_to_cpu(prop32[0]);
2985 		hose->last_busno = be32_to_cpu(prop32[1]);
2986 	} else {
2987 		pr_warn("  Broken <bus-range> on %pOF\n", np);
2988 		hose->first_busno = 0;
2989 		hose->last_busno = 0xff;
2990 	}
2991 	hose->private_data = phb;
2992 	phb->hub_id = hub_id;
2993 	phb->opal_id = phb_id;
2994 	phb->type = ioda_type;
2995 	mutex_init(&phb->ioda.pe_alloc_mutex);
2996 
2997 	/* Detect specific models for error handling */
2998 	if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2999 		phb->model = PNV_PHB_MODEL_P7IOC;
3000 	else if (of_device_is_compatible(np, "ibm,power8-pciex"))
3001 		phb->model = PNV_PHB_MODEL_PHB3;
3002 	else
3003 		phb->model = PNV_PHB_MODEL_UNKNOWN;
3004 
3005 	/* Initialize diagnostic data buffer */
3006 	prop32 = of_get_property(np, "ibm,phb-diag-data-size", NULL);
3007 	if (prop32)
3008 		phb->diag_data_size = be32_to_cpup(prop32);
3009 	else
3010 		phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE;
3011 
3012 	phb->diag_data = kzalloc(phb->diag_data_size, GFP_KERNEL);
3013 	if (!phb->diag_data)
3014 		panic("%s: Failed to allocate %u bytes\n", __func__,
3015 		      phb->diag_data_size);
3016 
3017 	/* Parse 32-bit and IO ranges (if any) */
3018 	pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
3019 
3020 	/* Get registers */
3021 	if (!of_address_to_resource(np, 0, &r)) {
3022 		phb->regs_phys = r.start;
3023 		phb->regs = ioremap(r.start, resource_size(&r));
3024 		if (phb->regs == NULL)
3025 			pr_err("  Failed to map registers !\n");
3026 	}
3027 
3028 	/* Initialize more IODA stuff */
3029 	phb->ioda.total_pe_num = 1;
3030 	prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
3031 	if (prop32)
3032 		phb->ioda.total_pe_num = be32_to_cpup(prop32);
3033 	prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
3034 	if (prop32)
3035 		phb->ioda.reserved_pe_idx = be32_to_cpup(prop32);
3036 
3037 	/* Invalidate RID to PE# mapping */
3038 	for (segno = 0; segno < ARRAY_SIZE(phb->ioda.pe_rmap); segno++)
3039 		phb->ioda.pe_rmap[segno] = IODA_INVALID_PE;
3040 
3041 	/* Parse 64-bit MMIO range */
3042 	pnv_ioda_parse_m64_window(phb);
3043 
3044 	phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
3045 	/* FW Has already off top 64k of M32 space (MSI space) */
3046 	phb->ioda.m32_size += 0x10000;
3047 
3048 	phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe_num;
3049 	phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
3050 	phb->ioda.io_size = hose->pci_io_size;
3051 	phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe_num;
3052 	phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
3053 
3054 	/* Calculate how many 32-bit TCE segments we have */
3055 	phb->ioda.dma32_count = phb->ioda.m32_pci_base /
3056 				PNV_IODA1_DMA32_SEGSIZE;
3057 
3058 	/* Allocate aux data & arrays. We don't have IO ports on PHB3 */
3059 	size = ALIGN(max_t(unsigned, phb->ioda.total_pe_num, 8) / 8,
3060 			sizeof(unsigned long));
3061 	m64map_off = size;
3062 	size += phb->ioda.total_pe_num * sizeof(phb->ioda.m64_segmap[0]);
3063 	m32map_off = size;
3064 	size += phb->ioda.total_pe_num * sizeof(phb->ioda.m32_segmap[0]);
3065 	if (phb->type == PNV_PHB_IODA1) {
3066 		iomap_off = size;
3067 		size += phb->ioda.total_pe_num * sizeof(phb->ioda.io_segmap[0]);
3068 		dma32map_off = size;
3069 		size += phb->ioda.dma32_count *
3070 			sizeof(phb->ioda.dma32_segmap[0]);
3071 	}
3072 	pemap_off = size;
3073 	size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe);
3074 	aux = kzalloc(size, GFP_KERNEL);
3075 	if (!aux)
3076 		panic("%s: Failed to allocate %lu bytes\n", __func__, size);
3077 
3078 	phb->ioda.pe_alloc = aux;
3079 	phb->ioda.m64_segmap = aux + m64map_off;
3080 	phb->ioda.m32_segmap = aux + m32map_off;
3081 	for (segno = 0; segno < phb->ioda.total_pe_num; segno++) {
3082 		phb->ioda.m64_segmap[segno] = IODA_INVALID_PE;
3083 		phb->ioda.m32_segmap[segno] = IODA_INVALID_PE;
3084 	}
3085 	if (phb->type == PNV_PHB_IODA1) {
3086 		phb->ioda.io_segmap = aux + iomap_off;
3087 		for (segno = 0; segno < phb->ioda.total_pe_num; segno++)
3088 			phb->ioda.io_segmap[segno] = IODA_INVALID_PE;
3089 
3090 		phb->ioda.dma32_segmap = aux + dma32map_off;
3091 		for (segno = 0; segno < phb->ioda.dma32_count; segno++)
3092 			phb->ioda.dma32_segmap[segno] = IODA_INVALID_PE;
3093 	}
3094 	phb->ioda.pe_array = aux + pemap_off;
3095 
3096 	/*
3097 	 * Choose PE number for root bus, which shouldn't have
3098 	 * M64 resources consumed by its child devices. To pick
3099 	 * the PE number adjacent to the reserved one if possible.
3100 	 */
3101 	pnv_ioda_reserve_pe(phb, phb->ioda.reserved_pe_idx);
3102 	if (phb->ioda.reserved_pe_idx == 0) {
3103 		phb->ioda.root_pe_idx = 1;
3104 		pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
3105 	} else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1)) {
3106 		phb->ioda.root_pe_idx = phb->ioda.reserved_pe_idx - 1;
3107 		pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
3108 	} else {
3109 		/* otherwise just allocate one */
3110 		root_pe = pnv_ioda_alloc_pe(phb, 1);
3111 		phb->ioda.root_pe_idx = root_pe->pe_number;
3112 	}
3113 
3114 	INIT_LIST_HEAD(&phb->ioda.pe_list);
3115 	mutex_init(&phb->ioda.pe_list_mutex);
3116 
3117 	/* Calculate how many 32-bit TCE segments we have */
3118 	phb->ioda.dma32_count = phb->ioda.m32_pci_base /
3119 				PNV_IODA1_DMA32_SEGSIZE;
3120 
3121 #if 0 /* We should really do that ... */
3122 	rc = opal_pci_set_phb_mem_window(opal->phb_id,
3123 					 window_type,
3124 					 window_num,
3125 					 starting_real_address,
3126 					 starting_pci_address,
3127 					 segment_size);
3128 #endif
3129 
3130 	pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
3131 		phb->ioda.total_pe_num, phb->ioda.reserved_pe_idx,
3132 		phb->ioda.m32_size, phb->ioda.m32_segsize);
3133 	if (phb->ioda.m64_size)
3134 		pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
3135 			phb->ioda.m64_size, phb->ioda.m64_segsize);
3136 	if (phb->ioda.io_size)
3137 		pr_info("                  IO: 0x%x [segment=0x%x]\n",
3138 			phb->ioda.io_size, phb->ioda.io_segsize);
3139 
3140 
3141 	phb->hose->ops = &pnv_pci_ops;
3142 	phb->get_pe_state = pnv_ioda_get_pe_state;
3143 	phb->freeze_pe = pnv_ioda_freeze_pe;
3144 	phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
3145 
3146 	/* Setup MSI support */
3147 	pnv_pci_init_ioda_msis(phb);
3148 
3149 	/*
3150 	 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
3151 	 * to let the PCI core do resource assignment. It's supposed
3152 	 * that the PCI core will do correct I/O and MMIO alignment
3153 	 * for the P2P bridge bars so that each PCI bus (excluding
3154 	 * the child P2P bridges) can form individual PE.
3155 	 */
3156 	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
3157 
3158 	switch (phb->type) {
3159 	case PNV_PHB_NPU_OCAPI:
3160 		hose->controller_ops = pnv_npu_ocapi_ioda_controller_ops;
3161 		break;
3162 	default:
3163 		hose->controller_ops = pnv_pci_ioda_controller_ops;
3164 	}
3165 
3166 	ppc_md.pcibios_default_alignment = pnv_pci_default_alignment;
3167 
3168 #ifdef CONFIG_PCI_IOV
3169 	ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov;
3170 	ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
3171 	ppc_md.pcibios_sriov_enable = pnv_pcibios_sriov_enable;
3172 	ppc_md.pcibios_sriov_disable = pnv_pcibios_sriov_disable;
3173 #endif
3174 
3175 	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
3176 
3177 	/* Reset IODA tables to a clean state */
3178 	rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
3179 	if (rc)
3180 		pr_warn("  OPAL Error %ld performing IODA table reset !\n", rc);
3181 
3182 	/*
3183 	 * If we're running in kdump kernel, the previous kernel never
3184 	 * shutdown PCI devices correctly. We already got IODA table
3185 	 * cleaned out. So we have to issue PHB reset to stop all PCI
3186 	 * transactions from previous kernel. The ppc_pci_reset_phbs
3187 	 * kernel parameter will force this reset too. Additionally,
3188 	 * if the IODA reset above failed then use a bigger hammer.
3189 	 * This can happen if we get a PHB fatal error in very early
3190 	 * boot.
3191 	 */
3192 	if (is_kdump_kernel() || pci_reset_phbs || rc) {
3193 		pr_info("  Issue PHB reset ...\n");
3194 		pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
3195 		pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
3196 	}
3197 
3198 	/* Remove M64 resource if we can't configure it successfully */
3199 	if (!phb->init_m64 || phb->init_m64(phb))
3200 		hose->mem_resources[1].flags = 0;
3201 
3202 	/* create pci_dn's for DT nodes under this PHB */
3203 	pci_devs_phb_init_dynamic(hose);
3204 }
3205 
3206 void __init pnv_pci_init_ioda2_phb(struct device_node *np)
3207 {
3208 	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
3209 }
3210 
3211 void __init pnv_pci_init_npu2_opencapi_phb(struct device_node *np)
3212 {
3213 	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_OCAPI);
3214 }
3215 
3216 static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev)
3217 {
3218 	struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
3219 
3220 	if (!machine_is(powernv))
3221 		return;
3222 
3223 	if (phb->type == PNV_PHB_NPU_OCAPI)
3224 		dev->cfg_size = PCI_CFG_SPACE_EXP_SIZE;
3225 }
3226 DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pnv_npu2_opencapi_cfg_size_fixup);
3227 
3228 void __init pnv_pci_init_ioda_hub(struct device_node *np)
3229 {
3230 	struct device_node *phbn;
3231 	const __be64 *prop64;
3232 	u64 hub_id;
3233 
3234 	pr_info("Probing IODA IO-Hub %pOF\n", np);
3235 
3236 	prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
3237 	if (!prop64) {
3238 		pr_err(" Missing \"ibm,opal-hubid\" property !\n");
3239 		return;
3240 	}
3241 	hub_id = be64_to_cpup(prop64);
3242 	pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
3243 
3244 	/* Count child PHBs */
3245 	for_each_child_of_node(np, phbn) {
3246 		/* Look for IODA1 PHBs */
3247 		if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
3248 			pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
3249 	}
3250 }
3251