1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Support PCI/PCIe on PowerNV platforms
4  *
5  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
6  */
7 
8 #undef DEBUG
9 
10 #include <linux/kernel.h>
11 #include <linux/pci.h>
12 #include <linux/crash_dump.h>
13 #include <linux/delay.h>
14 #include <linux/string.h>
15 #include <linux/init.h>
16 #include <linux/memblock.h>
17 #include <linux/irq.h>
18 #include <linux/io.h>
19 #include <linux/msi.h>
20 #include <linux/iommu.h>
21 #include <linux/rculist.h>
22 #include <linux/sizes.h>
23 #include <linux/debugfs.h>
24 #include <linux/of_address.h>
25 #include <linux/of_irq.h>
26 
27 #include <asm/sections.h>
28 #include <asm/io.h>
29 #include <asm/pci-bridge.h>
30 #include <asm/machdep.h>
31 #include <asm/msi_bitmap.h>
32 #include <asm/ppc-pci.h>
33 #include <asm/opal.h>
34 #include <asm/iommu.h>
35 #include <asm/tce.h>
36 #include <asm/xics.h>
37 #include <asm/firmware.h>
38 #include <asm/pnv-pci.h>
39 #include <asm/mmzone.h>
40 #include <asm/xive.h>
41 
42 #include <misc/cxl-base.h>
43 
44 #include "powernv.h"
45 #include "pci.h"
46 #include "../../../../drivers/pci/pci.h"
47 
48 /* This array is indexed with enum pnv_phb_type */
49 static const char * const pnv_phb_names[] = { "IODA2", "NPU_OCAPI" };
50 
51 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable);
52 static void pnv_pci_configure_bus(struct pci_bus *bus);
53 
54 void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
55 			    const char *fmt, ...)
56 {
57 	struct va_format vaf;
58 	va_list args;
59 	char pfix[32];
60 
61 	va_start(args, fmt);
62 
63 	vaf.fmt = fmt;
64 	vaf.va = &args;
65 
66 	if (pe->flags & PNV_IODA_PE_DEV)
67 		strscpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
68 	else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
69 		sprintf(pfix, "%04x:%02x     ",
70 			pci_domain_nr(pe->pbus), pe->pbus->number);
71 #ifdef CONFIG_PCI_IOV
72 	else if (pe->flags & PNV_IODA_PE_VF)
73 		sprintf(pfix, "%04x:%02x:%2x.%d",
74 			pci_domain_nr(pe->parent_dev->bus),
75 			(pe->rid & 0xff00) >> 8,
76 			PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
77 #endif /* CONFIG_PCI_IOV*/
78 
79 	printk("%spci %s: [PE# %.2x] %pV",
80 	       level, pfix, pe->pe_number, &vaf);
81 
82 	va_end(args);
83 }
84 
85 static bool pnv_iommu_bypass_disabled __read_mostly;
86 static bool pci_reset_phbs __read_mostly;
87 
88 static int __init iommu_setup(char *str)
89 {
90 	if (!str)
91 		return -EINVAL;
92 
93 	while (*str) {
94 		if (!strncmp(str, "nobypass", 8)) {
95 			pnv_iommu_bypass_disabled = true;
96 			pr_info("PowerNV: IOMMU bypass window disabled.\n");
97 			break;
98 		}
99 		str += strcspn(str, ",");
100 		if (*str == ',')
101 			str++;
102 	}
103 
104 	return 0;
105 }
106 early_param("iommu", iommu_setup);
107 
108 static int __init pci_reset_phbs_setup(char *str)
109 {
110 	pci_reset_phbs = true;
111 	return 0;
112 }
113 
114 early_param("ppc_pci_reset_phbs", pci_reset_phbs_setup);
115 
116 static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
117 {
118 	s64 rc;
119 
120 	phb->ioda.pe_array[pe_no].phb = phb;
121 	phb->ioda.pe_array[pe_no].pe_number = pe_no;
122 	phb->ioda.pe_array[pe_no].dma_setup_done = false;
123 
124 	/*
125 	 * Clear the PE frozen state as it might be put into frozen state
126 	 * in the last PCI remove path. It's not harmful to do so when the
127 	 * PE is already in unfrozen state.
128 	 */
129 	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
130 				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
131 	if (rc != OPAL_SUCCESS && rc != OPAL_UNSUPPORTED)
132 		pr_warn("%s: Error %lld unfreezing PHB#%x-PE#%x\n",
133 			__func__, rc, phb->hose->global_number, pe_no);
134 
135 	return &phb->ioda.pe_array[pe_no];
136 }
137 
138 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
139 {
140 	if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe_num)) {
141 		pr_warn("%s: Invalid PE %x on PHB#%x\n",
142 			__func__, pe_no, phb->hose->global_number);
143 		return;
144 	}
145 
146 	mutex_lock(&phb->ioda.pe_alloc_mutex);
147 	if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
148 		pr_debug("%s: PE %x was reserved on PHB#%x\n",
149 			 __func__, pe_no, phb->hose->global_number);
150 	mutex_unlock(&phb->ioda.pe_alloc_mutex);
151 
152 	pnv_ioda_init_pe(phb, pe_no);
153 }
154 
155 struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb, int count)
156 {
157 	struct pnv_ioda_pe *ret = NULL;
158 	int run = 0, pe, i;
159 
160 	mutex_lock(&phb->ioda.pe_alloc_mutex);
161 
162 	/* scan backwards for a run of @count cleared bits */
163 	for (pe = phb->ioda.total_pe_num - 1; pe >= 0; pe--) {
164 		if (test_bit(pe, phb->ioda.pe_alloc)) {
165 			run = 0;
166 			continue;
167 		}
168 
169 		run++;
170 		if (run == count)
171 			break;
172 	}
173 	if (run != count)
174 		goto out;
175 
176 	for (i = pe; i < pe + count; i++) {
177 		set_bit(i, phb->ioda.pe_alloc);
178 		pnv_ioda_init_pe(phb, i);
179 	}
180 	ret = &phb->ioda.pe_array[pe];
181 
182 out:
183 	mutex_unlock(&phb->ioda.pe_alloc_mutex);
184 	return ret;
185 }
186 
187 void pnv_ioda_free_pe(struct pnv_ioda_pe *pe)
188 {
189 	struct pnv_phb *phb = pe->phb;
190 	unsigned int pe_num = pe->pe_number;
191 
192 	WARN_ON(pe->pdev);
193 	memset(pe, 0, sizeof(struct pnv_ioda_pe));
194 
195 	mutex_lock(&phb->ioda.pe_alloc_mutex);
196 	clear_bit(pe_num, phb->ioda.pe_alloc);
197 	mutex_unlock(&phb->ioda.pe_alloc_mutex);
198 }
199 
200 /* The default M64 BAR is shared by all PEs */
201 static int pnv_ioda2_init_m64(struct pnv_phb *phb)
202 {
203 	const char *desc;
204 	struct resource *r;
205 	s64 rc;
206 
207 	/* Configure the default M64 BAR */
208 	rc = opal_pci_set_phb_mem_window(phb->opal_id,
209 					 OPAL_M64_WINDOW_TYPE,
210 					 phb->ioda.m64_bar_idx,
211 					 phb->ioda.m64_base,
212 					 0, /* unused */
213 					 phb->ioda.m64_size);
214 	if (rc != OPAL_SUCCESS) {
215 		desc = "configuring";
216 		goto fail;
217 	}
218 
219 	/* Enable the default M64 BAR */
220 	rc = opal_pci_phb_mmio_enable(phb->opal_id,
221 				      OPAL_M64_WINDOW_TYPE,
222 				      phb->ioda.m64_bar_idx,
223 				      OPAL_ENABLE_M64_SPLIT);
224 	if (rc != OPAL_SUCCESS) {
225 		desc = "enabling";
226 		goto fail;
227 	}
228 
229 	/*
230 	 * Exclude the segments for reserved and root bus PE, which
231 	 * are first or last two PEs.
232 	 */
233 	r = &phb->hose->mem_resources[1];
234 	if (phb->ioda.reserved_pe_idx == 0)
235 		r->start += (2 * phb->ioda.m64_segsize);
236 	else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
237 		r->end -= (2 * phb->ioda.m64_segsize);
238 	else
239 		pr_warn("  Cannot strip M64 segment for reserved PE#%x\n",
240 			phb->ioda.reserved_pe_idx);
241 
242 	return 0;
243 
244 fail:
245 	pr_warn("  Failure %lld %s M64 BAR#%d\n",
246 		rc, desc, phb->ioda.m64_bar_idx);
247 	opal_pci_phb_mmio_enable(phb->opal_id,
248 				 OPAL_M64_WINDOW_TYPE,
249 				 phb->ioda.m64_bar_idx,
250 				 OPAL_DISABLE_M64);
251 	return -EIO;
252 }
253 
254 static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
255 					 unsigned long *pe_bitmap)
256 {
257 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
258 	struct resource *r;
259 	resource_size_t base, sgsz, start, end;
260 	int segno, i;
261 
262 	base = phb->ioda.m64_base;
263 	sgsz = phb->ioda.m64_segsize;
264 	for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
265 		r = &pdev->resource[i];
266 		if (!r->parent || !pnv_pci_is_m64(phb, r))
267 			continue;
268 
269 		start = ALIGN_DOWN(r->start - base, sgsz);
270 		end = ALIGN(r->end - base, sgsz);
271 		for (segno = start / sgsz; segno < end / sgsz; segno++) {
272 			if (pe_bitmap)
273 				set_bit(segno, pe_bitmap);
274 			else
275 				pnv_ioda_reserve_pe(phb, segno);
276 		}
277 	}
278 }
279 
280 static void pnv_ioda_reserve_m64_pe(struct pci_bus *bus,
281 				    unsigned long *pe_bitmap,
282 				    bool all)
283 {
284 	struct pci_dev *pdev;
285 
286 	list_for_each_entry(pdev, &bus->devices, bus_list) {
287 		pnv_ioda_reserve_dev_m64_pe(pdev, pe_bitmap);
288 
289 		if (all && pdev->subordinate)
290 			pnv_ioda_reserve_m64_pe(pdev->subordinate,
291 						pe_bitmap, all);
292 	}
293 }
294 
295 static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all)
296 {
297 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
298 	struct pnv_ioda_pe *master_pe, *pe;
299 	unsigned long size, *pe_alloc;
300 	int i;
301 
302 	/* Root bus shouldn't use M64 */
303 	if (pci_is_root_bus(bus))
304 		return NULL;
305 
306 	/* Allocate bitmap */
307 	size = ALIGN(phb->ioda.total_pe_num / 8, sizeof(unsigned long));
308 	pe_alloc = kzalloc(size, GFP_KERNEL);
309 	if (!pe_alloc) {
310 		pr_warn("%s: Out of memory !\n",
311 			__func__);
312 		return NULL;
313 	}
314 
315 	/* Figure out reserved PE numbers by the PE */
316 	pnv_ioda_reserve_m64_pe(bus, pe_alloc, all);
317 
318 	/*
319 	 * the current bus might not own M64 window and that's all
320 	 * contributed by its child buses. For the case, we needn't
321 	 * pick M64 dependent PE#.
322 	 */
323 	if (bitmap_empty(pe_alloc, phb->ioda.total_pe_num)) {
324 		kfree(pe_alloc);
325 		return NULL;
326 	}
327 
328 	/*
329 	 * Figure out the master PE and put all slave PEs to master
330 	 * PE's list to form compound PE.
331 	 */
332 	master_pe = NULL;
333 	i = -1;
334 	while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe_num, i + 1)) <
335 		phb->ioda.total_pe_num) {
336 		pe = &phb->ioda.pe_array[i];
337 
338 		phb->ioda.m64_segmap[pe->pe_number] = pe->pe_number;
339 		if (!master_pe) {
340 			pe->flags |= PNV_IODA_PE_MASTER;
341 			INIT_LIST_HEAD(&pe->slaves);
342 			master_pe = pe;
343 		} else {
344 			pe->flags |= PNV_IODA_PE_SLAVE;
345 			pe->master = master_pe;
346 			list_add_tail(&pe->list, &master_pe->slaves);
347 		}
348 	}
349 
350 	kfree(pe_alloc);
351 	return master_pe;
352 }
353 
354 static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
355 {
356 	struct pci_controller *hose = phb->hose;
357 	struct device_node *dn = hose->dn;
358 	struct resource *res;
359 	u32 m64_range[2], i;
360 	const __be32 *r;
361 	u64 pci_addr;
362 
363 	if (phb->type != PNV_PHB_IODA2) {
364 		pr_info("  Not support M64 window\n");
365 		return;
366 	}
367 
368 	if (!firmware_has_feature(FW_FEATURE_OPAL)) {
369 		pr_info("  Firmware too old to support M64 window\n");
370 		return;
371 	}
372 
373 	r = of_get_property(dn, "ibm,opal-m64-window", NULL);
374 	if (!r) {
375 		pr_info("  No <ibm,opal-m64-window> on %pOF\n",
376 			dn);
377 		return;
378 	}
379 
380 	/*
381 	 * Find the available M64 BAR range and pickup the last one for
382 	 * covering the whole 64-bits space. We support only one range.
383 	 */
384 	if (of_property_read_u32_array(dn, "ibm,opal-available-m64-ranges",
385 				       m64_range, 2)) {
386 		/* In absence of the property, assume 0..15 */
387 		m64_range[0] = 0;
388 		m64_range[1] = 16;
389 	}
390 	/* We only support 64 bits in our allocator */
391 	if (m64_range[1] > 63) {
392 		pr_warn("%s: Limiting M64 range to 63 (from %d) on PHB#%x\n",
393 			__func__, m64_range[1], phb->hose->global_number);
394 		m64_range[1] = 63;
395 	}
396 	/* Empty range, no m64 */
397 	if (m64_range[1] <= m64_range[0]) {
398 		pr_warn("%s: M64 empty, disabling M64 usage on PHB#%x\n",
399 			__func__, phb->hose->global_number);
400 		return;
401 	}
402 
403 	/* Configure M64 informations */
404 	res = &hose->mem_resources[1];
405 	res->name = dn->full_name;
406 	res->start = of_translate_address(dn, r + 2);
407 	res->end = res->start + of_read_number(r + 4, 2) - 1;
408 	res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
409 	pci_addr = of_read_number(r, 2);
410 	hose->mem_offset[1] = res->start - pci_addr;
411 
412 	phb->ioda.m64_size = resource_size(res);
413 	phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe_num;
414 	phb->ioda.m64_base = pci_addr;
415 
416 	/* This lines up nicely with the display from processing OF ranges */
417 	pr_info(" MEM 0x%016llx..0x%016llx -> 0x%016llx (M64 #%d..%d)\n",
418 		res->start, res->end, pci_addr, m64_range[0],
419 		m64_range[0] + m64_range[1] - 1);
420 
421 	/* Mark all M64 used up by default */
422 	phb->ioda.m64_bar_alloc = (unsigned long)-1;
423 
424 	/* Use last M64 BAR to cover M64 window */
425 	m64_range[1]--;
426 	phb->ioda.m64_bar_idx = m64_range[0] + m64_range[1];
427 
428 	pr_info(" Using M64 #%d as default window\n", phb->ioda.m64_bar_idx);
429 
430 	/* Mark remaining ones free */
431 	for (i = m64_range[0]; i < m64_range[1]; i++)
432 		clear_bit(i, &phb->ioda.m64_bar_alloc);
433 
434 	/*
435 	 * Setup init functions for M64 based on IODA version, IODA3 uses
436 	 * the IODA2 code.
437 	 */
438 	phb->init_m64 = pnv_ioda2_init_m64;
439 }
440 
441 static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
442 {
443 	struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
444 	struct pnv_ioda_pe *slave;
445 	s64 rc;
446 
447 	/* Fetch master PE */
448 	if (pe->flags & PNV_IODA_PE_SLAVE) {
449 		pe = pe->master;
450 		if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
451 			return;
452 
453 		pe_no = pe->pe_number;
454 	}
455 
456 	/* Freeze master PE */
457 	rc = opal_pci_eeh_freeze_set(phb->opal_id,
458 				     pe_no,
459 				     OPAL_EEH_ACTION_SET_FREEZE_ALL);
460 	if (rc != OPAL_SUCCESS) {
461 		pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
462 			__func__, rc, phb->hose->global_number, pe_no);
463 		return;
464 	}
465 
466 	/* Freeze slave PEs */
467 	if (!(pe->flags & PNV_IODA_PE_MASTER))
468 		return;
469 
470 	list_for_each_entry(slave, &pe->slaves, list) {
471 		rc = opal_pci_eeh_freeze_set(phb->opal_id,
472 					     slave->pe_number,
473 					     OPAL_EEH_ACTION_SET_FREEZE_ALL);
474 		if (rc != OPAL_SUCCESS)
475 			pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
476 				__func__, rc, phb->hose->global_number,
477 				slave->pe_number);
478 	}
479 }
480 
481 static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
482 {
483 	struct pnv_ioda_pe *pe, *slave;
484 	s64 rc;
485 
486 	/* Find master PE */
487 	pe = &phb->ioda.pe_array[pe_no];
488 	if (pe->flags & PNV_IODA_PE_SLAVE) {
489 		pe = pe->master;
490 		WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
491 		pe_no = pe->pe_number;
492 	}
493 
494 	/* Clear frozen state for master PE */
495 	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
496 	if (rc != OPAL_SUCCESS) {
497 		pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
498 			__func__, rc, opt, phb->hose->global_number, pe_no);
499 		return -EIO;
500 	}
501 
502 	if (!(pe->flags & PNV_IODA_PE_MASTER))
503 		return 0;
504 
505 	/* Clear frozen state for slave PEs */
506 	list_for_each_entry(slave, &pe->slaves, list) {
507 		rc = opal_pci_eeh_freeze_clear(phb->opal_id,
508 					     slave->pe_number,
509 					     opt);
510 		if (rc != OPAL_SUCCESS) {
511 			pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
512 				__func__, rc, opt, phb->hose->global_number,
513 				slave->pe_number);
514 			return -EIO;
515 		}
516 	}
517 
518 	return 0;
519 }
520 
521 static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
522 {
523 	struct pnv_ioda_pe *slave, *pe;
524 	u8 fstate = 0, state;
525 	__be16 pcierr = 0;
526 	s64 rc;
527 
528 	/* Sanity check on PE number */
529 	if (pe_no < 0 || pe_no >= phb->ioda.total_pe_num)
530 		return OPAL_EEH_STOPPED_PERM_UNAVAIL;
531 
532 	/*
533 	 * Fetch the master PE and the PE instance might be
534 	 * not initialized yet.
535 	 */
536 	pe = &phb->ioda.pe_array[pe_no];
537 	if (pe->flags & PNV_IODA_PE_SLAVE) {
538 		pe = pe->master;
539 		WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
540 		pe_no = pe->pe_number;
541 	}
542 
543 	/* Check the master PE */
544 	rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
545 					&state, &pcierr, NULL);
546 	if (rc != OPAL_SUCCESS) {
547 		pr_warn("%s: Failure %lld getting "
548 			"PHB#%x-PE#%x state\n",
549 			__func__, rc,
550 			phb->hose->global_number, pe_no);
551 		return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
552 	}
553 
554 	/* Check the slave PE */
555 	if (!(pe->flags & PNV_IODA_PE_MASTER))
556 		return state;
557 
558 	list_for_each_entry(slave, &pe->slaves, list) {
559 		rc = opal_pci_eeh_freeze_status(phb->opal_id,
560 						slave->pe_number,
561 						&fstate,
562 						&pcierr,
563 						NULL);
564 		if (rc != OPAL_SUCCESS) {
565 			pr_warn("%s: Failure %lld getting "
566 				"PHB#%x-PE#%x state\n",
567 				__func__, rc,
568 				phb->hose->global_number, slave->pe_number);
569 			return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
570 		}
571 
572 		/*
573 		 * Override the result based on the ascending
574 		 * priority.
575 		 */
576 		if (fstate > state)
577 			state = fstate;
578 	}
579 
580 	return state;
581 }
582 
583 struct pnv_ioda_pe *pnv_pci_bdfn_to_pe(struct pnv_phb *phb, u16 bdfn)
584 {
585 	int pe_number = phb->ioda.pe_rmap[bdfn];
586 
587 	if (pe_number == IODA_INVALID_PE)
588 		return NULL;
589 
590 	return &phb->ioda.pe_array[pe_number];
591 }
592 
593 struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
594 {
595 	struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
596 	struct pci_dn *pdn = pci_get_pdn(dev);
597 
598 	if (!pdn)
599 		return NULL;
600 	if (pdn->pe_number == IODA_INVALID_PE)
601 		return NULL;
602 	return &phb->ioda.pe_array[pdn->pe_number];
603 }
604 
605 static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
606 				  struct pnv_ioda_pe *parent,
607 				  struct pnv_ioda_pe *child,
608 				  bool is_add)
609 {
610 	const char *desc = is_add ? "adding" : "removing";
611 	uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
612 			      OPAL_REMOVE_PE_FROM_DOMAIN;
613 	struct pnv_ioda_pe *slave;
614 	long rc;
615 
616 	/* Parent PE affects child PE */
617 	rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
618 				child->pe_number, op);
619 	if (rc != OPAL_SUCCESS) {
620 		pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
621 			rc, desc);
622 		return -ENXIO;
623 	}
624 
625 	if (!(child->flags & PNV_IODA_PE_MASTER))
626 		return 0;
627 
628 	/* Compound case: parent PE affects slave PEs */
629 	list_for_each_entry(slave, &child->slaves, list) {
630 		rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
631 					slave->pe_number, op);
632 		if (rc != OPAL_SUCCESS) {
633 			pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
634 				rc, desc);
635 			return -ENXIO;
636 		}
637 	}
638 
639 	return 0;
640 }
641 
642 static int pnv_ioda_set_peltv(struct pnv_phb *phb,
643 			      struct pnv_ioda_pe *pe,
644 			      bool is_add)
645 {
646 	struct pnv_ioda_pe *slave;
647 	struct pci_dev *pdev = NULL;
648 	int ret;
649 
650 	/*
651 	 * Clear PE frozen state. If it's master PE, we need
652 	 * clear slave PE frozen state as well.
653 	 */
654 	if (is_add) {
655 		opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
656 					  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
657 		if (pe->flags & PNV_IODA_PE_MASTER) {
658 			list_for_each_entry(slave, &pe->slaves, list)
659 				opal_pci_eeh_freeze_clear(phb->opal_id,
660 							  slave->pe_number,
661 							  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
662 		}
663 	}
664 
665 	/*
666 	 * Associate PE in PELT. We need add the PE into the
667 	 * corresponding PELT-V as well. Otherwise, the error
668 	 * originated from the PE might contribute to other
669 	 * PEs.
670 	 */
671 	ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
672 	if (ret)
673 		return ret;
674 
675 	/* For compound PEs, any one affects all of them */
676 	if (pe->flags & PNV_IODA_PE_MASTER) {
677 		list_for_each_entry(slave, &pe->slaves, list) {
678 			ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
679 			if (ret)
680 				return ret;
681 		}
682 	}
683 
684 	if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
685 		pdev = pe->pbus->self;
686 	else if (pe->flags & PNV_IODA_PE_DEV)
687 		pdev = pe->pdev->bus->self;
688 #ifdef CONFIG_PCI_IOV
689 	else if (pe->flags & PNV_IODA_PE_VF)
690 		pdev = pe->parent_dev;
691 #endif /* CONFIG_PCI_IOV */
692 	while (pdev) {
693 		struct pci_dn *pdn = pci_get_pdn(pdev);
694 		struct pnv_ioda_pe *parent;
695 
696 		if (pdn && pdn->pe_number != IODA_INVALID_PE) {
697 			parent = &phb->ioda.pe_array[pdn->pe_number];
698 			ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
699 			if (ret)
700 				return ret;
701 		}
702 
703 		pdev = pdev->bus->self;
704 	}
705 
706 	return 0;
707 }
708 
709 static void pnv_ioda_unset_peltv(struct pnv_phb *phb,
710 				 struct pnv_ioda_pe *pe,
711 				 struct pci_dev *parent)
712 {
713 	int64_t rc;
714 
715 	while (parent) {
716 		struct pci_dn *pdn = pci_get_pdn(parent);
717 
718 		if (pdn && pdn->pe_number != IODA_INVALID_PE) {
719 			rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
720 						pe->pe_number,
721 						OPAL_REMOVE_PE_FROM_DOMAIN);
722 			/* XXX What to do in case of error ? */
723 		}
724 		parent = parent->bus->self;
725 	}
726 
727 	opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
728 				  OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
729 
730 	/* Disassociate PE in PELT */
731 	rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
732 				pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
733 	if (rc)
734 		pe_warn(pe, "OPAL error %lld remove self from PELTV\n", rc);
735 }
736 
737 int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
738 {
739 	struct pci_dev *parent;
740 	uint8_t bcomp, dcomp, fcomp;
741 	int64_t rc;
742 	long rid_end, rid;
743 
744 	/* Currently, we just deconfigure VF PE. Bus PE will always there.*/
745 	if (pe->pbus) {
746 		int count;
747 
748 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
749 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
750 		parent = pe->pbus->self;
751 		if (pe->flags & PNV_IODA_PE_BUS_ALL)
752 			count = resource_size(&pe->pbus->busn_res);
753 		else
754 			count = 1;
755 
756 		switch(count) {
757 		case  1: bcomp = OpalPciBusAll;         break;
758 		case  2: bcomp = OpalPciBus7Bits;       break;
759 		case  4: bcomp = OpalPciBus6Bits;       break;
760 		case  8: bcomp = OpalPciBus5Bits;       break;
761 		case 16: bcomp = OpalPciBus4Bits;       break;
762 		case 32: bcomp = OpalPciBus3Bits;       break;
763 		default:
764 			dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
765 			        count);
766 			/* Do an exact match only */
767 			bcomp = OpalPciBusAll;
768 		}
769 		rid_end = pe->rid + (count << 8);
770 	} else {
771 #ifdef CONFIG_PCI_IOV
772 		if (pe->flags & PNV_IODA_PE_VF)
773 			parent = pe->parent_dev;
774 		else
775 #endif
776 			parent = pe->pdev->bus->self;
777 		bcomp = OpalPciBusAll;
778 		dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
779 		fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
780 		rid_end = pe->rid + 1;
781 	}
782 
783 	/* Clear the reverse map */
784 	for (rid = pe->rid; rid < rid_end; rid++)
785 		phb->ioda.pe_rmap[rid] = IODA_INVALID_PE;
786 
787 	/*
788 	 * Release from all parents PELT-V. NPUs don't have a PELTV
789 	 * table
790 	 */
791 	if (phb->type != PNV_PHB_NPU_OCAPI)
792 		pnv_ioda_unset_peltv(phb, pe, parent);
793 
794 	rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
795 			     bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
796 	if (rc)
797 		pe_err(pe, "OPAL error %lld trying to setup PELT table\n", rc);
798 
799 	pe->pbus = NULL;
800 	pe->pdev = NULL;
801 #ifdef CONFIG_PCI_IOV
802 	pe->parent_dev = NULL;
803 #endif
804 
805 	return 0;
806 }
807 
808 int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
809 {
810 	uint8_t bcomp, dcomp, fcomp;
811 	long rc, rid_end, rid;
812 
813 	/* Bus validation ? */
814 	if (pe->pbus) {
815 		int count;
816 
817 		dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
818 		fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
819 		if (pe->flags & PNV_IODA_PE_BUS_ALL)
820 			count = resource_size(&pe->pbus->busn_res);
821 		else
822 			count = 1;
823 
824 		switch(count) {
825 		case  1: bcomp = OpalPciBusAll;		break;
826 		case  2: bcomp = OpalPciBus7Bits;	break;
827 		case  4: bcomp = OpalPciBus6Bits;	break;
828 		case  8: bcomp = OpalPciBus5Bits;	break;
829 		case 16: bcomp = OpalPciBus4Bits;	break;
830 		case 32: bcomp = OpalPciBus3Bits;	break;
831 		default:
832 			dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
833 			        count);
834 			/* Do an exact match only */
835 			bcomp = OpalPciBusAll;
836 		}
837 		rid_end = pe->rid + (count << 8);
838 	} else {
839 		bcomp = OpalPciBusAll;
840 		dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
841 		fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
842 		rid_end = pe->rid + 1;
843 	}
844 
845 	/*
846 	 * Associate PE in PELT. We need add the PE into the
847 	 * corresponding PELT-V as well. Otherwise, the error
848 	 * originated from the PE might contribute to other
849 	 * PEs.
850 	 */
851 	rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
852 			     bcomp, dcomp, fcomp, OPAL_MAP_PE);
853 	if (rc) {
854 		pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
855 		return -ENXIO;
856 	}
857 
858 	/*
859 	 * Configure PELTV. NPUs don't have a PELTV table so skip
860 	 * configuration on them.
861 	 */
862 	if (phb->type != PNV_PHB_NPU_OCAPI)
863 		pnv_ioda_set_peltv(phb, pe, true);
864 
865 	/* Setup reverse map */
866 	for (rid = pe->rid; rid < rid_end; rid++)
867 		phb->ioda.pe_rmap[rid] = pe->pe_number;
868 
869 	pe->mve_number = 0;
870 
871 	return 0;
872 }
873 
874 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
875 {
876 	struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
877 	struct pci_dn *pdn = pci_get_pdn(dev);
878 	struct pnv_ioda_pe *pe;
879 
880 	if (!pdn) {
881 		pr_err("%s: Device tree node not associated properly\n",
882 			   pci_name(dev));
883 		return NULL;
884 	}
885 	if (pdn->pe_number != IODA_INVALID_PE)
886 		return NULL;
887 
888 	pe = pnv_ioda_alloc_pe(phb, 1);
889 	if (!pe) {
890 		pr_warn("%s: Not enough PE# available, disabling device\n",
891 			pci_name(dev));
892 		return NULL;
893 	}
894 
895 	/* NOTE: We don't get a reference for the pointer in the PE
896 	 * data structure, both the device and PE structures should be
897 	 * destroyed at the same time.
898 	 *
899 	 * At some point we want to remove the PDN completely anyways
900 	 */
901 	pdn->pe_number = pe->pe_number;
902 	pe->flags = PNV_IODA_PE_DEV;
903 	pe->pdev = dev;
904 	pe->pbus = NULL;
905 	pe->mve_number = -1;
906 	pe->rid = dev->bus->number << 8 | pdn->devfn;
907 	pe->device_count++;
908 
909 	pe_info(pe, "Associated device to PE\n");
910 
911 	if (pnv_ioda_configure_pe(phb, pe)) {
912 		/* XXX What do we do here ? */
913 		pnv_ioda_free_pe(pe);
914 		pdn->pe_number = IODA_INVALID_PE;
915 		pe->pdev = NULL;
916 		return NULL;
917 	}
918 
919 	/* Put PE to the list */
920 	mutex_lock(&phb->ioda.pe_list_mutex);
921 	list_add_tail(&pe->list, &phb->ioda.pe_list);
922 	mutex_unlock(&phb->ioda.pe_list_mutex);
923 	return pe;
924 }
925 
926 /*
927  * There're 2 types of PCI bus sensitive PEs: One that is compromised of
928  * single PCI bus. Another one that contains the primary PCI bus and its
929  * subordinate PCI devices and buses. The second type of PE is normally
930  * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
931  */
932 static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
933 {
934 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
935 	struct pnv_ioda_pe *pe = NULL;
936 	unsigned int pe_num;
937 
938 	/*
939 	 * In partial hotplug case, the PE instance might be still alive.
940 	 * We should reuse it instead of allocating a new one.
941 	 */
942 	pe_num = phb->ioda.pe_rmap[bus->number << 8];
943 	if (WARN_ON(pe_num != IODA_INVALID_PE)) {
944 		pe = &phb->ioda.pe_array[pe_num];
945 		return NULL;
946 	}
947 
948 	/* PE number for root bus should have been reserved */
949 	if (pci_is_root_bus(bus))
950 		pe = &phb->ioda.pe_array[phb->ioda.root_pe_idx];
951 
952 	/* Check if PE is determined by M64 */
953 	if (!pe)
954 		pe = pnv_ioda_pick_m64_pe(bus, all);
955 
956 	/* The PE number isn't pinned by M64 */
957 	if (!pe)
958 		pe = pnv_ioda_alloc_pe(phb, 1);
959 
960 	if (!pe) {
961 		pr_warn("%s: Not enough PE# available for PCI bus %04x:%02x\n",
962 			__func__, pci_domain_nr(bus), bus->number);
963 		return NULL;
964 	}
965 
966 	pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
967 	pe->pbus = bus;
968 	pe->pdev = NULL;
969 	pe->mve_number = -1;
970 	pe->rid = bus->busn_res.start << 8;
971 
972 	if (all)
973 		pe_info(pe, "Secondary bus %pad..%pad associated with PE#%x\n",
974 			&bus->busn_res.start, &bus->busn_res.end,
975 			pe->pe_number);
976 	else
977 		pe_info(pe, "Secondary bus %pad associated with PE#%x\n",
978 			&bus->busn_res.start, pe->pe_number);
979 
980 	if (pnv_ioda_configure_pe(phb, pe)) {
981 		/* XXX What do we do here ? */
982 		pnv_ioda_free_pe(pe);
983 		pe->pbus = NULL;
984 		return NULL;
985 	}
986 
987 	/* Put PE to the list */
988 	list_add_tail(&pe->list, &phb->ioda.pe_list);
989 
990 	return pe;
991 }
992 
993 static void pnv_pci_ioda_dma_dev_setup(struct pci_dev *pdev)
994 {
995 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
996 	struct pci_dn *pdn = pci_get_pdn(pdev);
997 	struct pnv_ioda_pe *pe;
998 
999 	/* Check if the BDFN for this device is associated with a PE yet */
1000 	pe = pnv_pci_bdfn_to_pe(phb, pdev->devfn | (pdev->bus->number << 8));
1001 	if (!pe) {
1002 		/* VF PEs should be pre-configured in pnv_pci_sriov_enable() */
1003 		if (WARN_ON(pdev->is_virtfn))
1004 			return;
1005 
1006 		pnv_pci_configure_bus(pdev->bus);
1007 		pe = pnv_pci_bdfn_to_pe(phb, pdev->devfn | (pdev->bus->number << 8));
1008 		pci_info(pdev, "Configured PE#%x\n", pe ? pe->pe_number : 0xfffff);
1009 
1010 
1011 		/*
1012 		 * If we can't setup the IODA PE something has gone horribly
1013 		 * wrong and we can't enable DMA for the device.
1014 		 */
1015 		if (WARN_ON(!pe))
1016 			return;
1017 	} else {
1018 		pci_info(pdev, "Added to existing PE#%x\n", pe->pe_number);
1019 	}
1020 
1021 	/*
1022 	 * We assume that bridges *probably* don't need to do any DMA so we can
1023 	 * skip allocating a TCE table, etc unless we get a non-bridge device.
1024 	 */
1025 	if (!pe->dma_setup_done && !pci_is_bridge(pdev)) {
1026 		switch (phb->type) {
1027 		case PNV_PHB_IODA2:
1028 			pnv_pci_ioda2_setup_dma_pe(phb, pe);
1029 			break;
1030 		default:
1031 			pr_warn("%s: No DMA for PHB#%x (type %d)\n",
1032 				__func__, phb->hose->global_number, phb->type);
1033 		}
1034 	}
1035 
1036 	if (pdn)
1037 		pdn->pe_number = pe->pe_number;
1038 	pe->device_count++;
1039 
1040 	WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
1041 	pdev->dev.archdata.dma_offset = pe->tce_bypass_base;
1042 	set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
1043 
1044 	/* PEs with a DMA weight of zero won't have a group */
1045 	if (pe->table_group.group)
1046 		iommu_add_device(&pe->table_group, &pdev->dev);
1047 }
1048 
1049 /*
1050  * Reconfigure TVE#0 to be usable as 64-bit DMA space.
1051  *
1052  * The first 4GB of virtual memory for a PE is reserved for 32-bit accesses.
1053  * Devices can only access more than that if bit 59 of the PCI address is set
1054  * by hardware, which indicates TVE#1 should be used instead of TVE#0.
1055  * Many PCI devices are not capable of addressing that many bits, and as a
1056  * result are limited to the 4GB of virtual memory made available to 32-bit
1057  * devices in TVE#0.
1058  *
1059  * In order to work around this, reconfigure TVE#0 to be suitable for 64-bit
1060  * devices by configuring the virtual memory past the first 4GB inaccessible
1061  * by 64-bit DMAs.  This should only be used by devices that want more than
1062  * 4GB, and only on PEs that have no 32-bit devices.
1063  *
1064  * Currently this will only work on PHB3 (POWER8).
1065  */
1066 static int pnv_pci_ioda_dma_64bit_bypass(struct pnv_ioda_pe *pe)
1067 {
1068 	u64 window_size, table_size, tce_count, addr;
1069 	struct page *table_pages;
1070 	u64 tce_order = 28; /* 256MB TCEs */
1071 	__be64 *tces;
1072 	s64 rc;
1073 
1074 	/*
1075 	 * Window size needs to be a power of two, but needs to account for
1076 	 * shifting memory by the 4GB offset required to skip 32bit space.
1077 	 */
1078 	window_size = roundup_pow_of_two(memory_hotplug_max() + (1ULL << 32));
1079 	tce_count = window_size >> tce_order;
1080 	table_size = tce_count << 3;
1081 
1082 	if (table_size < PAGE_SIZE)
1083 		table_size = PAGE_SIZE;
1084 
1085 	table_pages = alloc_pages_node(pe->phb->hose->node, GFP_KERNEL,
1086 				       get_order(table_size));
1087 	if (!table_pages)
1088 		goto err;
1089 
1090 	tces = page_address(table_pages);
1091 	if (!tces)
1092 		goto err;
1093 
1094 	memset(tces, 0, table_size);
1095 
1096 	for (addr = 0; addr < memory_hotplug_max(); addr += (1 << tce_order)) {
1097 		tces[(addr + (1ULL << 32)) >> tce_order] =
1098 			cpu_to_be64(addr | TCE_PCI_READ | TCE_PCI_WRITE);
1099 	}
1100 
1101 	rc = opal_pci_map_pe_dma_window(pe->phb->opal_id,
1102 					pe->pe_number,
1103 					/* reconfigure window 0 */
1104 					(pe->pe_number << 1) + 0,
1105 					1,
1106 					__pa(tces),
1107 					table_size,
1108 					1 << tce_order);
1109 	if (rc == OPAL_SUCCESS) {
1110 		pe_info(pe, "Using 64-bit DMA iommu bypass (through TVE#0)\n");
1111 		return 0;
1112 	}
1113 err:
1114 	pe_err(pe, "Error configuring 64-bit DMA bypass\n");
1115 	return -EIO;
1116 }
1117 
1118 static bool pnv_pci_ioda_iommu_bypass_supported(struct pci_dev *pdev,
1119 		u64 dma_mask)
1120 {
1121 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
1122 	struct pci_dn *pdn = pci_get_pdn(pdev);
1123 	struct pnv_ioda_pe *pe;
1124 
1125 	if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
1126 		return false;
1127 
1128 	pe = &phb->ioda.pe_array[pdn->pe_number];
1129 	if (pe->tce_bypass_enabled) {
1130 		u64 top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
1131 		if (dma_mask >= top)
1132 			return true;
1133 	}
1134 
1135 	/*
1136 	 * If the device can't set the TCE bypass bit but still wants
1137 	 * to access 4GB or more, on PHB3 we can reconfigure TVE#0 to
1138 	 * bypass the 32-bit region and be usable for 64-bit DMAs.
1139 	 * The device needs to be able to address all of this space.
1140 	 */
1141 	if (dma_mask >> 32 &&
1142 	    dma_mask > (memory_hotplug_max() + (1ULL << 32)) &&
1143 	    /* pe->pdev should be set if it's a single device, pe->pbus if not */
1144 	    (pe->device_count == 1 || !pe->pbus) &&
1145 	    phb->model == PNV_PHB_MODEL_PHB3) {
1146 		/* Configure the bypass mode */
1147 		s64 rc = pnv_pci_ioda_dma_64bit_bypass(pe);
1148 		if (rc)
1149 			return false;
1150 		/* 4GB offset bypasses 32-bit space */
1151 		pdev->dev.archdata.dma_offset = (1ULL << 32);
1152 		return true;
1153 	}
1154 
1155 	return false;
1156 }
1157 
1158 static inline __be64 __iomem *pnv_ioda_get_inval_reg(struct pnv_phb *phb)
1159 {
1160 	return phb->regs + 0x210;
1161 }
1162 
1163 #ifdef CONFIG_IOMMU_API
1164 /* Common for IODA1 and IODA2 */
1165 static int pnv_ioda_tce_xchg_no_kill(struct iommu_table *tbl, long index,
1166 		unsigned long *hpa, enum dma_data_direction *direction)
1167 {
1168 	return pnv_tce_xchg(tbl, index, hpa, direction);
1169 }
1170 #endif
1171 
1172 #define PHB3_TCE_KILL_INVAL_ALL		PPC_BIT(0)
1173 #define PHB3_TCE_KILL_INVAL_PE		PPC_BIT(1)
1174 #define PHB3_TCE_KILL_INVAL_ONE		PPC_BIT(2)
1175 
1176 static inline void pnv_pci_phb3_tce_invalidate_pe(struct pnv_ioda_pe *pe)
1177 {
1178 	/* 01xb - invalidate TCEs that match the specified PE# */
1179 	__be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb);
1180 	unsigned long val = PHB3_TCE_KILL_INVAL_PE | (pe->pe_number & 0xFF);
1181 
1182 	mb(); /* Ensure above stores are visible */
1183 	__raw_writeq_be(val, invalidate);
1184 }
1185 
1186 static void pnv_pci_phb3_tce_invalidate(struct pnv_ioda_pe *pe,
1187 					unsigned shift, unsigned long index,
1188 					unsigned long npages)
1189 {
1190 	__be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb);
1191 	unsigned long start, end, inc;
1192 
1193 	/* We'll invalidate DMA address in PE scope */
1194 	start = PHB3_TCE_KILL_INVAL_ONE;
1195 	start |= (pe->pe_number & 0xFF);
1196 	end = start;
1197 
1198 	/* Figure out the start, end and step */
1199 	start |= (index << shift);
1200 	end |= ((index + npages - 1) << shift);
1201 	inc = (0x1ull << shift);
1202 	mb();
1203 
1204 	while (start <= end) {
1205 		__raw_writeq_be(start, invalidate);
1206 		start += inc;
1207 	}
1208 }
1209 
1210 static inline void pnv_pci_ioda2_tce_invalidate_pe(struct pnv_ioda_pe *pe)
1211 {
1212 	struct pnv_phb *phb = pe->phb;
1213 
1214 	if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
1215 		pnv_pci_phb3_tce_invalidate_pe(pe);
1216 	else
1217 		opal_pci_tce_kill(phb->opal_id, OPAL_PCI_TCE_KILL_PE,
1218 				  pe->pe_number, 0, 0, 0);
1219 }
1220 
1221 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
1222 		unsigned long index, unsigned long npages)
1223 {
1224 	struct iommu_table_group_link *tgl;
1225 
1226 	list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {
1227 		struct pnv_ioda_pe *pe = container_of(tgl->table_group,
1228 				struct pnv_ioda_pe, table_group);
1229 		struct pnv_phb *phb = pe->phb;
1230 		unsigned int shift = tbl->it_page_shift;
1231 
1232 		if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
1233 			pnv_pci_phb3_tce_invalidate(pe, shift,
1234 						    index, npages);
1235 		else
1236 			opal_pci_tce_kill(phb->opal_id,
1237 					  OPAL_PCI_TCE_KILL_PAGES,
1238 					  pe->pe_number, 1u << shift,
1239 					  index << shift, npages);
1240 	}
1241 }
1242 
1243 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
1244 		long npages, unsigned long uaddr,
1245 		enum dma_data_direction direction,
1246 		unsigned long attrs)
1247 {
1248 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
1249 			attrs);
1250 
1251 	if (!ret)
1252 		pnv_pci_ioda2_tce_invalidate(tbl, index, npages);
1253 
1254 	return ret;
1255 }
1256 
1257 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
1258 		long npages)
1259 {
1260 	pnv_tce_free(tbl, index, npages);
1261 
1262 	pnv_pci_ioda2_tce_invalidate(tbl, index, npages);
1263 }
1264 
1265 static struct iommu_table_ops pnv_ioda2_iommu_ops = {
1266 	.set = pnv_ioda2_tce_build,
1267 #ifdef CONFIG_IOMMU_API
1268 	.xchg_no_kill = pnv_ioda_tce_xchg_no_kill,
1269 	.tce_kill = pnv_pci_ioda2_tce_invalidate,
1270 	.useraddrptr = pnv_tce_useraddrptr,
1271 #endif
1272 	.clear = pnv_ioda2_tce_free,
1273 	.get = pnv_tce_get,
1274 	.free = pnv_pci_ioda2_table_free_pages,
1275 };
1276 
1277 static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
1278 		int num, struct iommu_table *tbl)
1279 {
1280 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1281 			table_group);
1282 	struct pnv_phb *phb = pe->phb;
1283 	int64_t rc;
1284 	const unsigned long size = tbl->it_indirect_levels ?
1285 			tbl->it_level_size : tbl->it_size;
1286 	const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
1287 	const __u64 win_size = tbl->it_size << tbl->it_page_shift;
1288 
1289 	pe_info(pe, "Setting up window#%d %llx..%llx pg=%lx\n",
1290 		num, start_addr, start_addr + win_size - 1,
1291 		IOMMU_PAGE_SIZE(tbl));
1292 
1293 	/*
1294 	 * Map TCE table through TVT. The TVE index is the PE number
1295 	 * shifted by 1 bit for 32-bits DMA space.
1296 	 */
1297 	rc = opal_pci_map_pe_dma_window(phb->opal_id,
1298 			pe->pe_number,
1299 			(pe->pe_number << 1) + num,
1300 			tbl->it_indirect_levels + 1,
1301 			__pa(tbl->it_base),
1302 			size << 3,
1303 			IOMMU_PAGE_SIZE(tbl));
1304 	if (rc) {
1305 		pe_err(pe, "Failed to configure TCE table, err %lld\n", rc);
1306 		return rc;
1307 	}
1308 
1309 	pnv_pci_link_table_and_group(phb->hose->node, num,
1310 			tbl, &pe->table_group);
1311 	pnv_pci_ioda2_tce_invalidate_pe(pe);
1312 
1313 	return 0;
1314 }
1315 
1316 static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
1317 {
1318 	uint16_t window_id = (pe->pe_number << 1 ) + 1;
1319 	int64_t rc;
1320 
1321 	pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
1322 	if (enable) {
1323 		phys_addr_t top = memblock_end_of_DRAM();
1324 
1325 		top = roundup_pow_of_two(top);
1326 		rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1327 						     pe->pe_number,
1328 						     window_id,
1329 						     pe->tce_bypass_base,
1330 						     top);
1331 	} else {
1332 		rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
1333 						     pe->pe_number,
1334 						     window_id,
1335 						     pe->tce_bypass_base,
1336 						     0);
1337 	}
1338 	if (rc)
1339 		pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
1340 	else
1341 		pe->tce_bypass_enabled = enable;
1342 }
1343 
1344 static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
1345 		int num, __u32 page_shift, __u64 window_size, __u32 levels,
1346 		bool alloc_userspace_copy, struct iommu_table **ptbl)
1347 {
1348 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1349 			table_group);
1350 	int nid = pe->phb->hose->node;
1351 	__u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
1352 	long ret;
1353 	struct iommu_table *tbl;
1354 
1355 	tbl = pnv_pci_table_alloc(nid);
1356 	if (!tbl)
1357 		return -ENOMEM;
1358 
1359 	tbl->it_ops = &pnv_ioda2_iommu_ops;
1360 
1361 	ret = pnv_pci_ioda2_table_alloc_pages(nid,
1362 			bus_offset, page_shift, window_size,
1363 			levels, alloc_userspace_copy, tbl);
1364 	if (ret) {
1365 		iommu_tce_table_put(tbl);
1366 		return ret;
1367 	}
1368 
1369 	*ptbl = tbl;
1370 
1371 	return 0;
1372 }
1373 
1374 static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
1375 {
1376 	struct iommu_table *tbl = NULL;
1377 	long rc;
1378 	unsigned long res_start, res_end;
1379 
1380 	/*
1381 	 * crashkernel= specifies the kdump kernel's maximum memory at
1382 	 * some offset and there is no guaranteed the result is a power
1383 	 * of 2, which will cause errors later.
1384 	 */
1385 	const u64 max_memory = __rounddown_pow_of_two(memory_hotplug_max());
1386 
1387 	/*
1388 	 * In memory constrained environments, e.g. kdump kernel, the
1389 	 * DMA window can be larger than available memory, which will
1390 	 * cause errors later.
1391 	 */
1392 	const u64 maxblock = 1UL << (PAGE_SHIFT + MAX_ORDER);
1393 
1394 	/*
1395 	 * We create the default window as big as we can. The constraint is
1396 	 * the max order of allocation possible. The TCE table is likely to
1397 	 * end up being multilevel and with on-demand allocation in place,
1398 	 * the initial use is not going to be huge as the default window aims
1399 	 * to support crippled devices (i.e. not fully 64bit DMAble) only.
1400 	 */
1401 	/* iommu_table::it_map uses 1 bit per IOMMU page, hence 8 */
1402 	const u64 window_size = min((maxblock * 8) << PAGE_SHIFT, max_memory);
1403 	/* Each TCE level cannot exceed maxblock so go multilevel if needed */
1404 	unsigned long tces_order = ilog2(window_size >> PAGE_SHIFT);
1405 	unsigned long tcelevel_order = ilog2(maxblock >> 3);
1406 	unsigned int levels = tces_order / tcelevel_order;
1407 
1408 	if (tces_order % tcelevel_order)
1409 		levels += 1;
1410 	/*
1411 	 * We try to stick to default levels (which is >1 at the moment) in
1412 	 * order to save memory by relying on on-demain TCE level allocation.
1413 	 */
1414 	levels = max_t(unsigned int, levels, POWERNV_IOMMU_DEFAULT_LEVELS);
1415 
1416 	rc = pnv_pci_ioda2_create_table(&pe->table_group, 0, PAGE_SHIFT,
1417 			window_size, levels, false, &tbl);
1418 	if (rc) {
1419 		pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
1420 				rc);
1421 		return rc;
1422 	}
1423 
1424 	/* We use top part of 32bit space for MMIO so exclude it from DMA */
1425 	res_start = 0;
1426 	res_end = 0;
1427 	if (window_size > pe->phb->ioda.m32_pci_base) {
1428 		res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift;
1429 		res_end = min(window_size, SZ_4G) >> tbl->it_page_shift;
1430 	}
1431 
1432 	tbl->it_index = (pe->phb->hose->global_number << 16) | pe->pe_number;
1433 	if (iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end))
1434 		rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
1435 	else
1436 		rc = -ENOMEM;
1437 	if (rc) {
1438 		pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n", rc);
1439 		iommu_tce_table_put(tbl);
1440 		tbl = NULL; /* This clears iommu_table_base below */
1441 	}
1442 	if (!pnv_iommu_bypass_disabled)
1443 		pnv_pci_ioda2_set_bypass(pe, true);
1444 
1445 	/*
1446 	 * Set table base for the case of IOMMU DMA use. Usually this is done
1447 	 * from dma_dev_setup() which is not called when a device is returned
1448 	 * from VFIO so do it here.
1449 	 */
1450 	if (pe->pdev)
1451 		set_iommu_table_base(&pe->pdev->dev, tbl);
1452 
1453 	return 0;
1454 }
1455 
1456 static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1457 		int num)
1458 {
1459 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1460 			table_group);
1461 	struct pnv_phb *phb = pe->phb;
1462 	long ret;
1463 
1464 	pe_info(pe, "Removing DMA window #%d\n", num);
1465 
1466 	ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
1467 			(pe->pe_number << 1) + num,
1468 			0/* levels */, 0/* table address */,
1469 			0/* table size */, 0/* page size */);
1470 	if (ret)
1471 		pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
1472 	else
1473 		pnv_pci_ioda2_tce_invalidate_pe(pe);
1474 
1475 	pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
1476 
1477 	return ret;
1478 }
1479 
1480 #ifdef CONFIG_IOMMU_API
1481 unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
1482 		__u64 window_size, __u32 levels)
1483 {
1484 	unsigned long bytes = 0;
1485 	const unsigned window_shift = ilog2(window_size);
1486 	unsigned entries_shift = window_shift - page_shift;
1487 	unsigned table_shift = entries_shift + 3;
1488 	unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
1489 	unsigned long direct_table_size;
1490 
1491 	if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
1492 			!is_power_of_2(window_size))
1493 		return 0;
1494 
1495 	/* Calculate a direct table size from window_size and levels */
1496 	entries_shift = (entries_shift + levels - 1) / levels;
1497 	table_shift = entries_shift + 3;
1498 	table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
1499 	direct_table_size =  1UL << table_shift;
1500 
1501 	for ( ; levels; --levels) {
1502 		bytes += ALIGN(tce_table_size, direct_table_size);
1503 
1504 		tce_table_size /= direct_table_size;
1505 		tce_table_size <<= 3;
1506 		tce_table_size = max_t(unsigned long,
1507 				tce_table_size, direct_table_size);
1508 	}
1509 
1510 	return bytes + bytes; /* one for HW table, one for userspace copy */
1511 }
1512 
1513 static long pnv_pci_ioda2_create_table_userspace(
1514 		struct iommu_table_group *table_group,
1515 		int num, __u32 page_shift, __u64 window_size, __u32 levels,
1516 		struct iommu_table **ptbl)
1517 {
1518 	long ret = pnv_pci_ioda2_create_table(table_group,
1519 			num, page_shift, window_size, levels, true, ptbl);
1520 
1521 	if (!ret)
1522 		(*ptbl)->it_allocated_size = pnv_pci_ioda2_get_table_size(
1523 				page_shift, window_size, levels);
1524 	return ret;
1525 }
1526 
1527 static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
1528 {
1529 	struct pci_dev *dev;
1530 
1531 	list_for_each_entry(dev, &bus->devices, bus_list) {
1532 		set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
1533 		dev->dev.archdata.dma_offset = pe->tce_bypass_base;
1534 
1535 		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
1536 			pnv_ioda_setup_bus_dma(pe, dev->subordinate);
1537 	}
1538 }
1539 
1540 static long pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
1541 {
1542 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1543 						table_group);
1544 	/* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
1545 	struct iommu_table *tbl = pe->table_group.tables[0];
1546 
1547 	/*
1548 	 * iommu_ops transfers the ownership per a device and we mode
1549 	 * the group ownership with the first device in the group.
1550 	 */
1551 	if (!tbl)
1552 		return 0;
1553 
1554 	pnv_pci_ioda2_set_bypass(pe, false);
1555 	pnv_pci_ioda2_unset_window(&pe->table_group, 0);
1556 	if (pe->pbus)
1557 		pnv_ioda_setup_bus_dma(pe, pe->pbus);
1558 	else if (pe->pdev)
1559 		set_iommu_table_base(&pe->pdev->dev, NULL);
1560 	iommu_tce_table_put(tbl);
1561 
1562 	return 0;
1563 }
1564 
1565 static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
1566 {
1567 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
1568 						table_group);
1569 
1570 	/* See the comment about iommu_ops above */
1571 	if (pe->table_group.tables[0])
1572 		return;
1573 	pnv_pci_ioda2_setup_default_config(pe);
1574 	if (pe->pbus)
1575 		pnv_ioda_setup_bus_dma(pe, pe->pbus);
1576 }
1577 
1578 static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
1579 	.get_table_size = pnv_pci_ioda2_get_table_size,
1580 	.create_table = pnv_pci_ioda2_create_table_userspace,
1581 	.set_window = pnv_pci_ioda2_set_window,
1582 	.unset_window = pnv_pci_ioda2_unset_window,
1583 	.take_ownership = pnv_ioda2_take_ownership,
1584 	.release_ownership = pnv_ioda2_release_ownership,
1585 };
1586 #endif
1587 
1588 void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
1589 				struct pnv_ioda_pe *pe)
1590 {
1591 	int64_t rc;
1592 
1593 	/* TVE #1 is selected by PCI address bit 59 */
1594 	pe->tce_bypass_base = 1ull << 59;
1595 
1596 	/* The PE will reserve all possible 32-bits space */
1597 	pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
1598 		phb->ioda.m32_pci_base);
1599 
1600 	/* Setup linux iommu table */
1601 	pe->table_group.tce32_start = 0;
1602 	pe->table_group.tce32_size = phb->ioda.m32_pci_base;
1603 	pe->table_group.max_dynamic_windows_supported =
1604 			IOMMU_TABLE_GROUP_MAX_TABLES;
1605 	pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
1606 	pe->table_group.pgsizes = pnv_ioda_parse_tce_sizes(phb);
1607 
1608 	rc = pnv_pci_ioda2_setup_default_config(pe);
1609 	if (rc)
1610 		return;
1611 
1612 #ifdef CONFIG_IOMMU_API
1613 	pe->table_group.ops = &pnv_pci_ioda2_ops;
1614 	iommu_register_group(&pe->table_group, phb->hose->global_number,
1615 			     pe->pe_number);
1616 #endif
1617 	pe->dma_setup_done = true;
1618 }
1619 
1620 /*
1621  * Called from KVM in real mode to EOI passthru interrupts. The ICP
1622  * EOI is handled directly in KVM in kvmppc_deliver_irq_passthru().
1623  *
1624  * The IRQ data is mapped in the PCI-MSI domain and the EOI OPAL call
1625  * needs an HW IRQ number mapped in the XICS IRQ domain. The HW IRQ
1626  * numbers of the in-the-middle MSI domain are vector numbers and it's
1627  * good enough for OPAL. Use that.
1628  */
1629 int64_t pnv_opal_pci_msi_eoi(struct irq_data *d)
1630 {
1631 	struct pci_controller *hose = irq_data_get_irq_chip_data(d->parent_data);
1632 	struct pnv_phb *phb = hose->private_data;
1633 
1634 	return opal_pci_msi_eoi(phb->opal_id, d->parent_data->hwirq);
1635 }
1636 
1637 /*
1638  * The IRQ data is mapped in the XICS domain, with OPAL HW IRQ numbers
1639  */
1640 static void pnv_ioda2_msi_eoi(struct irq_data *d)
1641 {
1642 	int64_t rc;
1643 	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
1644 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
1645 	struct pnv_phb *phb = hose->private_data;
1646 
1647 	rc = opal_pci_msi_eoi(phb->opal_id, hw_irq);
1648 	WARN_ON_ONCE(rc);
1649 
1650 	icp_native_eoi(d);
1651 }
1652 
1653 /* P8/CXL only */
1654 void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
1655 {
1656 	struct irq_data *idata;
1657 	struct irq_chip *ichip;
1658 
1659 	/* The MSI EOI OPAL call is only needed on PHB3 */
1660 	if (phb->model != PNV_PHB_MODEL_PHB3)
1661 		return;
1662 
1663 	if (!phb->ioda.irq_chip_init) {
1664 		/*
1665 		 * First time we setup an MSI IRQ, we need to setup the
1666 		 * corresponding IRQ chip to route correctly.
1667 		 */
1668 		idata = irq_get_irq_data(virq);
1669 		ichip = irq_data_get_irq_chip(idata);
1670 		phb->ioda.irq_chip_init = 1;
1671 		phb->ioda.irq_chip = *ichip;
1672 		phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
1673 	}
1674 	irq_set_chip(virq, &phb->ioda.irq_chip);
1675 	irq_set_chip_data(virq, phb->hose);
1676 }
1677 
1678 static struct irq_chip pnv_pci_msi_irq_chip;
1679 
1680 /*
1681  * Returns true iff chip is something that we could call
1682  * pnv_opal_pci_msi_eoi for.
1683  */
1684 bool is_pnv_opal_msi(struct irq_chip *chip)
1685 {
1686 	return chip == &pnv_pci_msi_irq_chip;
1687 }
1688 EXPORT_SYMBOL_GPL(is_pnv_opal_msi);
1689 
1690 static int __pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
1691 				    unsigned int xive_num,
1692 				    unsigned int is_64, struct msi_msg *msg)
1693 {
1694 	struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
1695 	__be32 data;
1696 	int rc;
1697 
1698 	dev_dbg(&dev->dev, "%s: setup %s-bit MSI for vector #%d\n", __func__,
1699 		is_64 ? "64" : "32", xive_num);
1700 
1701 	/* No PE assigned ? bail out ... no MSI for you ! */
1702 	if (pe == NULL)
1703 		return -ENXIO;
1704 
1705 	/* Check if we have an MVE */
1706 	if (pe->mve_number < 0)
1707 		return -ENXIO;
1708 
1709 	/* Force 32-bit MSI on some broken devices */
1710 	if (dev->no_64bit_msi)
1711 		is_64 = 0;
1712 
1713 	/* Assign XIVE to PE */
1714 	rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
1715 	if (rc) {
1716 		pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
1717 			pci_name(dev), rc, xive_num);
1718 		return -EIO;
1719 	}
1720 
1721 	if (is_64) {
1722 		__be64 addr64;
1723 
1724 		rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
1725 				     &addr64, &data);
1726 		if (rc) {
1727 			pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
1728 				pci_name(dev), rc);
1729 			return -EIO;
1730 		}
1731 		msg->address_hi = be64_to_cpu(addr64) >> 32;
1732 		msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
1733 	} else {
1734 		__be32 addr32;
1735 
1736 		rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
1737 				     &addr32, &data);
1738 		if (rc) {
1739 			pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
1740 				pci_name(dev), rc);
1741 			return -EIO;
1742 		}
1743 		msg->address_hi = 0;
1744 		msg->address_lo = be32_to_cpu(addr32);
1745 	}
1746 	msg->data = be32_to_cpu(data);
1747 
1748 	return 0;
1749 }
1750 
1751 /*
1752  * The msi_free() op is called before irq_domain_free_irqs_top() when
1753  * the handler data is still available. Use that to clear the XIVE
1754  * controller.
1755  */
1756 static void pnv_msi_ops_msi_free(struct irq_domain *domain,
1757 				 struct msi_domain_info *info,
1758 				 unsigned int irq)
1759 {
1760 	if (xive_enabled())
1761 		xive_irq_free_data(irq);
1762 }
1763 
1764 static struct msi_domain_ops pnv_pci_msi_domain_ops = {
1765 	.msi_free	= pnv_msi_ops_msi_free,
1766 };
1767 
1768 static void pnv_msi_shutdown(struct irq_data *d)
1769 {
1770 	d = d->parent_data;
1771 	if (d->chip->irq_shutdown)
1772 		d->chip->irq_shutdown(d);
1773 }
1774 
1775 static void pnv_msi_mask(struct irq_data *d)
1776 {
1777 	pci_msi_mask_irq(d);
1778 	irq_chip_mask_parent(d);
1779 }
1780 
1781 static void pnv_msi_unmask(struct irq_data *d)
1782 {
1783 	pci_msi_unmask_irq(d);
1784 	irq_chip_unmask_parent(d);
1785 }
1786 
1787 static struct irq_chip pnv_pci_msi_irq_chip = {
1788 	.name		= "PNV-PCI-MSI",
1789 	.irq_shutdown	= pnv_msi_shutdown,
1790 	.irq_mask	= pnv_msi_mask,
1791 	.irq_unmask	= pnv_msi_unmask,
1792 	.irq_eoi	= irq_chip_eoi_parent,
1793 };
1794 
1795 static struct msi_domain_info pnv_msi_domain_info = {
1796 	.flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1797 		  MSI_FLAG_MULTI_PCI_MSI  | MSI_FLAG_PCI_MSIX),
1798 	.ops   = &pnv_pci_msi_domain_ops,
1799 	.chip  = &pnv_pci_msi_irq_chip,
1800 };
1801 
1802 static void pnv_msi_compose_msg(struct irq_data *d, struct msi_msg *msg)
1803 {
1804 	struct msi_desc *entry = irq_data_get_msi_desc(d);
1805 	struct pci_dev *pdev = msi_desc_to_pci_dev(entry);
1806 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
1807 	struct pnv_phb *phb = hose->private_data;
1808 	int rc;
1809 
1810 	rc = __pnv_pci_ioda_msi_setup(phb, pdev, d->hwirq,
1811 				      entry->pci.msi_attrib.is_64, msg);
1812 	if (rc)
1813 		dev_err(&pdev->dev, "Failed to setup %s-bit MSI #%ld : %d\n",
1814 			entry->pci.msi_attrib.is_64 ? "64" : "32", d->hwirq, rc);
1815 }
1816 
1817 /*
1818  * The IRQ data is mapped in the MSI domain in which HW IRQ numbers
1819  * correspond to vector numbers.
1820  */
1821 static void pnv_msi_eoi(struct irq_data *d)
1822 {
1823 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
1824 	struct pnv_phb *phb = hose->private_data;
1825 
1826 	if (phb->model == PNV_PHB_MODEL_PHB3) {
1827 		/*
1828 		 * The EOI OPAL call takes an OPAL HW IRQ number but
1829 		 * since it is translated into a vector number in
1830 		 * OPAL, use that directly.
1831 		 */
1832 		WARN_ON_ONCE(opal_pci_msi_eoi(phb->opal_id, d->hwirq));
1833 	}
1834 
1835 	irq_chip_eoi_parent(d);
1836 }
1837 
1838 static struct irq_chip pnv_msi_irq_chip = {
1839 	.name			= "PNV-MSI",
1840 	.irq_shutdown		= pnv_msi_shutdown,
1841 	.irq_mask		= irq_chip_mask_parent,
1842 	.irq_unmask		= irq_chip_unmask_parent,
1843 	.irq_eoi		= pnv_msi_eoi,
1844 	.irq_set_affinity	= irq_chip_set_affinity_parent,
1845 	.irq_compose_msi_msg	= pnv_msi_compose_msg,
1846 };
1847 
1848 static int pnv_irq_parent_domain_alloc(struct irq_domain *domain,
1849 				       unsigned int virq, int hwirq)
1850 {
1851 	struct irq_fwspec parent_fwspec;
1852 	int ret;
1853 
1854 	parent_fwspec.fwnode = domain->parent->fwnode;
1855 	parent_fwspec.param_count = 2;
1856 	parent_fwspec.param[0] = hwirq;
1857 	parent_fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
1858 
1859 	ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &parent_fwspec);
1860 	if (ret)
1861 		return ret;
1862 
1863 	return 0;
1864 }
1865 
1866 static int pnv_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1867 				unsigned int nr_irqs, void *arg)
1868 {
1869 	struct pci_controller *hose = domain->host_data;
1870 	struct pnv_phb *phb = hose->private_data;
1871 	msi_alloc_info_t *info = arg;
1872 	struct pci_dev *pdev = msi_desc_to_pci_dev(info->desc);
1873 	int hwirq;
1874 	int i, ret;
1875 
1876 	hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, nr_irqs);
1877 	if (hwirq < 0) {
1878 		dev_warn(&pdev->dev, "failed to find a free MSI\n");
1879 		return -ENOSPC;
1880 	}
1881 
1882 	dev_dbg(&pdev->dev, "%s bridge %pOF %d/%x #%d\n", __func__,
1883 		hose->dn, virq, hwirq, nr_irqs);
1884 
1885 	for (i = 0; i < nr_irqs; i++) {
1886 		ret = pnv_irq_parent_domain_alloc(domain, virq + i,
1887 						  phb->msi_base + hwirq + i);
1888 		if (ret)
1889 			goto out;
1890 
1891 		irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
1892 					      &pnv_msi_irq_chip, hose);
1893 	}
1894 
1895 	return 0;
1896 
1897 out:
1898 	irq_domain_free_irqs_parent(domain, virq, i - 1);
1899 	msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, nr_irqs);
1900 	return ret;
1901 }
1902 
1903 static void pnv_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1904 				unsigned int nr_irqs)
1905 {
1906 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1907 	struct pci_controller *hose = irq_data_get_irq_chip_data(d);
1908 	struct pnv_phb *phb = hose->private_data;
1909 
1910 	pr_debug("%s bridge %pOF %d/%lx #%d\n", __func__, hose->dn,
1911 		 virq, d->hwirq, nr_irqs);
1912 
1913 	msi_bitmap_free_hwirqs(&phb->msi_bmp, d->hwirq, nr_irqs);
1914 	/* XIVE domain is cleared through ->msi_free() */
1915 }
1916 
1917 static const struct irq_domain_ops pnv_irq_domain_ops = {
1918 	.alloc  = pnv_irq_domain_alloc,
1919 	.free   = pnv_irq_domain_free,
1920 };
1921 
1922 static int __init pnv_msi_allocate_domains(struct pci_controller *hose, unsigned int count)
1923 {
1924 	struct pnv_phb *phb = hose->private_data;
1925 	struct irq_domain *parent = irq_get_default_host();
1926 
1927 	hose->fwnode = irq_domain_alloc_named_id_fwnode("PNV-MSI", phb->opal_id);
1928 	if (!hose->fwnode)
1929 		return -ENOMEM;
1930 
1931 	hose->dev_domain = irq_domain_create_hierarchy(parent, 0, count,
1932 						       hose->fwnode,
1933 						       &pnv_irq_domain_ops, hose);
1934 	if (!hose->dev_domain) {
1935 		pr_err("PCI: failed to create IRQ domain bridge %pOF (domain %d)\n",
1936 		       hose->dn, hose->global_number);
1937 		irq_domain_free_fwnode(hose->fwnode);
1938 		return -ENOMEM;
1939 	}
1940 
1941 	hose->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(hose->dn),
1942 						     &pnv_msi_domain_info,
1943 						     hose->dev_domain);
1944 	if (!hose->msi_domain) {
1945 		pr_err("PCI: failed to create MSI IRQ domain bridge %pOF (domain %d)\n",
1946 		       hose->dn, hose->global_number);
1947 		irq_domain_free_fwnode(hose->fwnode);
1948 		irq_domain_remove(hose->dev_domain);
1949 		return -ENOMEM;
1950 	}
1951 
1952 	return 0;
1953 }
1954 
1955 static void __init pnv_pci_init_ioda_msis(struct pnv_phb *phb)
1956 {
1957 	unsigned int count;
1958 	const __be32 *prop = of_get_property(phb->hose->dn,
1959 					     "ibm,opal-msi-ranges", NULL);
1960 	if (!prop) {
1961 		/* BML Fallback */
1962 		prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
1963 	}
1964 	if (!prop)
1965 		return;
1966 
1967 	phb->msi_base = be32_to_cpup(prop);
1968 	count = be32_to_cpup(prop + 1);
1969 	if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
1970 		pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
1971 		       phb->hose->global_number);
1972 		return;
1973 	}
1974 
1975 	pr_info("  Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
1976 		count, phb->msi_base);
1977 
1978 	pnv_msi_allocate_domains(phb->hose, count);
1979 }
1980 
1981 static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
1982 				  struct resource *res)
1983 {
1984 	struct pnv_phb *phb = pe->phb;
1985 	struct pci_bus_region region;
1986 	int index;
1987 	int64_t rc;
1988 
1989 	if (!res || !res->flags || res->start > res->end ||
1990 	    res->flags & IORESOURCE_UNSET)
1991 		return;
1992 
1993 	if (res->flags & IORESOURCE_IO) {
1994 		region.start = res->start - phb->ioda.io_pci_base;
1995 		region.end   = res->end - phb->ioda.io_pci_base;
1996 		index = region.start / phb->ioda.io_segsize;
1997 
1998 		while (index < phb->ioda.total_pe_num &&
1999 		       region.start <= region.end) {
2000 			phb->ioda.io_segmap[index] = pe->pe_number;
2001 			rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2002 				pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
2003 			if (rc != OPAL_SUCCESS) {
2004 				pr_err("%s: Error %lld mapping IO segment#%d to PE#%x\n",
2005 				       __func__, rc, index, pe->pe_number);
2006 				break;
2007 			}
2008 
2009 			region.start += phb->ioda.io_segsize;
2010 			index++;
2011 		}
2012 	} else if ((res->flags & IORESOURCE_MEM) &&
2013 		   !pnv_pci_is_m64(phb, res)) {
2014 		region.start = res->start -
2015 			       phb->hose->mem_offset[0] -
2016 			       phb->ioda.m32_pci_base;
2017 		region.end   = res->end -
2018 			       phb->hose->mem_offset[0] -
2019 			       phb->ioda.m32_pci_base;
2020 		index = region.start / phb->ioda.m32_segsize;
2021 
2022 		while (index < phb->ioda.total_pe_num &&
2023 		       region.start <= region.end) {
2024 			phb->ioda.m32_segmap[index] = pe->pe_number;
2025 			rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2026 				pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
2027 			if (rc != OPAL_SUCCESS) {
2028 				pr_err("%s: Error %lld mapping M32 segment#%d to PE#%x",
2029 				       __func__, rc, index, pe->pe_number);
2030 				break;
2031 			}
2032 
2033 			region.start += phb->ioda.m32_segsize;
2034 			index++;
2035 		}
2036 	}
2037 }
2038 
2039 /*
2040  * This function is supposed to be called on basis of PE from top
2041  * to bottom style. So the I/O or MMIO segment assigned to
2042  * parent PE could be overridden by its child PEs if necessary.
2043  */
2044 static void pnv_ioda_setup_pe_seg(struct pnv_ioda_pe *pe)
2045 {
2046 	struct pci_dev *pdev;
2047 	int i;
2048 
2049 	/*
2050 	 * NOTE: We only care PCI bus based PE for now. For PCI
2051 	 * device based PE, for example SRIOV sensitive VF should
2052 	 * be figured out later.
2053 	 */
2054 	BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
2055 
2056 	list_for_each_entry(pdev, &pe->pbus->devices, bus_list) {
2057 		for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2058 			pnv_ioda_setup_pe_res(pe, &pdev->resource[i]);
2059 
2060 		/*
2061 		 * If the PE contains all subordinate PCI buses, the
2062 		 * windows of the child bridges should be mapped to
2063 		 * the PE as well.
2064 		 */
2065 		if (!(pe->flags & PNV_IODA_PE_BUS_ALL) || !pci_is_bridge(pdev))
2066 			continue;
2067 		for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
2068 			pnv_ioda_setup_pe_res(pe,
2069 				&pdev->resource[PCI_BRIDGE_RESOURCES + i]);
2070 	}
2071 }
2072 
2073 #ifdef CONFIG_DEBUG_FS
2074 static int pnv_pci_diag_data_set(void *data, u64 val)
2075 {
2076 	struct pnv_phb *phb = data;
2077 	s64 ret;
2078 
2079 	/* Retrieve the diag data from firmware */
2080 	ret = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag_data,
2081 					  phb->diag_data_size);
2082 	if (ret != OPAL_SUCCESS)
2083 		return -EIO;
2084 
2085 	/* Print the diag data to the kernel log */
2086 	pnv_pci_dump_phb_diag_data(phb->hose, phb->diag_data);
2087 	return 0;
2088 }
2089 
2090 DEFINE_DEBUGFS_ATTRIBUTE(pnv_pci_diag_data_fops, NULL, pnv_pci_diag_data_set,
2091 			 "%llu\n");
2092 
2093 static int pnv_pci_ioda_pe_dump(void *data, u64 val)
2094 {
2095 	struct pnv_phb *phb = data;
2096 	int pe_num;
2097 
2098 	for (pe_num = 0; pe_num < phb->ioda.total_pe_num; pe_num++) {
2099 		struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_num];
2100 
2101 		if (!test_bit(pe_num, phb->ioda.pe_alloc))
2102 			continue;
2103 
2104 		pe_warn(pe, "rid: %04x dev count: %2d flags: %s%s%s%s%s%s\n",
2105 			pe->rid, pe->device_count,
2106 			(pe->flags & PNV_IODA_PE_DEV) ? "dev " : "",
2107 			(pe->flags & PNV_IODA_PE_BUS) ? "bus " : "",
2108 			(pe->flags & PNV_IODA_PE_BUS_ALL) ? "all " : "",
2109 			(pe->flags & PNV_IODA_PE_MASTER) ? "master " : "",
2110 			(pe->flags & PNV_IODA_PE_SLAVE) ? "slave " : "",
2111 			(pe->flags & PNV_IODA_PE_VF) ? "vf " : "");
2112 	}
2113 
2114 	return 0;
2115 }
2116 
2117 DEFINE_DEBUGFS_ATTRIBUTE(pnv_pci_ioda_pe_dump_fops, NULL,
2118 			 pnv_pci_ioda_pe_dump, "%llu\n");
2119 
2120 #endif /* CONFIG_DEBUG_FS */
2121 
2122 static void pnv_pci_ioda_create_dbgfs(void)
2123 {
2124 #ifdef CONFIG_DEBUG_FS
2125 	struct pci_controller *hose, *tmp;
2126 	struct pnv_phb *phb;
2127 	char name[16];
2128 
2129 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
2130 		phb = hose->private_data;
2131 
2132 		sprintf(name, "PCI%04x", hose->global_number);
2133 		phb->dbgfs = debugfs_create_dir(name, arch_debugfs_dir);
2134 
2135 		debugfs_create_file_unsafe("dump_diag_regs", 0200, phb->dbgfs,
2136 					   phb, &pnv_pci_diag_data_fops);
2137 		debugfs_create_file_unsafe("dump_ioda_pe_state", 0200, phb->dbgfs,
2138 					   phb, &pnv_pci_ioda_pe_dump_fops);
2139 	}
2140 #endif /* CONFIG_DEBUG_FS */
2141 }
2142 
2143 static void pnv_pci_enable_bridge(struct pci_bus *bus)
2144 {
2145 	struct pci_dev *dev = bus->self;
2146 	struct pci_bus *child;
2147 
2148 	/* Empty bus ? bail */
2149 	if (list_empty(&bus->devices))
2150 		return;
2151 
2152 	/*
2153 	 * If there's a bridge associated with that bus enable it. This works
2154 	 * around races in the generic code if the enabling is done during
2155 	 * parallel probing. This can be removed once those races have been
2156 	 * fixed.
2157 	 */
2158 	if (dev) {
2159 		int rc = pci_enable_device(dev);
2160 		if (rc)
2161 			pci_err(dev, "Error enabling bridge (%d)\n", rc);
2162 		pci_set_master(dev);
2163 	}
2164 
2165 	/* Perform the same to child busses */
2166 	list_for_each_entry(child, &bus->children, node)
2167 		pnv_pci_enable_bridge(child);
2168 }
2169 
2170 static void pnv_pci_enable_bridges(void)
2171 {
2172 	struct pci_controller *hose;
2173 
2174 	list_for_each_entry(hose, &hose_list, list_node)
2175 		pnv_pci_enable_bridge(hose->bus);
2176 }
2177 
2178 static void pnv_pci_ioda_fixup(void)
2179 {
2180 	pnv_pci_ioda_create_dbgfs();
2181 
2182 	pnv_pci_enable_bridges();
2183 
2184 #ifdef CONFIG_EEH
2185 	pnv_eeh_post_init();
2186 #endif
2187 }
2188 
2189 /*
2190  * Returns the alignment for I/O or memory windows for P2P
2191  * bridges. That actually depends on how PEs are segmented.
2192  * For now, we return I/O or M32 segment size for PE sensitive
2193  * P2P bridges. Otherwise, the default values (4KiB for I/O,
2194  * 1MiB for memory) will be returned.
2195  *
2196  * The current PCI bus might be put into one PE, which was
2197  * create against the parent PCI bridge. For that case, we
2198  * needn't enlarge the alignment so that we can save some
2199  * resources.
2200  */
2201 static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
2202 						unsigned long type)
2203 {
2204 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
2205 	int num_pci_bridges = 0;
2206 	struct pci_dev *bridge;
2207 
2208 	bridge = bus->self;
2209 	while (bridge) {
2210 		if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
2211 			num_pci_bridges++;
2212 			if (num_pci_bridges >= 2)
2213 				return 1;
2214 		}
2215 
2216 		bridge = bridge->bus->self;
2217 	}
2218 
2219 	/*
2220 	 * We fall back to M32 if M64 isn't supported. We enforce the M64
2221 	 * alignment for any 64-bit resource, PCIe doesn't care and
2222 	 * bridges only do 64-bit prefetchable anyway.
2223 	 */
2224 	if (phb->ioda.m64_segsize && pnv_pci_is_m64_flags(type))
2225 		return phb->ioda.m64_segsize;
2226 	if (type & IORESOURCE_MEM)
2227 		return phb->ioda.m32_segsize;
2228 
2229 	return phb->ioda.io_segsize;
2230 }
2231 
2232 /*
2233  * We are updating root port or the upstream port of the
2234  * bridge behind the root port with PHB's windows in order
2235  * to accommodate the changes on required resources during
2236  * PCI (slot) hotplug, which is connected to either root
2237  * port or the downstream ports of PCIe switch behind the
2238  * root port.
2239  */
2240 static void pnv_pci_fixup_bridge_resources(struct pci_bus *bus,
2241 					   unsigned long type)
2242 {
2243 	struct pci_controller *hose = pci_bus_to_host(bus);
2244 	struct pnv_phb *phb = hose->private_data;
2245 	struct pci_dev *bridge = bus->self;
2246 	struct resource *r, *w;
2247 	bool msi_region = false;
2248 	int i;
2249 
2250 	/* Check if we need apply fixup to the bridge's windows */
2251 	if (!pci_is_root_bus(bridge->bus) &&
2252 	    !pci_is_root_bus(bridge->bus->self->bus))
2253 		return;
2254 
2255 	/* Fixup the resources */
2256 	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
2257 		r = &bridge->resource[PCI_BRIDGE_RESOURCES + i];
2258 		if (!r->flags || !r->parent)
2259 			continue;
2260 
2261 		w = NULL;
2262 		if (r->flags & type & IORESOURCE_IO)
2263 			w = &hose->io_resource;
2264 		else if (pnv_pci_is_m64(phb, r) &&
2265 			 (type & IORESOURCE_PREFETCH) &&
2266 			 phb->ioda.m64_segsize)
2267 			w = &hose->mem_resources[1];
2268 		else if (r->flags & type & IORESOURCE_MEM) {
2269 			w = &hose->mem_resources[0];
2270 			msi_region = true;
2271 		}
2272 
2273 		r->start = w->start;
2274 		r->end = w->end;
2275 
2276 		/* The 64KB 32-bits MSI region shouldn't be included in
2277 		 * the 32-bits bridge window. Otherwise, we can see strange
2278 		 * issues. One of them is EEH error observed on Garrison.
2279 		 *
2280 		 * Exclude top 1MB region which is the minimal alignment of
2281 		 * 32-bits bridge window.
2282 		 */
2283 		if (msi_region) {
2284 			r->end += 0x10000;
2285 			r->end -= 0x100000;
2286 		}
2287 	}
2288 }
2289 
2290 static void pnv_pci_configure_bus(struct pci_bus *bus)
2291 {
2292 	struct pci_dev *bridge = bus->self;
2293 	struct pnv_ioda_pe *pe;
2294 	bool all = (bridge && pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE);
2295 
2296 	dev_info(&bus->dev, "Configuring PE for bus\n");
2297 
2298 	/* Don't assign PE to PCI bus, which doesn't have subordinate devices */
2299 	if (WARN_ON(list_empty(&bus->devices)))
2300 		return;
2301 
2302 	/* Reserve PEs according to used M64 resources */
2303 	pnv_ioda_reserve_m64_pe(bus, NULL, all);
2304 
2305 	/*
2306 	 * Assign PE. We might run here because of partial hotplug.
2307 	 * For the case, we just pick up the existing PE and should
2308 	 * not allocate resources again.
2309 	 */
2310 	pe = pnv_ioda_setup_bus_PE(bus, all);
2311 	if (!pe)
2312 		return;
2313 
2314 	pnv_ioda_setup_pe_seg(pe);
2315 }
2316 
2317 static resource_size_t pnv_pci_default_alignment(void)
2318 {
2319 	return PAGE_SIZE;
2320 }
2321 
2322 /* Prevent enabling devices for which we couldn't properly
2323  * assign a PE
2324  */
2325 static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
2326 {
2327 	struct pci_dn *pdn;
2328 
2329 	pdn = pci_get_pdn(dev);
2330 	if (!pdn || pdn->pe_number == IODA_INVALID_PE) {
2331 		pci_err(dev, "pci_enable_device() blocked, no PE assigned.\n");
2332 		return false;
2333 	}
2334 
2335 	return true;
2336 }
2337 
2338 static bool pnv_ocapi_enable_device_hook(struct pci_dev *dev)
2339 {
2340 	struct pci_dn *pdn;
2341 	struct pnv_ioda_pe *pe;
2342 
2343 	pdn = pci_get_pdn(dev);
2344 	if (!pdn)
2345 		return false;
2346 
2347 	if (pdn->pe_number == IODA_INVALID_PE) {
2348 		pe = pnv_ioda_setup_dev_PE(dev);
2349 		if (!pe)
2350 			return false;
2351 	}
2352 	return true;
2353 }
2354 
2355 void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
2356 {
2357 	struct iommu_table *tbl = pe->table_group.tables[0];
2358 	int64_t rc;
2359 
2360 	if (!pe->dma_setup_done)
2361 		return;
2362 
2363 	rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
2364 	if (rc)
2365 		pe_warn(pe, "OPAL error %lld release DMA window\n", rc);
2366 
2367 	pnv_pci_ioda2_set_bypass(pe, false);
2368 	if (pe->table_group.group) {
2369 		iommu_group_put(pe->table_group.group);
2370 		WARN_ON(pe->table_group.group);
2371 	}
2372 
2373 	iommu_tce_table_put(tbl);
2374 }
2375 
2376 static void pnv_ioda_free_pe_seg(struct pnv_ioda_pe *pe,
2377 				 unsigned short win,
2378 				 unsigned int *map)
2379 {
2380 	struct pnv_phb *phb = pe->phb;
2381 	int idx;
2382 	int64_t rc;
2383 
2384 	for (idx = 0; idx < phb->ioda.total_pe_num; idx++) {
2385 		if (map[idx] != pe->pe_number)
2386 			continue;
2387 
2388 		rc = opal_pci_map_pe_mmio_window(phb->opal_id,
2389 				phb->ioda.reserved_pe_idx, win, 0, idx);
2390 
2391 		if (rc != OPAL_SUCCESS)
2392 			pe_warn(pe, "Error %lld unmapping (%d) segment#%d\n",
2393 				rc, win, idx);
2394 
2395 		map[idx] = IODA_INVALID_PE;
2396 	}
2397 }
2398 
2399 static void pnv_ioda_release_pe_seg(struct pnv_ioda_pe *pe)
2400 {
2401 	struct pnv_phb *phb = pe->phb;
2402 
2403 	if (phb->type == PNV_PHB_IODA2) {
2404 		pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
2405 				     phb->ioda.m32_segmap);
2406 	}
2407 }
2408 
2409 static void pnv_ioda_release_pe(struct pnv_ioda_pe *pe)
2410 {
2411 	struct pnv_phb *phb = pe->phb;
2412 	struct pnv_ioda_pe *slave, *tmp;
2413 
2414 	pe_info(pe, "Releasing PE\n");
2415 
2416 	mutex_lock(&phb->ioda.pe_list_mutex);
2417 	list_del(&pe->list);
2418 	mutex_unlock(&phb->ioda.pe_list_mutex);
2419 
2420 	switch (phb->type) {
2421 	case PNV_PHB_IODA2:
2422 		pnv_pci_ioda2_release_pe_dma(pe);
2423 		break;
2424 	case PNV_PHB_NPU_OCAPI:
2425 		break;
2426 	default:
2427 		WARN_ON(1);
2428 	}
2429 
2430 	pnv_ioda_release_pe_seg(pe);
2431 	pnv_ioda_deconfigure_pe(pe->phb, pe);
2432 
2433 	/* Release slave PEs in the compound PE */
2434 	if (pe->flags & PNV_IODA_PE_MASTER) {
2435 		list_for_each_entry_safe(slave, tmp, &pe->slaves, list) {
2436 			list_del(&slave->list);
2437 			pnv_ioda_free_pe(slave);
2438 		}
2439 	}
2440 
2441 	/*
2442 	 * The PE for root bus can be removed because of hotplug in EEH
2443 	 * recovery for fenced PHB error. We need to mark the PE dead so
2444 	 * that it can be populated again in PCI hot add path. The PE
2445 	 * shouldn't be destroyed as it's the global reserved resource.
2446 	 */
2447 	if (phb->ioda.root_pe_idx == pe->pe_number)
2448 		return;
2449 
2450 	pnv_ioda_free_pe(pe);
2451 }
2452 
2453 static void pnv_pci_release_device(struct pci_dev *pdev)
2454 {
2455 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
2456 	struct pci_dn *pdn = pci_get_pdn(pdev);
2457 	struct pnv_ioda_pe *pe;
2458 
2459 	/* The VF PE state is torn down when sriov_disable() is called */
2460 	if (pdev->is_virtfn)
2461 		return;
2462 
2463 	if (!pdn || pdn->pe_number == IODA_INVALID_PE)
2464 		return;
2465 
2466 #ifdef CONFIG_PCI_IOV
2467 	/*
2468 	 * FIXME: Try move this to sriov_disable(). It's here since we allocate
2469 	 * the iov state at probe time since we need to fiddle with the IOV
2470 	 * resources.
2471 	 */
2472 	if (pdev->is_physfn)
2473 		kfree(pdev->dev.archdata.iov_data);
2474 #endif
2475 
2476 	/*
2477 	 * PCI hotplug can happen as part of EEH error recovery. The @pdn
2478 	 * isn't removed and added afterwards in this scenario. We should
2479 	 * set the PE number in @pdn to an invalid one. Otherwise, the PE's
2480 	 * device count is decreased on removing devices while failing to
2481 	 * be increased on adding devices. It leads to unbalanced PE's device
2482 	 * count and eventually make normal PCI hotplug path broken.
2483 	 */
2484 	pe = &phb->ioda.pe_array[pdn->pe_number];
2485 	pdn->pe_number = IODA_INVALID_PE;
2486 
2487 	WARN_ON(--pe->device_count < 0);
2488 	if (pe->device_count == 0)
2489 		pnv_ioda_release_pe(pe);
2490 }
2491 
2492 static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
2493 {
2494 	struct pnv_phb *phb = hose->private_data;
2495 
2496 	opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
2497 		       OPAL_ASSERT_RESET);
2498 }
2499 
2500 static void pnv_pci_ioda_dma_bus_setup(struct pci_bus *bus)
2501 {
2502 	struct pnv_phb *phb = pci_bus_to_pnvhb(bus);
2503 	struct pnv_ioda_pe *pe;
2504 
2505 	list_for_each_entry(pe, &phb->ioda.pe_list, list) {
2506 		if (!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)))
2507 			continue;
2508 
2509 		if (!pe->pbus)
2510 			continue;
2511 
2512 		if (bus->number == ((pe->rid >> 8) & 0xFF)) {
2513 			pe->pbus = bus;
2514 			break;
2515 		}
2516 	}
2517 }
2518 
2519 #ifdef CONFIG_IOMMU_API
2520 static struct iommu_group *pnv_pci_device_group(struct pci_controller *hose,
2521 						struct pci_dev *pdev)
2522 {
2523 	struct pnv_phb *phb = hose->private_data;
2524 	struct pnv_ioda_pe *pe;
2525 
2526 	if (WARN_ON(!phb))
2527 		return ERR_PTR(-ENODEV);
2528 
2529 	pe = pnv_pci_bdfn_to_pe(phb, pdev->devfn | (pdev->bus->number << 8));
2530 	if (!pe)
2531 		return ERR_PTR(-ENODEV);
2532 
2533 	if (!pe->table_group.group)
2534 		return ERR_PTR(-ENODEV);
2535 
2536 	return iommu_group_ref_get(pe->table_group.group);
2537 }
2538 #endif
2539 
2540 static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
2541 	.dma_dev_setup		= pnv_pci_ioda_dma_dev_setup,
2542 	.dma_bus_setup		= pnv_pci_ioda_dma_bus_setup,
2543 	.iommu_bypass_supported	= pnv_pci_ioda_iommu_bypass_supported,
2544 	.enable_device_hook	= pnv_pci_enable_device_hook,
2545 	.release_device		= pnv_pci_release_device,
2546 	.window_alignment	= pnv_pci_window_alignment,
2547 	.setup_bridge		= pnv_pci_fixup_bridge_resources,
2548 	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
2549 	.shutdown		= pnv_pci_ioda_shutdown,
2550 #ifdef CONFIG_IOMMU_API
2551 	.device_group		= pnv_pci_device_group,
2552 #endif
2553 };
2554 
2555 static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
2556 	.enable_device_hook	= pnv_ocapi_enable_device_hook,
2557 	.release_device		= pnv_pci_release_device,
2558 	.window_alignment	= pnv_pci_window_alignment,
2559 	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
2560 	.shutdown		= pnv_pci_ioda_shutdown,
2561 };
2562 
2563 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
2564 					 u64 hub_id, int ioda_type)
2565 {
2566 	struct pci_controller *hose;
2567 	struct pnv_phb *phb;
2568 	unsigned long size, m64map_off, m32map_off, pemap_off;
2569 	struct pnv_ioda_pe *root_pe;
2570 	struct resource r;
2571 	const __be64 *prop64;
2572 	const __be32 *prop32;
2573 	int len;
2574 	unsigned int segno;
2575 	u64 phb_id;
2576 	void *aux;
2577 	long rc;
2578 
2579 	if (!of_device_is_available(np))
2580 		return;
2581 
2582 	pr_info("Initializing %s PHB (%pOF)\n",	pnv_phb_names[ioda_type], np);
2583 
2584 	prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
2585 	if (!prop64) {
2586 		pr_err("  Missing \"ibm,opal-phbid\" property !\n");
2587 		return;
2588 	}
2589 	phb_id = be64_to_cpup(prop64);
2590 	pr_debug("  PHB-ID  : 0x%016llx\n", phb_id);
2591 
2592 	phb = kzalloc(sizeof(*phb), GFP_KERNEL);
2593 	if (!phb)
2594 		panic("%s: Failed to allocate %zu bytes\n", __func__,
2595 		      sizeof(*phb));
2596 
2597 	/* Allocate PCI controller */
2598 	phb->hose = hose = pcibios_alloc_controller(np);
2599 	if (!phb->hose) {
2600 		pr_err("  Can't allocate PCI controller for %pOF\n",
2601 		       np);
2602 		memblock_free(phb, sizeof(struct pnv_phb));
2603 		return;
2604 	}
2605 
2606 	spin_lock_init(&phb->lock);
2607 	prop32 = of_get_property(np, "bus-range", &len);
2608 	if (prop32 && len == 8) {
2609 		hose->first_busno = be32_to_cpu(prop32[0]);
2610 		hose->last_busno = be32_to_cpu(prop32[1]);
2611 	} else {
2612 		pr_warn("  Broken <bus-range> on %pOF\n", np);
2613 		hose->first_busno = 0;
2614 		hose->last_busno = 0xff;
2615 	}
2616 	hose->private_data = phb;
2617 	phb->hub_id = hub_id;
2618 	phb->opal_id = phb_id;
2619 	phb->type = ioda_type;
2620 	mutex_init(&phb->ioda.pe_alloc_mutex);
2621 
2622 	/* Detect specific models for error handling */
2623 	if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
2624 		phb->model = PNV_PHB_MODEL_P7IOC;
2625 	else if (of_device_is_compatible(np, "ibm,power8-pciex"))
2626 		phb->model = PNV_PHB_MODEL_PHB3;
2627 	else
2628 		phb->model = PNV_PHB_MODEL_UNKNOWN;
2629 
2630 	/* Initialize diagnostic data buffer */
2631 	prop32 = of_get_property(np, "ibm,phb-diag-data-size", NULL);
2632 	if (prop32)
2633 		phb->diag_data_size = be32_to_cpup(prop32);
2634 	else
2635 		phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE;
2636 
2637 	phb->diag_data = kzalloc(phb->diag_data_size, GFP_KERNEL);
2638 	if (!phb->diag_data)
2639 		panic("%s: Failed to allocate %u bytes\n", __func__,
2640 		      phb->diag_data_size);
2641 
2642 	/* Parse 32-bit and IO ranges (if any) */
2643 	pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
2644 
2645 	/* Get registers */
2646 	if (!of_address_to_resource(np, 0, &r)) {
2647 		phb->regs_phys = r.start;
2648 		phb->regs = ioremap(r.start, resource_size(&r));
2649 		if (phb->regs == NULL)
2650 			pr_err("  Failed to map registers !\n");
2651 	}
2652 
2653 	/* Initialize more IODA stuff */
2654 	phb->ioda.total_pe_num = 1;
2655 	prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
2656 	if (prop32)
2657 		phb->ioda.total_pe_num = be32_to_cpup(prop32);
2658 	prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
2659 	if (prop32)
2660 		phb->ioda.reserved_pe_idx = be32_to_cpup(prop32);
2661 
2662 	/* Invalidate RID to PE# mapping */
2663 	for (segno = 0; segno < ARRAY_SIZE(phb->ioda.pe_rmap); segno++)
2664 		phb->ioda.pe_rmap[segno] = IODA_INVALID_PE;
2665 
2666 	/* Parse 64-bit MMIO range */
2667 	pnv_ioda_parse_m64_window(phb);
2668 
2669 	phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
2670 	/* FW Has already off top 64k of M32 space (MSI space) */
2671 	phb->ioda.m32_size += 0x10000;
2672 
2673 	phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe_num;
2674 	phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
2675 	phb->ioda.io_size = hose->pci_io_size;
2676 	phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe_num;
2677 	phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
2678 
2679 	/* Allocate aux data & arrays. We don't have IO ports on PHB3 */
2680 	size = ALIGN(max_t(unsigned, phb->ioda.total_pe_num, 8) / 8,
2681 			sizeof(unsigned long));
2682 	m64map_off = size;
2683 	size += phb->ioda.total_pe_num * sizeof(phb->ioda.m64_segmap[0]);
2684 	m32map_off = size;
2685 	size += phb->ioda.total_pe_num * sizeof(phb->ioda.m32_segmap[0]);
2686 	pemap_off = size;
2687 	size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe);
2688 	aux = kzalloc(size, GFP_KERNEL);
2689 	if (!aux)
2690 		panic("%s: Failed to allocate %lu bytes\n", __func__, size);
2691 
2692 	phb->ioda.pe_alloc = aux;
2693 	phb->ioda.m64_segmap = aux + m64map_off;
2694 	phb->ioda.m32_segmap = aux + m32map_off;
2695 	for (segno = 0; segno < phb->ioda.total_pe_num; segno++) {
2696 		phb->ioda.m64_segmap[segno] = IODA_INVALID_PE;
2697 		phb->ioda.m32_segmap[segno] = IODA_INVALID_PE;
2698 	}
2699 	phb->ioda.pe_array = aux + pemap_off;
2700 
2701 	/*
2702 	 * Choose PE number for root bus, which shouldn't have
2703 	 * M64 resources consumed by its child devices. To pick
2704 	 * the PE number adjacent to the reserved one if possible.
2705 	 */
2706 	pnv_ioda_reserve_pe(phb, phb->ioda.reserved_pe_idx);
2707 	if (phb->ioda.reserved_pe_idx == 0) {
2708 		phb->ioda.root_pe_idx = 1;
2709 		pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
2710 	} else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1)) {
2711 		phb->ioda.root_pe_idx = phb->ioda.reserved_pe_idx - 1;
2712 		pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
2713 	} else {
2714 		/* otherwise just allocate one */
2715 		root_pe = pnv_ioda_alloc_pe(phb, 1);
2716 		phb->ioda.root_pe_idx = root_pe->pe_number;
2717 	}
2718 
2719 	INIT_LIST_HEAD(&phb->ioda.pe_list);
2720 	mutex_init(&phb->ioda.pe_list_mutex);
2721 
2722 #if 0 /* We should really do that ... */
2723 	rc = opal_pci_set_phb_mem_window(opal->phb_id,
2724 					 window_type,
2725 					 window_num,
2726 					 starting_real_address,
2727 					 starting_pci_address,
2728 					 segment_size);
2729 #endif
2730 
2731 	pr_info("  %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
2732 		phb->ioda.total_pe_num, phb->ioda.reserved_pe_idx,
2733 		phb->ioda.m32_size, phb->ioda.m32_segsize);
2734 	if (phb->ioda.m64_size)
2735 		pr_info("                 M64: 0x%lx [segment=0x%lx]\n",
2736 			phb->ioda.m64_size, phb->ioda.m64_segsize);
2737 	if (phb->ioda.io_size)
2738 		pr_info("                  IO: 0x%x [segment=0x%x]\n",
2739 			phb->ioda.io_size, phb->ioda.io_segsize);
2740 
2741 
2742 	phb->hose->ops = &pnv_pci_ops;
2743 	phb->get_pe_state = pnv_ioda_get_pe_state;
2744 	phb->freeze_pe = pnv_ioda_freeze_pe;
2745 	phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
2746 
2747 	/* Setup MSI support */
2748 	pnv_pci_init_ioda_msis(phb);
2749 
2750 	/*
2751 	 * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
2752 	 * to let the PCI core do resource assignment. It's supposed
2753 	 * that the PCI core will do correct I/O and MMIO alignment
2754 	 * for the P2P bridge bars so that each PCI bus (excluding
2755 	 * the child P2P bridges) can form individual PE.
2756 	 */
2757 	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
2758 
2759 	switch (phb->type) {
2760 	case PNV_PHB_NPU_OCAPI:
2761 		hose->controller_ops = pnv_npu_ocapi_ioda_controller_ops;
2762 		break;
2763 	default:
2764 		hose->controller_ops = pnv_pci_ioda_controller_ops;
2765 	}
2766 
2767 	ppc_md.pcibios_default_alignment = pnv_pci_default_alignment;
2768 
2769 #ifdef CONFIG_PCI_IOV
2770 	ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov;
2771 	ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
2772 	ppc_md.pcibios_sriov_enable = pnv_pcibios_sriov_enable;
2773 	ppc_md.pcibios_sriov_disable = pnv_pcibios_sriov_disable;
2774 #endif
2775 
2776 	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
2777 
2778 	/* Reset IODA tables to a clean state */
2779 	rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
2780 	if (rc)
2781 		pr_warn("  OPAL Error %ld performing IODA table reset !\n", rc);
2782 
2783 	/*
2784 	 * If we're running in kdump kernel, the previous kernel never
2785 	 * shutdown PCI devices correctly. We already got IODA table
2786 	 * cleaned out. So we have to issue PHB reset to stop all PCI
2787 	 * transactions from previous kernel. The ppc_pci_reset_phbs
2788 	 * kernel parameter will force this reset too. Additionally,
2789 	 * if the IODA reset above failed then use a bigger hammer.
2790 	 * This can happen if we get a PHB fatal error in very early
2791 	 * boot.
2792 	 */
2793 	if (is_kdump_kernel() || pci_reset_phbs || rc) {
2794 		pr_info("  Issue PHB reset ...\n");
2795 		pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
2796 		pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
2797 	}
2798 
2799 	/* Remove M64 resource if we can't configure it successfully */
2800 	if (!phb->init_m64 || phb->init_m64(phb))
2801 		hose->mem_resources[1].flags = 0;
2802 
2803 	/* create pci_dn's for DT nodes under this PHB */
2804 	pci_devs_phb_init_dynamic(hose);
2805 }
2806 
2807 void __init pnv_pci_init_ioda2_phb(struct device_node *np)
2808 {
2809 	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
2810 }
2811 
2812 void __init pnv_pci_init_npu2_opencapi_phb(struct device_node *np)
2813 {
2814 	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_OCAPI);
2815 }
2816 
2817 static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev)
2818 {
2819 	struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus);
2820 
2821 	if (!machine_is(powernv))
2822 		return;
2823 
2824 	if (phb->type == PNV_PHB_NPU_OCAPI)
2825 		dev->cfg_size = PCI_CFG_SPACE_EXP_SIZE;
2826 }
2827 DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pnv_npu2_opencapi_cfg_size_fixup);
2828