1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCIe host bridge driver for Apple system-on-chips.
4  *
5  * The HW is ECAM compliant, so once the controller is initialized,
6  * the driver mostly deals MSI mapping and handling of per-port
7  * interrupts (INTx, management and error signals).
8  *
9  * Initialization requires enabling power and clocks, along with a
10  * number of register pokes.
11  *
12  * Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io>
13  * Copyright (C) 2021 Google LLC
14  * Copyright (C) 2021 Corellium LLC
15  * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org>
16  *
17  * Author: Alyssa Rosenzweig <alyssa@rosenzweig.io>
18  * Author: Marc Zyngier <maz@kernel.org>
19  */
20 
21 #include <linux/gpio/consumer.h>
22 #include <linux/kernel.h>
23 #include <linux/iopoll.h>
24 #include <linux/irqchip/chained_irq.h>
25 #include <linux/irqdomain.h>
26 #include <linux/list.h>
27 #include <linux/module.h>
28 #include <linux/msi.h>
29 #include <linux/notifier.h>
30 #include <linux/of_irq.h>
31 #include <linux/pci-ecam.h>
32 
33 #define CORE_RC_PHYIF_CTL		0x00024
34 #define   CORE_RC_PHYIF_CTL_RUN		BIT(0)
35 #define CORE_RC_PHYIF_STAT		0x00028
36 #define   CORE_RC_PHYIF_STAT_REFCLK	BIT(4)
37 #define CORE_RC_CTL			0x00050
38 #define   CORE_RC_CTL_RUN		BIT(0)
39 #define CORE_RC_STAT			0x00058
40 #define   CORE_RC_STAT_READY		BIT(0)
41 #define CORE_FABRIC_STAT		0x04000
42 #define   CORE_FABRIC_STAT_MASK		0x001F001F
43 #define CORE_LANE_CFG(port)		(0x84000 + 0x4000 * (port))
44 #define   CORE_LANE_CFG_REFCLK0REQ	BIT(0)
45 #define   CORE_LANE_CFG_REFCLK1REQ	BIT(1)
46 #define   CORE_LANE_CFG_REFCLK0ACK	BIT(2)
47 #define   CORE_LANE_CFG_REFCLK1ACK	BIT(3)
48 #define   CORE_LANE_CFG_REFCLKEN	(BIT(9) | BIT(10))
49 #define CORE_LANE_CTL(port)		(0x84004 + 0x4000 * (port))
50 #define   CORE_LANE_CTL_CFGACC		BIT(15)
51 
52 #define PORT_LTSSMCTL			0x00080
53 #define   PORT_LTSSMCTL_START		BIT(0)
54 #define PORT_INTSTAT			0x00100
55 #define   PORT_INT_TUNNEL_ERR		31
56 #define   PORT_INT_CPL_TIMEOUT		23
57 #define   PORT_INT_RID2SID_MAPERR	22
58 #define   PORT_INT_CPL_ABORT		21
59 #define   PORT_INT_MSI_BAD_DATA		19
60 #define   PORT_INT_MSI_ERR		18
61 #define   PORT_INT_REQADDR_GT32		17
62 #define   PORT_INT_AF_TIMEOUT		15
63 #define   PORT_INT_LINK_DOWN		14
64 #define   PORT_INT_LINK_UP		12
65 #define   PORT_INT_LINK_BWMGMT		11
66 #define   PORT_INT_AER_MASK		(15 << 4)
67 #define   PORT_INT_PORT_ERR		4
68 #define   PORT_INT_INTx(i)		i
69 #define   PORT_INT_INTx_MASK		15
70 #define PORT_INTMSK			0x00104
71 #define PORT_INTMSKSET			0x00108
72 #define PORT_INTMSKCLR			0x0010c
73 #define PORT_MSICFG			0x00124
74 #define   PORT_MSICFG_EN		BIT(0)
75 #define   PORT_MSICFG_L2MSINUM_SHIFT	4
76 #define PORT_MSIBASE			0x00128
77 #define   PORT_MSIBASE_1_SHIFT		16
78 #define PORT_MSIADDR			0x00168
79 #define PORT_LINKSTS			0x00208
80 #define   PORT_LINKSTS_UP		BIT(0)
81 #define   PORT_LINKSTS_BUSY		BIT(2)
82 #define PORT_LINKCMDSTS			0x00210
83 #define PORT_OUTS_NPREQS		0x00284
84 #define   PORT_OUTS_NPREQS_REQ		BIT(24)
85 #define   PORT_OUTS_NPREQS_CPL		BIT(16)
86 #define PORT_RXWR_FIFO			0x00288
87 #define   PORT_RXWR_FIFO_HDR		GENMASK(15, 10)
88 #define   PORT_RXWR_FIFO_DATA		GENMASK(9, 0)
89 #define PORT_RXRD_FIFO			0x0028C
90 #define   PORT_RXRD_FIFO_REQ		GENMASK(6, 0)
91 #define PORT_OUTS_CPLS			0x00290
92 #define   PORT_OUTS_CPLS_SHRD		GENMASK(14, 8)
93 #define   PORT_OUTS_CPLS_WAIT		GENMASK(6, 0)
94 #define PORT_APPCLK			0x00800
95 #define   PORT_APPCLK_EN		BIT(0)
96 #define   PORT_APPCLK_CGDIS		BIT(8)
97 #define PORT_STATUS			0x00804
98 #define   PORT_STATUS_READY		BIT(0)
99 #define PORT_REFCLK			0x00810
100 #define   PORT_REFCLK_EN		BIT(0)
101 #define   PORT_REFCLK_CGDIS		BIT(8)
102 #define PORT_PERST			0x00814
103 #define   PORT_PERST_OFF		BIT(0)
104 #define PORT_RID2SID(i16)		(0x00828 + 4 * (i16))
105 #define   PORT_RID2SID_VALID		BIT(31)
106 #define   PORT_RID2SID_SID_SHIFT	16
107 #define   PORT_RID2SID_BUS_SHIFT	8
108 #define   PORT_RID2SID_DEV_SHIFT	3
109 #define   PORT_RID2SID_FUNC_SHIFT	0
110 #define PORT_OUTS_PREQS_HDR		0x00980
111 #define   PORT_OUTS_PREQS_HDR_MASK	GENMASK(9, 0)
112 #define PORT_OUTS_PREQS_DATA		0x00984
113 #define   PORT_OUTS_PREQS_DATA_MASK	GENMASK(15, 0)
114 #define PORT_TUNCTRL			0x00988
115 #define   PORT_TUNCTRL_PERST_ON		BIT(0)
116 #define   PORT_TUNCTRL_PERST_ACK_REQ	BIT(1)
117 #define PORT_TUNSTAT			0x0098c
118 #define   PORT_TUNSTAT_PERST_ON		BIT(0)
119 #define   PORT_TUNSTAT_PERST_ACK_PEND	BIT(1)
120 #define PORT_PREFMEM_ENABLE		0x00994
121 
122 #define MAX_RID2SID			64
123 
124 /*
125  * The doorbell address is set to 0xfffff000, which by convention
126  * matches what MacOS does, and it is possible to use any other
127  * address (in the bottom 4GB, as the base register is only 32bit).
128  * However, it has to be excluded from the IOVA range, and the DART
129  * driver has to know about it.
130  */
131 #define DOORBELL_ADDR		CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
132 
133 struct apple_pcie {
134 	struct mutex		lock;
135 	struct device		*dev;
136 	void __iomem            *base;
137 	struct irq_domain	*domain;
138 	unsigned long		*bitmap;
139 	struct list_head	ports;
140 	struct completion	event;
141 	struct irq_fwspec	fwspec;
142 	u32			nvecs;
143 };
144 
145 struct apple_pcie_port {
146 	struct apple_pcie	*pcie;
147 	struct device_node	*np;
148 	void __iomem		*base;
149 	struct irq_domain	*domain;
150 	struct list_head	entry;
151 	DECLARE_BITMAP(sid_map, MAX_RID2SID);
152 	int			sid_map_sz;
153 	int			idx;
154 };
155 
156 static void rmw_set(u32 set, void __iomem *addr)
157 {
158 	writel_relaxed(readl_relaxed(addr) | set, addr);
159 }
160 
161 static void rmw_clear(u32 clr, void __iomem *addr)
162 {
163 	writel_relaxed(readl_relaxed(addr) & ~clr, addr);
164 }
165 
166 static void apple_msi_top_irq_mask(struct irq_data *d)
167 {
168 	pci_msi_mask_irq(d);
169 	irq_chip_mask_parent(d);
170 }
171 
172 static void apple_msi_top_irq_unmask(struct irq_data *d)
173 {
174 	pci_msi_unmask_irq(d);
175 	irq_chip_unmask_parent(d);
176 }
177 
178 static struct irq_chip apple_msi_top_chip = {
179 	.name			= "PCIe MSI",
180 	.irq_mask		= apple_msi_top_irq_mask,
181 	.irq_unmask		= apple_msi_top_irq_unmask,
182 	.irq_eoi		= irq_chip_eoi_parent,
183 	.irq_set_affinity	= irq_chip_set_affinity_parent,
184 	.irq_set_type		= irq_chip_set_type_parent,
185 };
186 
187 static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg)
188 {
189 	msg->address_hi = upper_32_bits(DOORBELL_ADDR);
190 	msg->address_lo = lower_32_bits(DOORBELL_ADDR);
191 	msg->data = data->hwirq;
192 }
193 
194 static struct irq_chip apple_msi_bottom_chip = {
195 	.name			= "MSI",
196 	.irq_mask		= irq_chip_mask_parent,
197 	.irq_unmask		= irq_chip_unmask_parent,
198 	.irq_eoi		= irq_chip_eoi_parent,
199 	.irq_set_affinity	= irq_chip_set_affinity_parent,
200 	.irq_set_type		= irq_chip_set_type_parent,
201 	.irq_compose_msi_msg	= apple_msi_compose_msg,
202 };
203 
204 static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
205 				  unsigned int nr_irqs, void *args)
206 {
207 	struct apple_pcie *pcie = domain->host_data;
208 	struct irq_fwspec fwspec = pcie->fwspec;
209 	unsigned int i;
210 	int ret, hwirq;
211 
212 	mutex_lock(&pcie->lock);
213 
214 	hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs,
215 					order_base_2(nr_irqs));
216 
217 	mutex_unlock(&pcie->lock);
218 
219 	if (hwirq < 0)
220 		return -ENOSPC;
221 
222 	fwspec.param[fwspec.param_count - 2] += hwirq;
223 
224 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec);
225 	if (ret)
226 		return ret;
227 
228 	for (i = 0; i < nr_irqs; i++) {
229 		irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
230 					      &apple_msi_bottom_chip,
231 					      domain->host_data);
232 	}
233 
234 	return 0;
235 }
236 
237 static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq,
238 				  unsigned int nr_irqs)
239 {
240 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
241 	struct apple_pcie *pcie = domain->host_data;
242 
243 	mutex_lock(&pcie->lock);
244 
245 	bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs));
246 
247 	mutex_unlock(&pcie->lock);
248 }
249 
250 static const struct irq_domain_ops apple_msi_domain_ops = {
251 	.alloc	= apple_msi_domain_alloc,
252 	.free	= apple_msi_domain_free,
253 };
254 
255 static struct msi_domain_info apple_msi_info = {
256 	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
257 		   MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX),
258 	.chip	= &apple_msi_top_chip,
259 };
260 
261 static void apple_port_irq_mask(struct irq_data *data)
262 {
263 	struct apple_pcie_port *port = irq_data_get_irq_chip_data(data);
264 
265 	writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKSET);
266 }
267 
268 static void apple_port_irq_unmask(struct irq_data *data)
269 {
270 	struct apple_pcie_port *port = irq_data_get_irq_chip_data(data);
271 
272 	writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKCLR);
273 }
274 
275 static bool hwirq_is_intx(unsigned int hwirq)
276 {
277 	return BIT(hwirq) & PORT_INT_INTx_MASK;
278 }
279 
280 static void apple_port_irq_ack(struct irq_data *data)
281 {
282 	struct apple_pcie_port *port = irq_data_get_irq_chip_data(data);
283 
284 	if (!hwirq_is_intx(data->hwirq))
285 		writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT);
286 }
287 
288 static int apple_port_irq_set_type(struct irq_data *data, unsigned int type)
289 {
290 	/*
291 	 * It doesn't seem that there is any way to configure the
292 	 * trigger, so assume INTx have to be level (as per the spec),
293 	 * and the rest is edge (which looks likely).
294 	 */
295 	if (hwirq_is_intx(data->hwirq) ^ !!(type & IRQ_TYPE_LEVEL_MASK))
296 		return -EINVAL;
297 
298 	irqd_set_trigger_type(data, type);
299 	return 0;
300 }
301 
302 static struct irq_chip apple_port_irqchip = {
303 	.name		= "PCIe",
304 	.irq_ack	= apple_port_irq_ack,
305 	.irq_mask	= apple_port_irq_mask,
306 	.irq_unmask	= apple_port_irq_unmask,
307 	.irq_set_type	= apple_port_irq_set_type,
308 };
309 
310 static int apple_port_irq_domain_alloc(struct irq_domain *domain,
311 				       unsigned int virq, unsigned int nr_irqs,
312 				       void *args)
313 {
314 	struct apple_pcie_port *port = domain->host_data;
315 	struct irq_fwspec *fwspec = args;
316 	int i;
317 
318 	for (i = 0; i < nr_irqs; i++) {
319 		irq_flow_handler_t flow = handle_edge_irq;
320 		unsigned int type = IRQ_TYPE_EDGE_RISING;
321 
322 		if (hwirq_is_intx(fwspec->param[0] + i)) {
323 			flow = handle_level_irq;
324 			type = IRQ_TYPE_LEVEL_HIGH;
325 		}
326 
327 		irq_domain_set_info(domain, virq + i, fwspec->param[0] + i,
328 				    &apple_port_irqchip, port, flow,
329 				    NULL, NULL);
330 
331 		irq_set_irq_type(virq + i, type);
332 	}
333 
334 	return 0;
335 }
336 
337 static void apple_port_irq_domain_free(struct irq_domain *domain,
338 				       unsigned int virq, unsigned int nr_irqs)
339 {
340 	int i;
341 
342 	for (i = 0; i < nr_irqs; i++) {
343 		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
344 
345 		irq_set_handler(virq + i, NULL);
346 		irq_domain_reset_irq_data(d);
347 	}
348 }
349 
350 static const struct irq_domain_ops apple_port_irq_domain_ops = {
351 	.translate	= irq_domain_translate_onecell,
352 	.alloc		= apple_port_irq_domain_alloc,
353 	.free		= apple_port_irq_domain_free,
354 };
355 
356 static void apple_port_irq_handler(struct irq_desc *desc)
357 {
358 	struct apple_pcie_port *port = irq_desc_get_handler_data(desc);
359 	struct irq_chip *chip = irq_desc_get_chip(desc);
360 	unsigned long stat;
361 	int i;
362 
363 	chained_irq_enter(chip, desc);
364 
365 	stat = readl_relaxed(port->base + PORT_INTSTAT);
366 
367 	for_each_set_bit(i, &stat, 32)
368 		generic_handle_domain_irq(port->domain, i);
369 
370 	chained_irq_exit(chip, desc);
371 }
372 
373 static int apple_pcie_port_setup_irq(struct apple_pcie_port *port)
374 {
375 	struct fwnode_handle *fwnode = &port->np->fwnode;
376 	unsigned int irq;
377 
378 	/* FIXME: consider moving each interrupt under each port */
379 	irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)),
380 				   port->idx);
381 	if (!irq)
382 		return -ENXIO;
383 
384 	port->domain = irq_domain_create_linear(fwnode, 32,
385 						&apple_port_irq_domain_ops,
386 						port);
387 	if (!port->domain)
388 		return -ENOMEM;
389 
390 	/* Disable all interrupts */
391 	writel_relaxed(~0, port->base + PORT_INTMSKSET);
392 	writel_relaxed(~0, port->base + PORT_INTSTAT);
393 
394 	irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port);
395 
396 	/* Configure MSI base address */
397 	BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR));
398 	writel_relaxed(lower_32_bits(DOORBELL_ADDR), port->base + PORT_MSIADDR);
399 
400 	/* Enable MSIs, shared between all ports */
401 	writel_relaxed(0, port->base + PORT_MSIBASE);
402 	writel_relaxed((ilog2(port->pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT) |
403 		       PORT_MSICFG_EN, port->base + PORT_MSICFG);
404 
405 	return 0;
406 }
407 
408 static irqreturn_t apple_pcie_port_irq(int irq, void *data)
409 {
410 	struct apple_pcie_port *port = data;
411 	unsigned int hwirq = irq_domain_get_irq_data(port->domain, irq)->hwirq;
412 
413 	switch (hwirq) {
414 	case PORT_INT_LINK_UP:
415 		dev_info_ratelimited(port->pcie->dev, "Link up on %pOF\n",
416 				     port->np);
417 		complete_all(&port->pcie->event);
418 		break;
419 	case PORT_INT_LINK_DOWN:
420 		dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n",
421 				     port->np);
422 		break;
423 	default:
424 		return IRQ_NONE;
425 	}
426 
427 	return IRQ_HANDLED;
428 }
429 
430 static int apple_pcie_port_register_irqs(struct apple_pcie_port *port)
431 {
432 	static struct {
433 		unsigned int	hwirq;
434 		const char	*name;
435 	} port_irqs[] = {
436 		{ PORT_INT_LINK_UP,	"Link up",	},
437 		{ PORT_INT_LINK_DOWN,	"Link down",	},
438 	};
439 	int i;
440 
441 	for (i = 0; i < ARRAY_SIZE(port_irqs); i++) {
442 		struct irq_fwspec fwspec = {
443 			.fwnode		= &port->np->fwnode,
444 			.param_count	= 1,
445 			.param		= {
446 				[0]	= port_irqs[i].hwirq,
447 			},
448 		};
449 		unsigned int irq;
450 		int ret;
451 
452 		irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE,
453 					    &fwspec);
454 		if (WARN_ON(!irq))
455 			continue;
456 
457 		ret = request_irq(irq, apple_pcie_port_irq, 0,
458 				  port_irqs[i].name, port);
459 		WARN_ON(ret);
460 	}
461 
462 	return 0;
463 }
464 
465 static int apple_pcie_setup_refclk(struct apple_pcie *pcie,
466 				   struct apple_pcie_port *port)
467 {
468 	u32 stat;
469 	int res;
470 
471 	res = readl_relaxed_poll_timeout(pcie->base + CORE_RC_PHYIF_STAT, stat,
472 					 stat & CORE_RC_PHYIF_STAT_REFCLK,
473 					 100, 50000);
474 	if (res < 0)
475 		return res;
476 
477 	rmw_set(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx));
478 	rmw_set(CORE_LANE_CFG_REFCLK0REQ, pcie->base + CORE_LANE_CFG(port->idx));
479 
480 	res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx),
481 					 stat, stat & CORE_LANE_CFG_REFCLK0ACK,
482 					 100, 50000);
483 	if (res < 0)
484 		return res;
485 
486 	rmw_set(CORE_LANE_CFG_REFCLK1REQ, pcie->base + CORE_LANE_CFG(port->idx));
487 	res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx),
488 					 stat, stat & CORE_LANE_CFG_REFCLK1ACK,
489 					 100, 50000);
490 
491 	if (res < 0)
492 		return res;
493 
494 	rmw_clear(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx));
495 
496 	rmw_set(CORE_LANE_CFG_REFCLKEN, pcie->base + CORE_LANE_CFG(port->idx));
497 	rmw_set(PORT_REFCLK_EN, port->base + PORT_REFCLK);
498 
499 	return 0;
500 }
501 
502 static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port,
503 				    int idx, u32 val)
504 {
505 	writel_relaxed(val, port->base + PORT_RID2SID(idx));
506 	/* Read back to ensure completion of the write */
507 	return readl_relaxed(port->base + PORT_RID2SID(idx));
508 }
509 
510 static int apple_pcie_setup_port(struct apple_pcie *pcie,
511 				 struct device_node *np)
512 {
513 	struct platform_device *platform = to_platform_device(pcie->dev);
514 	struct apple_pcie_port *port;
515 	struct gpio_desc *reset;
516 	u32 stat, idx;
517 	int ret, i;
518 
519 	reset = gpiod_get_from_of_node(np, "reset-gpios", 0,
520 				       GPIOD_OUT_LOW, "PERST#");
521 	if (IS_ERR(reset))
522 		return PTR_ERR(reset);
523 
524 	port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL);
525 	if (!port)
526 		return -ENOMEM;
527 
528 	ret = of_property_read_u32_index(np, "reg", 0, &idx);
529 	if (ret)
530 		return ret;
531 
532 	/* Use the first reg entry to work out the port index */
533 	port->idx = idx >> 11;
534 	port->pcie = pcie;
535 	port->np = np;
536 
537 	port->base = devm_platform_ioremap_resource(platform, port->idx + 2);
538 	if (IS_ERR(port->base))
539 		return PTR_ERR(port->base);
540 
541 	rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK);
542 
543 	/* Assert PERST# before setting up the clock */
544 	gpiod_set_value(reset, 1);
545 
546 	ret = apple_pcie_setup_refclk(pcie, port);
547 	if (ret < 0)
548 		return ret;
549 
550 	/* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */
551 	usleep_range(100, 200);
552 
553 	/* Deassert PERST# */
554 	rmw_set(PORT_PERST_OFF, port->base + PORT_PERST);
555 	gpiod_set_value(reset, 0);
556 
557 	/* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */
558 	msleep(100);
559 
560 	ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat,
561 					 stat & PORT_STATUS_READY, 100, 250000);
562 	if (ret < 0) {
563 		dev_err(pcie->dev, "port %pOF ready wait timeout\n", np);
564 		return ret;
565 	}
566 
567 	rmw_clear(PORT_REFCLK_CGDIS, port->base + PORT_REFCLK);
568 	rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK);
569 
570 	ret = apple_pcie_port_setup_irq(port);
571 	if (ret)
572 		return ret;
573 
574 	/* Reset all RID/SID mappings, and check for RAZ/WI registers */
575 	for (i = 0; i < MAX_RID2SID; i++) {
576 		if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d)
577 			break;
578 		apple_pcie_rid2sid_write(port, i, 0);
579 	}
580 
581 	dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i);
582 
583 	port->sid_map_sz = i;
584 
585 	list_add_tail(&port->entry, &pcie->ports);
586 	init_completion(&pcie->event);
587 
588 	ret = apple_pcie_port_register_irqs(port);
589 	WARN_ON(ret);
590 
591 	writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL);
592 
593 	if (!wait_for_completion_timeout(&pcie->event, HZ / 10))
594 		dev_warn(pcie->dev, "%pOF link didn't come up\n", np);
595 
596 	return 0;
597 }
598 
599 static int apple_msi_init(struct apple_pcie *pcie)
600 {
601 	struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
602 	struct of_phandle_args args = {};
603 	struct irq_domain *parent;
604 	int ret;
605 
606 	ret = of_parse_phandle_with_args(to_of_node(fwnode), "msi-ranges",
607 					 "#interrupt-cells", 0, &args);
608 	if (ret)
609 		return ret;
610 
611 	ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges",
612 					 args.args_count + 1, &pcie->nvecs);
613 	if (ret)
614 		return ret;
615 
616 	of_phandle_args_to_fwspec(args.np, args.args, args.args_count,
617 				  &pcie->fwspec);
618 
619 	pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL);
620 	if (!pcie->bitmap)
621 		return -ENOMEM;
622 
623 	parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED);
624 	if (!parent) {
625 		dev_err(pcie->dev, "failed to find parent domain\n");
626 		return -ENXIO;
627 	}
628 
629 	parent = irq_domain_create_hierarchy(parent, 0, pcie->nvecs, fwnode,
630 					     &apple_msi_domain_ops, pcie);
631 	if (!parent) {
632 		dev_err(pcie->dev, "failed to create IRQ domain\n");
633 		return -ENOMEM;
634 	}
635 	irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
636 
637 	pcie->domain = pci_msi_create_irq_domain(fwnode, &apple_msi_info,
638 						 parent);
639 	if (!pcie->domain) {
640 		dev_err(pcie->dev, "failed to create MSI domain\n");
641 		irq_domain_remove(parent);
642 		return -ENOMEM;
643 	}
644 
645 	return 0;
646 }
647 
648 static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev)
649 {
650 	struct pci_config_window *cfg = pdev->sysdata;
651 	struct apple_pcie *pcie = cfg->priv;
652 	struct pci_dev *port_pdev;
653 	struct apple_pcie_port *port;
654 
655 	/* Find the root port this device is on */
656 	port_pdev = pcie_find_root_port(pdev);
657 
658 	/* If finding the port itself, nothing to do */
659 	if (WARN_ON(!port_pdev) || pdev == port_pdev)
660 		return NULL;
661 
662 	list_for_each_entry(port, &pcie->ports, entry) {
663 		if (port->idx == PCI_SLOT(port_pdev->devfn))
664 			return port;
665 	}
666 
667 	return NULL;
668 }
669 
670 static int apple_pcie_add_device(struct apple_pcie_port *port,
671 				 struct pci_dev *pdev)
672 {
673 	u32 sid, rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
674 	int idx, err;
675 
676 	dev_dbg(&pdev->dev, "added to bus %s, index %d\n",
677 		pci_name(pdev->bus->self), port->idx);
678 
679 	err = of_map_id(port->pcie->dev->of_node, rid, "iommu-map",
680 			"iommu-map-mask", NULL, &sid);
681 	if (err)
682 		return err;
683 
684 	mutex_lock(&port->pcie->lock);
685 
686 	idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0);
687 	if (idx >= 0) {
688 		apple_pcie_rid2sid_write(port, idx,
689 					 PORT_RID2SID_VALID |
690 					 (sid << PORT_RID2SID_SID_SHIFT) | rid);
691 
692 		dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n",
693 			rid, sid, idx);
694 	}
695 
696 	mutex_unlock(&port->pcie->lock);
697 
698 	return idx >= 0 ? 0 : -ENOSPC;
699 }
700 
701 static void apple_pcie_release_device(struct apple_pcie_port *port,
702 				      struct pci_dev *pdev)
703 {
704 	u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
705 	int idx;
706 
707 	mutex_lock(&port->pcie->lock);
708 
709 	for_each_set_bit(idx, port->sid_map, port->sid_map_sz) {
710 		u32 val;
711 
712 		val = readl_relaxed(port->base + PORT_RID2SID(idx));
713 		if ((val & 0xffff) == rid) {
714 			apple_pcie_rid2sid_write(port, idx, 0);
715 			bitmap_release_region(port->sid_map, idx, 0);
716 			dev_dbg(&pdev->dev, "Released %x (%d)\n", val, idx);
717 			break;
718 		}
719 	}
720 
721 	mutex_unlock(&port->pcie->lock);
722 }
723 
724 static int apple_pcie_bus_notifier(struct notifier_block *nb,
725 				   unsigned long action,
726 				   void *data)
727 {
728 	struct device *dev = data;
729 	struct pci_dev *pdev = to_pci_dev(dev);
730 	struct apple_pcie_port *port;
731 	int err;
732 
733 	/*
734 	 * This is a bit ugly. We assume that if we get notified for
735 	 * any PCI device, we must be in charge of it, and that there
736 	 * is no other PCI controller in the whole system. It probably
737 	 * holds for now, but who knows for how long?
738 	 */
739 	port = apple_pcie_get_port(pdev);
740 	if (!port)
741 		return NOTIFY_DONE;
742 
743 	switch (action) {
744 	case BUS_NOTIFY_ADD_DEVICE:
745 		err = apple_pcie_add_device(port, pdev);
746 		if (err)
747 			return notifier_from_errno(err);
748 		break;
749 	case BUS_NOTIFY_DEL_DEVICE:
750 		apple_pcie_release_device(port, pdev);
751 		break;
752 	default:
753 		return NOTIFY_DONE;
754 	}
755 
756 	return NOTIFY_OK;
757 }
758 
759 static struct notifier_block apple_pcie_nb = {
760 	.notifier_call = apple_pcie_bus_notifier,
761 };
762 
763 static int apple_pcie_init(struct pci_config_window *cfg)
764 {
765 	struct device *dev = cfg->parent;
766 	struct platform_device *platform = to_platform_device(dev);
767 	struct device_node *of_port;
768 	struct apple_pcie *pcie;
769 	int ret;
770 
771 	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
772 	if (!pcie)
773 		return -ENOMEM;
774 
775 	pcie->dev = dev;
776 
777 	mutex_init(&pcie->lock);
778 
779 	pcie->base = devm_platform_ioremap_resource(platform, 1);
780 	if (IS_ERR(pcie->base))
781 		return PTR_ERR(pcie->base);
782 
783 	cfg->priv = pcie;
784 	INIT_LIST_HEAD(&pcie->ports);
785 
786 	for_each_child_of_node(dev->of_node, of_port) {
787 		ret = apple_pcie_setup_port(pcie, of_port);
788 		if (ret) {
789 			dev_err(pcie->dev, "Port %pOF setup fail: %d\n", of_port, ret);
790 			of_node_put(of_port);
791 			return ret;
792 		}
793 	}
794 
795 	return apple_msi_init(pcie);
796 }
797 
798 static int apple_pcie_probe(struct platform_device *pdev)
799 {
800 	int ret;
801 
802 	ret = bus_register_notifier(&pci_bus_type, &apple_pcie_nb);
803 	if (ret)
804 		return ret;
805 
806 	ret = pci_host_common_probe(pdev);
807 	if (ret)
808 		bus_unregister_notifier(&pci_bus_type, &apple_pcie_nb);
809 
810 	return ret;
811 }
812 
813 static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = {
814 	.init		= apple_pcie_init,
815 	.pci_ops	= {
816 		.map_bus	= pci_ecam_map_bus,
817 		.read		= pci_generic_config_read,
818 		.write		= pci_generic_config_write,
819 	}
820 };
821 
822 static const struct of_device_id apple_pcie_of_match[] = {
823 	{ .compatible = "apple,pcie", .data = &apple_pcie_cfg_ecam_ops },
824 	{ }
825 };
826 MODULE_DEVICE_TABLE(of, apple_pcie_of_match);
827 
828 static struct platform_driver apple_pcie_driver = {
829 	.probe	= apple_pcie_probe,
830 	.driver	= {
831 		.name			= "pcie-apple",
832 		.of_match_table		= apple_pcie_of_match,
833 		.suppress_bind_attrs	= true,
834 	},
835 };
836 module_platform_driver(apple_pcie_driver);
837 
838 MODULE_LICENSE("GPL v2");
839