1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017 Cadence
3 // Cadence PCIe host controller driver.
4 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
5 
6 #include <linux/delay.h>
7 #include <linux/kernel.h>
8 #include <linux/list_sort.h>
9 #include <linux/of_address.h>
10 #include <linux/of_pci.h>
11 #include <linux/platform_device.h>
12 
13 #include "pcie-cadence.h"
14 
15 #define LINK_RETRAIN_TIMEOUT HZ
16 
17 static u64 bar_max_size[] = {
18 	[RP_BAR0] = _ULL(128 * SZ_2G),
19 	[RP_BAR1] = SZ_2G,
20 	[RP_NO_BAR] = _BITULL(63),
21 };
22 
23 static u8 bar_aperture_mask[] = {
24 	[RP_BAR0] = 0x1F,
25 	[RP_BAR1] = 0xF,
26 };
27 
cdns_pci_map_bus(struct pci_bus * bus,unsigned int devfn,int where)28 void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
29 			       int where)
30 {
31 	struct pci_host_bridge *bridge = pci_find_host_bridge(bus);
32 	struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);
33 	struct cdns_pcie *pcie = &rc->pcie;
34 	unsigned int busn = bus->number;
35 	u32 addr0, desc0;
36 
37 	if (pci_is_root_bus(bus)) {
38 		/*
39 		 * Only the root port (devfn == 0) is connected to this bus.
40 		 * All other PCI devices are behind some bridge hence on another
41 		 * bus.
42 		 */
43 		if (devfn)
44 			return NULL;
45 
46 		return pcie->reg_base + (where & 0xfff);
47 	}
48 	/* Check that the link is up */
49 	if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))
50 		return NULL;
51 	/* Clear AXI link-down status */
52 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);
53 
54 	/* Update Output registers for AXI region 0. */
55 	addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |
56 		CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |
57 		CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);
58 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);
59 
60 	/* Configuration Type 0 or Type 1 access. */
61 	desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |
62 		CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);
63 	/*
64 	 * The bus number was already set once for all in desc1 by
65 	 * cdns_pcie_host_init_address_translation().
66 	 */
67 	if (busn == bridge->busnr + 1)
68 		desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;
69 	else
70 		desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;
71 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);
72 
73 	return rc->cfg_base + (where & 0xfff);
74 }
75 
76 static struct pci_ops cdns_pcie_host_ops = {
77 	.map_bus	= cdns_pci_map_bus,
78 	.read		= pci_generic_config_read,
79 	.write		= pci_generic_config_write,
80 };
81 
cdns_pcie_host_training_complete(struct cdns_pcie * pcie)82 static int cdns_pcie_host_training_complete(struct cdns_pcie *pcie)
83 {
84 	u32 pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
85 	unsigned long end_jiffies;
86 	u16 lnk_stat;
87 
88 	/* Wait for link training to complete. Exit after timeout. */
89 	end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
90 	do {
91 		lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
92 		if (!(lnk_stat & PCI_EXP_LNKSTA_LT))
93 			break;
94 		usleep_range(0, 1000);
95 	} while (time_before(jiffies, end_jiffies));
96 
97 	if (!(lnk_stat & PCI_EXP_LNKSTA_LT))
98 		return 0;
99 
100 	return -ETIMEDOUT;
101 }
102 
cdns_pcie_host_wait_for_link(struct cdns_pcie * pcie)103 static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)
104 {
105 	struct device *dev = pcie->dev;
106 	int retries;
107 
108 	/* Check if the link is up or not */
109 	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
110 		if (cdns_pcie_link_up(pcie)) {
111 			dev_info(dev, "Link up\n");
112 			return 0;
113 		}
114 		usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
115 	}
116 
117 	return -ETIMEDOUT;
118 }
119 
cdns_pcie_retrain(struct cdns_pcie * pcie)120 static int cdns_pcie_retrain(struct cdns_pcie *pcie)
121 {
122 	u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;
123 	u16 lnk_stat, lnk_ctl;
124 	int ret = 0;
125 
126 	/*
127 	 * Set retrain bit if current speed is 2.5 GB/s,
128 	 * but the PCIe root port support is > 2.5 GB/s.
129 	 */
130 
131 	lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +
132 					     PCI_EXP_LNKCAP));
133 	if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
134 		return ret;
135 
136 	lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);
137 	if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {
138 		lnk_ctl = cdns_pcie_rp_readw(pcie,
139 					     pcie_cap_off + PCI_EXP_LNKCTL);
140 		lnk_ctl |= PCI_EXP_LNKCTL_RL;
141 		cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,
142 				    lnk_ctl);
143 
144 		ret = cdns_pcie_host_training_complete(pcie);
145 		if (ret)
146 			return ret;
147 
148 		ret = cdns_pcie_host_wait_for_link(pcie);
149 	}
150 	return ret;
151 }
152 
cdns_pcie_host_enable_ptm_response(struct cdns_pcie * pcie)153 static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)
154 {
155 	u32 val;
156 
157 	val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);
158 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);
159 }
160 
cdns_pcie_host_start_link(struct cdns_pcie_rc * rc)161 static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)
162 {
163 	struct cdns_pcie *pcie = &rc->pcie;
164 	int ret;
165 
166 	ret = cdns_pcie_host_wait_for_link(pcie);
167 
168 	/*
169 	 * Retrain link for Gen2 training defect
170 	 * if quirk flag is set.
171 	 */
172 	if (!ret && rc->quirk_retrain_flag)
173 		ret = cdns_pcie_retrain(pcie);
174 
175 	return ret;
176 }
177 
cdns_pcie_host_init_root_port(struct cdns_pcie_rc * rc)178 static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)
179 {
180 	struct cdns_pcie *pcie = &rc->pcie;
181 	u32 value, ctrl;
182 	u32 id;
183 
184 	/*
185 	 * Set the root complex BAR configuration register:
186 	 * - disable both BAR0 and BAR1.
187 	 * - enable Prefetchable Memory Base and Limit registers in type 1
188 	 *   config space (64 bits).
189 	 * - enable IO Base and Limit registers in type 1 config
190 	 *   space (32 bits).
191 	 */
192 	ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
193 	value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |
194 		CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |
195 		CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |
196 		CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |
197 		CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |
198 		CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;
199 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
200 
201 	/* Set root port configuration space */
202 	if (rc->vendor_id != 0xffff) {
203 		id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |
204 			CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);
205 		cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);
206 	}
207 
208 	if (rc->device_id != 0xffff)
209 		cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);
210 
211 	cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);
212 	cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);
213 	cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
214 
215 	return 0;
216 }
217 
cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc * rc,enum cdns_pcie_rp_bar bar,u64 cpu_addr,u64 size,unsigned long flags)218 static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,
219 					enum cdns_pcie_rp_bar bar,
220 					u64 cpu_addr, u64 size,
221 					unsigned long flags)
222 {
223 	struct cdns_pcie *pcie = &rc->pcie;
224 	u32 addr0, addr1, aperture, value;
225 
226 	if (!rc->avail_ib_bar[bar])
227 		return -EBUSY;
228 
229 	rc->avail_ib_bar[bar] = false;
230 
231 	aperture = ilog2(size);
232 	addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |
233 		(lower_32_bits(cpu_addr) & GENMASK(31, 8));
234 	addr1 = upper_32_bits(cpu_addr);
235 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);
236 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);
237 
238 	if (bar == RP_NO_BAR)
239 		return 0;
240 
241 	value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);
242 	value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |
243 		   LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |
244 		   LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |
245 		   LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |
246 		   LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));
247 	if (size + cpu_addr >= SZ_4G) {
248 		if (!(flags & IORESOURCE_PREFETCH))
249 			value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);
250 		value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);
251 	} else {
252 		if (!(flags & IORESOURCE_PREFETCH))
253 			value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);
254 		value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);
255 	}
256 
257 	value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);
258 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);
259 
260 	return 0;
261 }
262 
263 static enum cdns_pcie_rp_bar
cdns_pcie_host_find_min_bar(struct cdns_pcie_rc * rc,u64 size)264 cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)
265 {
266 	enum cdns_pcie_rp_bar bar, sel_bar;
267 
268 	sel_bar = RP_BAR_UNDEFINED;
269 	for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
270 		if (!rc->avail_ib_bar[bar])
271 			continue;
272 
273 		if (size <= bar_max_size[bar]) {
274 			if (sel_bar == RP_BAR_UNDEFINED) {
275 				sel_bar = bar;
276 				continue;
277 			}
278 
279 			if (bar_max_size[bar] < bar_max_size[sel_bar])
280 				sel_bar = bar;
281 		}
282 	}
283 
284 	return sel_bar;
285 }
286 
287 static enum cdns_pcie_rp_bar
cdns_pcie_host_find_max_bar(struct cdns_pcie_rc * rc,u64 size)288 cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)
289 {
290 	enum cdns_pcie_rp_bar bar, sel_bar;
291 
292 	sel_bar = RP_BAR_UNDEFINED;
293 	for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {
294 		if (!rc->avail_ib_bar[bar])
295 			continue;
296 
297 		if (size >= bar_max_size[bar]) {
298 			if (sel_bar == RP_BAR_UNDEFINED) {
299 				sel_bar = bar;
300 				continue;
301 			}
302 
303 			if (bar_max_size[bar] > bar_max_size[sel_bar])
304 				sel_bar = bar;
305 		}
306 	}
307 
308 	return sel_bar;
309 }
310 
cdns_pcie_host_bar_config(struct cdns_pcie_rc * rc,struct resource_entry * entry)311 static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
312 				     struct resource_entry *entry)
313 {
314 	u64 cpu_addr, pci_addr, size, winsize;
315 	struct cdns_pcie *pcie = &rc->pcie;
316 	struct device *dev = pcie->dev;
317 	enum cdns_pcie_rp_bar bar;
318 	unsigned long flags;
319 	int ret;
320 
321 	cpu_addr = entry->res->start;
322 	pci_addr = entry->res->start - entry->offset;
323 	flags = entry->res->flags;
324 	size = resource_size(entry->res);
325 
326 	if (entry->offset) {
327 		dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",
328 			pci_addr, cpu_addr);
329 		return -EINVAL;
330 	}
331 
332 	while (size > 0) {
333 		/*
334 		 * Try to find a minimum BAR whose size is greater than
335 		 * or equal to the remaining resource_entry size. This will
336 		 * fail if the size of each of the available BARs is less than
337 		 * the remaining resource_entry size.
338 		 * If a minimum BAR is found, IB ATU will be configured and
339 		 * exited.
340 		 */
341 		bar = cdns_pcie_host_find_min_bar(rc, size);
342 		if (bar != RP_BAR_UNDEFINED) {
343 			ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,
344 							   size, flags);
345 			if (ret)
346 				dev_err(dev, "IB BAR: %d config failed\n", bar);
347 			return ret;
348 		}
349 
350 		/*
351 		 * If the control reaches here, it would mean the remaining
352 		 * resource_entry size cannot be fitted in a single BAR. So we
353 		 * find a maximum BAR whose size is less than or equal to the
354 		 * remaining resource_entry size and split the resource entry
355 		 * so that part of resource entry is fitted inside the maximum
356 		 * BAR. The remaining size would be fitted during the next
357 		 * iteration of the loop.
358 		 * If a maximum BAR is not found, there is no way we can fit
359 		 * this resource_entry, so we error out.
360 		 */
361 		bar = cdns_pcie_host_find_max_bar(rc, size);
362 		if (bar == RP_BAR_UNDEFINED) {
363 			dev_err(dev, "No free BAR to map cpu_addr %llx\n",
364 				cpu_addr);
365 			return -EINVAL;
366 		}
367 
368 		winsize = bar_max_size[bar];
369 		ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,
370 						   flags);
371 		if (ret) {
372 			dev_err(dev, "IB BAR: %d config failed\n", bar);
373 			return ret;
374 		}
375 
376 		size -= winsize;
377 		cpu_addr += winsize;
378 	}
379 
380 	return 0;
381 }
382 
cdns_pcie_host_dma_ranges_cmp(void * priv,const struct list_head * a,const struct list_head * b)383 static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
384 					 const struct list_head *b)
385 {
386 	struct resource_entry *entry1, *entry2;
387 
388         entry1 = container_of(a, struct resource_entry, node);
389         entry2 = container_of(b, struct resource_entry, node);
390 
391         return resource_size(entry2->res) - resource_size(entry1->res);
392 }
393 
cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc * rc)394 static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)
395 {
396 	struct cdns_pcie *pcie = &rc->pcie;
397 	struct device *dev = pcie->dev;
398 	struct device_node *np = dev->of_node;
399 	struct pci_host_bridge *bridge;
400 	struct resource_entry *entry;
401 	u32 no_bar_nbits = 32;
402 	int err;
403 
404 	bridge = pci_host_bridge_from_priv(rc);
405 	if (!bridge)
406 		return -ENOMEM;
407 
408 	if (list_empty(&bridge->dma_ranges)) {
409 		of_property_read_u32(np, "cdns,no-bar-match-nbits",
410 				     &no_bar_nbits);
411 		err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,
412 						   (u64)1 << no_bar_nbits, 0);
413 		if (err)
414 			dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);
415 		return err;
416 	}
417 
418 	list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);
419 
420 	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
421 		err = cdns_pcie_host_bar_config(rc, entry);
422 		if (err) {
423 			dev_err(dev, "Fail to configure IB using dma-ranges\n");
424 			return err;
425 		}
426 	}
427 
428 	return 0;
429 }
430 
cdns_pcie_host_init_address_translation(struct cdns_pcie_rc * rc)431 static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
432 {
433 	struct cdns_pcie *pcie = &rc->pcie;
434 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);
435 	struct resource *cfg_res = rc->cfg_res;
436 	struct resource_entry *entry;
437 	u64 cpu_addr = cfg_res->start;
438 	u32 addr0, addr1, desc1;
439 	int r, busnr = 0;
440 
441 	entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);
442 	if (entry)
443 		busnr = entry->res->start;
444 
445 	/*
446 	 * Reserve region 0 for PCI configure space accesses:
447 	 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by
448 	 * cdns_pci_map_bus(), other region registers are set here once for all.
449 	 */
450 	addr1 = 0; /* Should be programmed to zero. */
451 	desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);
452 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);
453 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);
454 
455 	if (pcie->ops->cpu_addr_fixup)
456 		cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);
457 
458 	addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |
459 		(lower_32_bits(cpu_addr) & GENMASK(31, 8));
460 	addr1 = upper_32_bits(cpu_addr);
461 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);
462 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);
463 
464 	r = 1;
465 	resource_list_for_each_entry(entry, &bridge->windows) {
466 		struct resource *res = entry->res;
467 		u64 pci_addr = res->start - entry->offset;
468 
469 		if (resource_type(res) == IORESOURCE_IO)
470 			cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
471 						      true,
472 						      pci_pio_to_address(res->start),
473 						      pci_addr,
474 						      resource_size(res));
475 		else
476 			cdns_pcie_set_outbound_region(pcie, busnr, 0, r,
477 						      false,
478 						      res->start,
479 						      pci_addr,
480 						      resource_size(res));
481 
482 		r++;
483 	}
484 
485 	return cdns_pcie_host_map_dma_ranges(rc);
486 }
487 
cdns_pcie_host_init(struct device * dev,struct cdns_pcie_rc * rc)488 static int cdns_pcie_host_init(struct device *dev,
489 			       struct cdns_pcie_rc *rc)
490 {
491 	int err;
492 
493 	err = cdns_pcie_host_init_root_port(rc);
494 	if (err)
495 		return err;
496 
497 	return cdns_pcie_host_init_address_translation(rc);
498 }
499 
cdns_pcie_host_setup(struct cdns_pcie_rc * rc)500 int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
501 {
502 	struct device *dev = rc->pcie.dev;
503 	struct platform_device *pdev = to_platform_device(dev);
504 	struct device_node *np = dev->of_node;
505 	struct pci_host_bridge *bridge;
506 	enum cdns_pcie_rp_bar bar;
507 	struct cdns_pcie *pcie;
508 	struct resource *res;
509 	int ret;
510 
511 	bridge = pci_host_bridge_from_priv(rc);
512 	if (!bridge)
513 		return -ENOMEM;
514 
515 	pcie = &rc->pcie;
516 	pcie->is_rc = true;
517 
518 	rc->vendor_id = 0xffff;
519 	of_property_read_u32(np, "vendor-id", &rc->vendor_id);
520 
521 	rc->device_id = 0xffff;
522 	of_property_read_u32(np, "device-id", &rc->device_id);
523 
524 	pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");
525 	if (IS_ERR(pcie->reg_base)) {
526 		dev_err(dev, "missing \"reg\"\n");
527 		return PTR_ERR(pcie->reg_base);
528 	}
529 
530 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");
531 	rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);
532 	if (IS_ERR(rc->cfg_base))
533 		return PTR_ERR(rc->cfg_base);
534 	rc->cfg_res = res;
535 
536 	if (rc->quirk_detect_quiet_flag)
537 		cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);
538 
539 	cdns_pcie_host_enable_ptm_response(pcie);
540 
541 	ret = cdns_pcie_start_link(pcie);
542 	if (ret) {
543 		dev_err(dev, "Failed to start link\n");
544 		return ret;
545 	}
546 
547 	ret = cdns_pcie_host_start_link(rc);
548 	if (ret)
549 		dev_dbg(dev, "PCIe link never came up\n");
550 
551 	for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)
552 		rc->avail_ib_bar[bar] = true;
553 
554 	ret = cdns_pcie_host_init(dev, rc);
555 	if (ret)
556 		return ret;
557 
558 	if (!bridge->ops)
559 		bridge->ops = &cdns_pcie_host_ops;
560 
561 	ret = pci_host_probe(bridge);
562 	if (ret < 0)
563 		goto err_init;
564 
565 	return 0;
566 
567  err_init:
568 	pm_runtime_put_sync(dev);
569 
570 	return ret;
571 }
572