xref: /openbmc/linux/drivers/memory/tegra/mc.c (revision b8d312aa)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/clk.h>
7 #include <linux/delay.h>
8 #include <linux/interrupt.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/of_device.h>
13 #include <linux/platform_device.h>
14 #include <linux/slab.h>
15 #include <linux/sort.h>
16 
17 #include <soc/tegra/fuse.h>
18 
19 #include "mc.h"
20 
21 #define MC_INTSTATUS 0x000
22 
23 #define MC_INTMASK 0x004
24 
25 #define MC_ERR_STATUS 0x08
26 #define  MC_ERR_STATUS_TYPE_SHIFT 28
27 #define  MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE (6 << MC_ERR_STATUS_TYPE_SHIFT)
28 #define  MC_ERR_STATUS_TYPE_MASK (0x7 << MC_ERR_STATUS_TYPE_SHIFT)
29 #define  MC_ERR_STATUS_READABLE (1 << 27)
30 #define  MC_ERR_STATUS_WRITABLE (1 << 26)
31 #define  MC_ERR_STATUS_NONSECURE (1 << 25)
32 #define  MC_ERR_STATUS_ADR_HI_SHIFT 20
33 #define  MC_ERR_STATUS_ADR_HI_MASK 0x3
34 #define  MC_ERR_STATUS_SECURITY (1 << 17)
35 #define  MC_ERR_STATUS_RW (1 << 16)
36 
37 #define MC_ERR_ADR 0x0c
38 
39 #define MC_GART_ERROR_REQ		0x30
40 #define MC_DECERR_EMEM_OTHERS_STATUS	0x58
41 #define MC_SECURITY_VIOLATION_STATUS	0x74
42 
43 #define MC_EMEM_ARB_CFG 0x90
44 #define  MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x)	(((x) & 0x1ff) << 0)
45 #define  MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK	0x1ff
46 #define MC_EMEM_ARB_MISC0 0xd8
47 
48 #define MC_EMEM_ADR_CFG 0x54
49 #define MC_EMEM_ADR_CFG_EMEM_NUMDEV BIT(0)
50 
51 #define MC_TIMING_CONTROL		0xfc
52 #define MC_TIMING_UPDATE		BIT(0)
53 
54 static const struct of_device_id tegra_mc_of_match[] = {
55 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
56 	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
57 #endif
58 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
59 	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
60 #endif
61 #ifdef CONFIG_ARCH_TEGRA_114_SOC
62 	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
63 #endif
64 #ifdef CONFIG_ARCH_TEGRA_124_SOC
65 	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
66 #endif
67 #ifdef CONFIG_ARCH_TEGRA_132_SOC
68 	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
69 #endif
70 #ifdef CONFIG_ARCH_TEGRA_210_SOC
71 	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
72 #endif
73 	{ }
74 };
75 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
76 
77 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
78 				     const struct tegra_mc_reset *rst)
79 {
80 	unsigned long flags;
81 	u32 value;
82 
83 	spin_lock_irqsave(&mc->lock, flags);
84 
85 	value = mc_readl(mc, rst->control) | BIT(rst->bit);
86 	mc_writel(mc, value, rst->control);
87 
88 	spin_unlock_irqrestore(&mc->lock, flags);
89 
90 	return 0;
91 }
92 
93 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
94 				       const struct tegra_mc_reset *rst)
95 {
96 	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
97 }
98 
99 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
100 				       const struct tegra_mc_reset *rst)
101 {
102 	unsigned long flags;
103 	u32 value;
104 
105 	spin_lock_irqsave(&mc->lock, flags);
106 
107 	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
108 	mc_writel(mc, value, rst->control);
109 
110 	spin_unlock_irqrestore(&mc->lock, flags);
111 
112 	return 0;
113 }
114 
115 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
116 					const struct tegra_mc_reset *rst)
117 {
118 	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
119 }
120 
121 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
122 	.block_dma = tegra_mc_block_dma_common,
123 	.dma_idling = tegra_mc_dma_idling_common,
124 	.unblock_dma = tegra_mc_unblock_dma_common,
125 	.reset_status = tegra_mc_reset_status_common,
126 };
127 
128 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
129 {
130 	return container_of(rcdev, struct tegra_mc, reset);
131 }
132 
133 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
134 							unsigned long id)
135 {
136 	unsigned int i;
137 
138 	for (i = 0; i < mc->soc->num_resets; i++)
139 		if (mc->soc->resets[i].id == id)
140 			return &mc->soc->resets[i];
141 
142 	return NULL;
143 }
144 
145 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
146 				    unsigned long id)
147 {
148 	struct tegra_mc *mc = reset_to_mc(rcdev);
149 	const struct tegra_mc_reset_ops *rst_ops;
150 	const struct tegra_mc_reset *rst;
151 	int retries = 500;
152 	int err;
153 
154 	rst = tegra_mc_reset_find(mc, id);
155 	if (!rst)
156 		return -ENODEV;
157 
158 	rst_ops = mc->soc->reset_ops;
159 	if (!rst_ops)
160 		return -ENODEV;
161 
162 	if (rst_ops->block_dma) {
163 		/* block clients DMA requests */
164 		err = rst_ops->block_dma(mc, rst);
165 		if (err) {
166 			dev_err(mc->dev, "failed to block %s DMA: %d\n",
167 				rst->name, err);
168 			return err;
169 		}
170 	}
171 
172 	if (rst_ops->dma_idling) {
173 		/* wait for completion of the outstanding DMA requests */
174 		while (!rst_ops->dma_idling(mc, rst)) {
175 			if (!retries--) {
176 				dev_err(mc->dev, "failed to flush %s DMA\n",
177 					rst->name);
178 				return -EBUSY;
179 			}
180 
181 			usleep_range(10, 100);
182 		}
183 	}
184 
185 	if (rst_ops->hotreset_assert) {
186 		/* clear clients DMA requests sitting before arbitration */
187 		err = rst_ops->hotreset_assert(mc, rst);
188 		if (err) {
189 			dev_err(mc->dev, "failed to hot reset %s: %d\n",
190 				rst->name, err);
191 			return err;
192 		}
193 	}
194 
195 	return 0;
196 }
197 
198 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
199 				      unsigned long id)
200 {
201 	struct tegra_mc *mc = reset_to_mc(rcdev);
202 	const struct tegra_mc_reset_ops *rst_ops;
203 	const struct tegra_mc_reset *rst;
204 	int err;
205 
206 	rst = tegra_mc_reset_find(mc, id);
207 	if (!rst)
208 		return -ENODEV;
209 
210 	rst_ops = mc->soc->reset_ops;
211 	if (!rst_ops)
212 		return -ENODEV;
213 
214 	if (rst_ops->hotreset_deassert) {
215 		/* take out client from hot reset */
216 		err = rst_ops->hotreset_deassert(mc, rst);
217 		if (err) {
218 			dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
219 				rst->name, err);
220 			return err;
221 		}
222 	}
223 
224 	if (rst_ops->unblock_dma) {
225 		/* allow new DMA requests to proceed to arbitration */
226 		err = rst_ops->unblock_dma(mc, rst);
227 		if (err) {
228 			dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
229 				rst->name, err);
230 			return err;
231 		}
232 	}
233 
234 	return 0;
235 }
236 
237 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
238 				    unsigned long id)
239 {
240 	struct tegra_mc *mc = reset_to_mc(rcdev);
241 	const struct tegra_mc_reset_ops *rst_ops;
242 	const struct tegra_mc_reset *rst;
243 
244 	rst = tegra_mc_reset_find(mc, id);
245 	if (!rst)
246 		return -ENODEV;
247 
248 	rst_ops = mc->soc->reset_ops;
249 	if (!rst_ops)
250 		return -ENODEV;
251 
252 	return rst_ops->reset_status(mc, rst);
253 }
254 
255 static const struct reset_control_ops tegra_mc_reset_ops = {
256 	.assert = tegra_mc_hotreset_assert,
257 	.deassert = tegra_mc_hotreset_deassert,
258 	.status = tegra_mc_hotreset_status,
259 };
260 
261 static int tegra_mc_reset_setup(struct tegra_mc *mc)
262 {
263 	int err;
264 
265 	mc->reset.ops = &tegra_mc_reset_ops;
266 	mc->reset.owner = THIS_MODULE;
267 	mc->reset.of_node = mc->dev->of_node;
268 	mc->reset.of_reset_n_cells = 1;
269 	mc->reset.nr_resets = mc->soc->num_resets;
270 
271 	err = reset_controller_register(&mc->reset);
272 	if (err < 0)
273 		return err;
274 
275 	return 0;
276 }
277 
278 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
279 {
280 	unsigned long long tick;
281 	unsigned int i;
282 	u32 value;
283 
284 	/* compute the number of MC clock cycles per tick */
285 	tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
286 	do_div(tick, NSEC_PER_SEC);
287 
288 	value = mc_readl(mc, MC_EMEM_ARB_CFG);
289 	value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
290 	value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
291 	mc_writel(mc, value, MC_EMEM_ARB_CFG);
292 
293 	/* write latency allowance defaults */
294 	for (i = 0; i < mc->soc->num_clients; i++) {
295 		const struct tegra_mc_la *la = &mc->soc->clients[i].la;
296 		u32 value;
297 
298 		value = mc_readl(mc, la->reg);
299 		value &= ~(la->mask << la->shift);
300 		value |= (la->def & la->mask) << la->shift;
301 		mc_writel(mc, value, la->reg);
302 	}
303 
304 	/* latch new values */
305 	mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
306 
307 	return 0;
308 }
309 
310 void tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
311 {
312 	unsigned int i;
313 	struct tegra_mc_timing *timing = NULL;
314 
315 	for (i = 0; i < mc->num_timings; i++) {
316 		if (mc->timings[i].rate == rate) {
317 			timing = &mc->timings[i];
318 			break;
319 		}
320 	}
321 
322 	if (!timing) {
323 		dev_err(mc->dev, "no memory timing registered for rate %lu\n",
324 			rate);
325 		return;
326 	}
327 
328 	for (i = 0; i < mc->soc->num_emem_regs; ++i)
329 		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
330 }
331 
332 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
333 {
334 	u8 dram_count;
335 
336 	dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
337 	dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
338 	dram_count++;
339 
340 	return dram_count;
341 }
342 
343 static int load_one_timing(struct tegra_mc *mc,
344 			   struct tegra_mc_timing *timing,
345 			   struct device_node *node)
346 {
347 	int err;
348 	u32 tmp;
349 
350 	err = of_property_read_u32(node, "clock-frequency", &tmp);
351 	if (err) {
352 		dev_err(mc->dev,
353 			"timing %pOFn: failed to read rate\n", node);
354 		return err;
355 	}
356 
357 	timing->rate = tmp;
358 	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
359 					 sizeof(u32), GFP_KERNEL);
360 	if (!timing->emem_data)
361 		return -ENOMEM;
362 
363 	err = of_property_read_u32_array(node, "nvidia,emem-configuration",
364 					 timing->emem_data,
365 					 mc->soc->num_emem_regs);
366 	if (err) {
367 		dev_err(mc->dev,
368 			"timing %pOFn: failed to read EMEM configuration\n",
369 			node);
370 		return err;
371 	}
372 
373 	return 0;
374 }
375 
376 static int load_timings(struct tegra_mc *mc, struct device_node *node)
377 {
378 	struct device_node *child;
379 	struct tegra_mc_timing *timing;
380 	int child_count = of_get_child_count(node);
381 	int i = 0, err;
382 
383 	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
384 				   GFP_KERNEL);
385 	if (!mc->timings)
386 		return -ENOMEM;
387 
388 	mc->num_timings = child_count;
389 
390 	for_each_child_of_node(node, child) {
391 		timing = &mc->timings[i++];
392 
393 		err = load_one_timing(mc, timing, child);
394 		if (err) {
395 			of_node_put(child);
396 			return err;
397 		}
398 	}
399 
400 	return 0;
401 }
402 
403 static int tegra_mc_setup_timings(struct tegra_mc *mc)
404 {
405 	struct device_node *node;
406 	u32 ram_code, node_ram_code;
407 	int err;
408 
409 	ram_code = tegra_read_ram_code();
410 
411 	mc->num_timings = 0;
412 
413 	for_each_child_of_node(mc->dev->of_node, node) {
414 		err = of_property_read_u32(node, "nvidia,ram-code",
415 					   &node_ram_code);
416 		if (err || (node_ram_code != ram_code))
417 			continue;
418 
419 		err = load_timings(mc, node);
420 		of_node_put(node);
421 		if (err)
422 			return err;
423 		break;
424 	}
425 
426 	if (mc->num_timings == 0)
427 		dev_warn(mc->dev,
428 			 "no memory timings for RAM code %u registered\n",
429 			 ram_code);
430 
431 	return 0;
432 }
433 
434 static const char *const status_names[32] = {
435 	[ 1] = "External interrupt",
436 	[ 6] = "EMEM address decode error",
437 	[ 7] = "GART page fault",
438 	[ 8] = "Security violation",
439 	[ 9] = "EMEM arbitration error",
440 	[10] = "Page fault",
441 	[11] = "Invalid APB ASID update",
442 	[12] = "VPR violation",
443 	[13] = "Secure carveout violation",
444 	[16] = "MTS carveout violation",
445 };
446 
447 static const char *const error_names[8] = {
448 	[2] = "EMEM decode error",
449 	[3] = "TrustZone violation",
450 	[4] = "Carveout violation",
451 	[6] = "SMMU translation error",
452 };
453 
454 static irqreturn_t tegra_mc_irq(int irq, void *data)
455 {
456 	struct tegra_mc *mc = data;
457 	unsigned long status;
458 	unsigned int bit;
459 
460 	/* mask all interrupts to avoid flooding */
461 	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
462 	if (!status)
463 		return IRQ_NONE;
464 
465 	for_each_set_bit(bit, &status, 32) {
466 		const char *error = status_names[bit] ?: "unknown";
467 		const char *client = "unknown", *desc;
468 		const char *direction, *secure;
469 		phys_addr_t addr = 0;
470 		unsigned int i;
471 		char perm[7];
472 		u8 id, type;
473 		u32 value;
474 
475 		value = mc_readl(mc, MC_ERR_STATUS);
476 
477 #ifdef CONFIG_PHYS_ADDR_T_64BIT
478 		if (mc->soc->num_address_bits > 32) {
479 			addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
480 				MC_ERR_STATUS_ADR_HI_MASK);
481 			addr <<= 32;
482 		}
483 #endif
484 
485 		if (value & MC_ERR_STATUS_RW)
486 			direction = "write";
487 		else
488 			direction = "read";
489 
490 		if (value & MC_ERR_STATUS_SECURITY)
491 			secure = "secure ";
492 		else
493 			secure = "";
494 
495 		id = value & mc->soc->client_id_mask;
496 
497 		for (i = 0; i < mc->soc->num_clients; i++) {
498 			if (mc->soc->clients[i].id == id) {
499 				client = mc->soc->clients[i].name;
500 				break;
501 			}
502 		}
503 
504 		type = (value & MC_ERR_STATUS_TYPE_MASK) >>
505 		       MC_ERR_STATUS_TYPE_SHIFT;
506 		desc = error_names[type];
507 
508 		switch (value & MC_ERR_STATUS_TYPE_MASK) {
509 		case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
510 			perm[0] = ' ';
511 			perm[1] = '[';
512 
513 			if (value & MC_ERR_STATUS_READABLE)
514 				perm[2] = 'R';
515 			else
516 				perm[2] = '-';
517 
518 			if (value & MC_ERR_STATUS_WRITABLE)
519 				perm[3] = 'W';
520 			else
521 				perm[3] = '-';
522 
523 			if (value & MC_ERR_STATUS_NONSECURE)
524 				perm[4] = '-';
525 			else
526 				perm[4] = 'S';
527 
528 			perm[5] = ']';
529 			perm[6] = '\0';
530 			break;
531 
532 		default:
533 			perm[0] = '\0';
534 			break;
535 		}
536 
537 		value = mc_readl(mc, MC_ERR_ADR);
538 		addr |= value;
539 
540 		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
541 				    client, secure, direction, &addr, error,
542 				    desc, perm);
543 	}
544 
545 	/* clear interrupts */
546 	mc_writel(mc, status, MC_INTSTATUS);
547 
548 	return IRQ_HANDLED;
549 }
550 
551 static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
552 {
553 	struct tegra_mc *mc = data;
554 	unsigned long status;
555 	unsigned int bit;
556 
557 	/* mask all interrupts to avoid flooding */
558 	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
559 	if (!status)
560 		return IRQ_NONE;
561 
562 	for_each_set_bit(bit, &status, 32) {
563 		const char *direction = "read", *secure = "";
564 		const char *error = status_names[bit];
565 		const char *client, *desc;
566 		phys_addr_t addr;
567 		u32 value, reg;
568 		u8 id, type;
569 
570 		switch (BIT(bit)) {
571 		case MC_INT_DECERR_EMEM:
572 			reg = MC_DECERR_EMEM_OTHERS_STATUS;
573 			value = mc_readl(mc, reg);
574 
575 			id = value & mc->soc->client_id_mask;
576 			desc = error_names[2];
577 
578 			if (value & BIT(31))
579 				direction = "write";
580 			break;
581 
582 		case MC_INT_INVALID_GART_PAGE:
583 			reg = MC_GART_ERROR_REQ;
584 			value = mc_readl(mc, reg);
585 
586 			id = (value >> 1) & mc->soc->client_id_mask;
587 			desc = error_names[2];
588 
589 			if (value & BIT(0))
590 				direction = "write";
591 			break;
592 
593 		case MC_INT_SECURITY_VIOLATION:
594 			reg = MC_SECURITY_VIOLATION_STATUS;
595 			value = mc_readl(mc, reg);
596 
597 			id = value & mc->soc->client_id_mask;
598 			type = (value & BIT(30)) ? 4 : 3;
599 			desc = error_names[type];
600 			secure = "secure ";
601 
602 			if (value & BIT(31))
603 				direction = "write";
604 			break;
605 
606 		default:
607 			continue;
608 		}
609 
610 		client = mc->soc->clients[id].name;
611 		addr = mc_readl(mc, reg + sizeof(u32));
612 
613 		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
614 				    client, secure, direction, &addr, error,
615 				    desc);
616 	}
617 
618 	/* clear interrupts */
619 	mc_writel(mc, status, MC_INTSTATUS);
620 
621 	return IRQ_HANDLED;
622 }
623 
624 static int tegra_mc_probe(struct platform_device *pdev)
625 {
626 	struct resource *res;
627 	struct tegra_mc *mc;
628 	void *isr;
629 	int err;
630 
631 	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
632 	if (!mc)
633 		return -ENOMEM;
634 
635 	platform_set_drvdata(pdev, mc);
636 	spin_lock_init(&mc->lock);
637 	mc->soc = of_device_get_match_data(&pdev->dev);
638 	mc->dev = &pdev->dev;
639 
640 	/* length of MC tick in nanoseconds */
641 	mc->tick = 30;
642 
643 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
644 	mc->regs = devm_ioremap_resource(&pdev->dev, res);
645 	if (IS_ERR(mc->regs))
646 		return PTR_ERR(mc->regs);
647 
648 	mc->clk = devm_clk_get(&pdev->dev, "mc");
649 	if (IS_ERR(mc->clk)) {
650 		dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
651 			PTR_ERR(mc->clk));
652 		return PTR_ERR(mc->clk);
653 	}
654 
655 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
656 	if (mc->soc == &tegra20_mc_soc) {
657 		isr = tegra20_mc_irq;
658 	} else
659 #endif
660 	{
661 		err = tegra_mc_setup_latency_allowance(mc);
662 		if (err < 0) {
663 			dev_err(&pdev->dev,
664 				"failed to setup latency allowance: %d\n",
665 				err);
666 			return err;
667 		}
668 
669 		isr = tegra_mc_irq;
670 
671 		err = tegra_mc_setup_timings(mc);
672 		if (err < 0) {
673 			dev_err(&pdev->dev, "failed to setup timings: %d\n",
674 				err);
675 			return err;
676 		}
677 	}
678 
679 	mc->irq = platform_get_irq(pdev, 0);
680 	if (mc->irq < 0) {
681 		dev_err(&pdev->dev, "interrupt not specified\n");
682 		return mc->irq;
683 	}
684 
685 	WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
686 
687 	mc_writel(mc, mc->soc->intmask, MC_INTMASK);
688 
689 	err = devm_request_irq(&pdev->dev, mc->irq, isr, 0,
690 			       dev_name(&pdev->dev), mc);
691 	if (err < 0) {
692 		dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
693 			err);
694 		return err;
695 	}
696 
697 	err = tegra_mc_reset_setup(mc);
698 	if (err < 0)
699 		dev_err(&pdev->dev, "failed to register reset controller: %d\n",
700 			err);
701 
702 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
703 		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
704 		if (IS_ERR(mc->smmu)) {
705 			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
706 				PTR_ERR(mc->smmu));
707 			mc->smmu = NULL;
708 		}
709 	}
710 
711 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
712 		mc->gart = tegra_gart_probe(&pdev->dev, mc);
713 		if (IS_ERR(mc->gart)) {
714 			dev_err(&pdev->dev, "failed to probe GART: %ld\n",
715 				PTR_ERR(mc->gart));
716 			mc->gart = NULL;
717 		}
718 	}
719 
720 	return 0;
721 }
722 
723 static int tegra_mc_suspend(struct device *dev)
724 {
725 	struct tegra_mc *mc = dev_get_drvdata(dev);
726 	int err;
727 
728 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
729 		err = tegra_gart_suspend(mc->gart);
730 		if (err)
731 			return err;
732 	}
733 
734 	return 0;
735 }
736 
737 static int tegra_mc_resume(struct device *dev)
738 {
739 	struct tegra_mc *mc = dev_get_drvdata(dev);
740 	int err;
741 
742 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
743 		err = tegra_gart_resume(mc->gart);
744 		if (err)
745 			return err;
746 	}
747 
748 	return 0;
749 }
750 
751 static const struct dev_pm_ops tegra_mc_pm_ops = {
752 	.suspend = tegra_mc_suspend,
753 	.resume = tegra_mc_resume,
754 };
755 
756 static struct platform_driver tegra_mc_driver = {
757 	.driver = {
758 		.name = "tegra-mc",
759 		.of_match_table = tegra_mc_of_match,
760 		.pm = &tegra_mc_pm_ops,
761 		.suppress_bind_attrs = true,
762 	},
763 	.prevent_deferred_probe = true,
764 	.probe = tegra_mc_probe,
765 };
766 
767 static int tegra_mc_init(void)
768 {
769 	return platform_driver_register(&tegra_mc_driver);
770 }
771 arch_initcall(tegra_mc_init);
772 
773 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
774 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
775 MODULE_LICENSE("GPL v2");
776