xref: /openbmc/linux/drivers/memory/tegra/mc.c (revision 0b483871)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/clk.h>
7 #include <linux/delay.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/export.h>
10 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/of.h>
14 #include <linux/of_platform.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/sort.h>
18 #include <linux/tegra-icc.h>
19 
20 #include <soc/tegra/fuse.h>
21 
22 #include "mc.h"
23 
24 static const struct of_device_id tegra_mc_of_match[] = {
25 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
26 	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
27 #endif
28 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
29 	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
30 #endif
31 #ifdef CONFIG_ARCH_TEGRA_114_SOC
32 	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
33 #endif
34 #ifdef CONFIG_ARCH_TEGRA_124_SOC
35 	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
36 #endif
37 #ifdef CONFIG_ARCH_TEGRA_132_SOC
38 	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
39 #endif
40 #ifdef CONFIG_ARCH_TEGRA_210_SOC
41 	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
42 #endif
43 #ifdef CONFIG_ARCH_TEGRA_186_SOC
44 	{ .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
45 #endif
46 #ifdef CONFIG_ARCH_TEGRA_194_SOC
47 	{ .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
48 #endif
49 #ifdef CONFIG_ARCH_TEGRA_234_SOC
50 	{ .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
51 #endif
52 	{ /* sentinel */ }
53 };
54 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
55 
tegra_mc_devm_action_put_device(void * data)56 static void tegra_mc_devm_action_put_device(void *data)
57 {
58 	struct tegra_mc *mc = data;
59 
60 	put_device(mc->dev);
61 }
62 
63 /**
64  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
65  * @dev: device pointer for the consumer device
66  *
67  * This function will search for the Memory Controller node in a device-tree
68  * and retrieve the Memory Controller handle.
69  *
70  * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
71  */
devm_tegra_memory_controller_get(struct device * dev)72 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
73 {
74 	struct platform_device *pdev;
75 	struct device_node *np;
76 	struct tegra_mc *mc;
77 	int err;
78 
79 	np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
80 	if (!np)
81 		return ERR_PTR(-ENOENT);
82 
83 	pdev = of_find_device_by_node(np);
84 	of_node_put(np);
85 	if (!pdev)
86 		return ERR_PTR(-ENODEV);
87 
88 	mc = platform_get_drvdata(pdev);
89 	if (!mc) {
90 		put_device(&pdev->dev);
91 		return ERR_PTR(-EPROBE_DEFER);
92 	}
93 
94 	err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
95 	if (err)
96 		return ERR_PTR(err);
97 
98 	return mc;
99 }
100 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
101 
tegra_mc_probe_device(struct tegra_mc * mc,struct device * dev)102 int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
103 {
104 	if (mc->soc->ops && mc->soc->ops->probe_device)
105 		return mc->soc->ops->probe_device(mc, dev);
106 
107 	return 0;
108 }
109 EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
110 
tegra_mc_get_carveout_info(struct tegra_mc * mc,unsigned int id,phys_addr_t * base,u64 * size)111 int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id,
112                                phys_addr_t *base, u64 *size)
113 {
114 	u32 offset;
115 
116 	if (id < 1 || id >= mc->soc->num_carveouts)
117 		return -EINVAL;
118 
119 	if (id < 6)
120 		offset = 0xc0c + 0x50 * (id - 1);
121 	else
122 		offset = 0x2004 + 0x50 * (id - 6);
123 
124 	*base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0);
125 #ifdef CONFIG_PHYS_ADDR_T_64BIT
126 	*base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32;
127 #endif
128 
129 	if (size)
130 		*size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17;
131 
132 	return 0;
133 }
134 EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info);
135 
tegra_mc_block_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)136 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
137 				     const struct tegra_mc_reset *rst)
138 {
139 	unsigned long flags;
140 	u32 value;
141 
142 	spin_lock_irqsave(&mc->lock, flags);
143 
144 	value = mc_readl(mc, rst->control) | BIT(rst->bit);
145 	mc_writel(mc, value, rst->control);
146 
147 	spin_unlock_irqrestore(&mc->lock, flags);
148 
149 	return 0;
150 }
151 
tegra_mc_dma_idling_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)152 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
153 				       const struct tegra_mc_reset *rst)
154 {
155 	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
156 }
157 
tegra_mc_unblock_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)158 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
159 				       const struct tegra_mc_reset *rst)
160 {
161 	unsigned long flags;
162 	u32 value;
163 
164 	spin_lock_irqsave(&mc->lock, flags);
165 
166 	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
167 	mc_writel(mc, value, rst->control);
168 
169 	spin_unlock_irqrestore(&mc->lock, flags);
170 
171 	return 0;
172 }
173 
tegra_mc_reset_status_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)174 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
175 					const struct tegra_mc_reset *rst)
176 {
177 	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
178 }
179 
180 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
181 	.block_dma = tegra_mc_block_dma_common,
182 	.dma_idling = tegra_mc_dma_idling_common,
183 	.unblock_dma = tegra_mc_unblock_dma_common,
184 	.reset_status = tegra_mc_reset_status_common,
185 };
186 
reset_to_mc(struct reset_controller_dev * rcdev)187 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
188 {
189 	return container_of(rcdev, struct tegra_mc, reset);
190 }
191 
tegra_mc_reset_find(struct tegra_mc * mc,unsigned long id)192 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
193 							unsigned long id)
194 {
195 	unsigned int i;
196 
197 	for (i = 0; i < mc->soc->num_resets; i++)
198 		if (mc->soc->resets[i].id == id)
199 			return &mc->soc->resets[i];
200 
201 	return NULL;
202 }
203 
tegra_mc_hotreset_assert(struct reset_controller_dev * rcdev,unsigned long id)204 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
205 				    unsigned long id)
206 {
207 	struct tegra_mc *mc = reset_to_mc(rcdev);
208 	const struct tegra_mc_reset_ops *rst_ops;
209 	const struct tegra_mc_reset *rst;
210 	int retries = 500;
211 	int err;
212 
213 	rst = tegra_mc_reset_find(mc, id);
214 	if (!rst)
215 		return -ENODEV;
216 
217 	rst_ops = mc->soc->reset_ops;
218 	if (!rst_ops)
219 		return -ENODEV;
220 
221 	/* DMA flushing will fail if reset is already asserted */
222 	if (rst_ops->reset_status) {
223 		/* check whether reset is asserted */
224 		if (rst_ops->reset_status(mc, rst))
225 			return 0;
226 	}
227 
228 	if (rst_ops->block_dma) {
229 		/* block clients DMA requests */
230 		err = rst_ops->block_dma(mc, rst);
231 		if (err) {
232 			dev_err(mc->dev, "failed to block %s DMA: %d\n",
233 				rst->name, err);
234 			return err;
235 		}
236 	}
237 
238 	if (rst_ops->dma_idling) {
239 		/* wait for completion of the outstanding DMA requests */
240 		while (!rst_ops->dma_idling(mc, rst)) {
241 			if (!retries--) {
242 				dev_err(mc->dev, "failed to flush %s DMA\n",
243 					rst->name);
244 				return -EBUSY;
245 			}
246 
247 			usleep_range(10, 100);
248 		}
249 	}
250 
251 	if (rst_ops->hotreset_assert) {
252 		/* clear clients DMA requests sitting before arbitration */
253 		err = rst_ops->hotreset_assert(mc, rst);
254 		if (err) {
255 			dev_err(mc->dev, "failed to hot reset %s: %d\n",
256 				rst->name, err);
257 			return err;
258 		}
259 	}
260 
261 	return 0;
262 }
263 
tegra_mc_hotreset_deassert(struct reset_controller_dev * rcdev,unsigned long id)264 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
265 				      unsigned long id)
266 {
267 	struct tegra_mc *mc = reset_to_mc(rcdev);
268 	const struct tegra_mc_reset_ops *rst_ops;
269 	const struct tegra_mc_reset *rst;
270 	int err;
271 
272 	rst = tegra_mc_reset_find(mc, id);
273 	if (!rst)
274 		return -ENODEV;
275 
276 	rst_ops = mc->soc->reset_ops;
277 	if (!rst_ops)
278 		return -ENODEV;
279 
280 	if (rst_ops->hotreset_deassert) {
281 		/* take out client from hot reset */
282 		err = rst_ops->hotreset_deassert(mc, rst);
283 		if (err) {
284 			dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
285 				rst->name, err);
286 			return err;
287 		}
288 	}
289 
290 	if (rst_ops->unblock_dma) {
291 		/* allow new DMA requests to proceed to arbitration */
292 		err = rst_ops->unblock_dma(mc, rst);
293 		if (err) {
294 			dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
295 				rst->name, err);
296 			return err;
297 		}
298 	}
299 
300 	return 0;
301 }
302 
tegra_mc_hotreset_status(struct reset_controller_dev * rcdev,unsigned long id)303 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
304 				    unsigned long id)
305 {
306 	struct tegra_mc *mc = reset_to_mc(rcdev);
307 	const struct tegra_mc_reset_ops *rst_ops;
308 	const struct tegra_mc_reset *rst;
309 
310 	rst = tegra_mc_reset_find(mc, id);
311 	if (!rst)
312 		return -ENODEV;
313 
314 	rst_ops = mc->soc->reset_ops;
315 	if (!rst_ops)
316 		return -ENODEV;
317 
318 	return rst_ops->reset_status(mc, rst);
319 }
320 
321 static const struct reset_control_ops tegra_mc_reset_ops = {
322 	.assert = tegra_mc_hotreset_assert,
323 	.deassert = tegra_mc_hotreset_deassert,
324 	.status = tegra_mc_hotreset_status,
325 };
326 
tegra_mc_reset_setup(struct tegra_mc * mc)327 static int tegra_mc_reset_setup(struct tegra_mc *mc)
328 {
329 	int err;
330 
331 	mc->reset.ops = &tegra_mc_reset_ops;
332 	mc->reset.owner = THIS_MODULE;
333 	mc->reset.of_node = mc->dev->of_node;
334 	mc->reset.of_reset_n_cells = 1;
335 	mc->reset.nr_resets = mc->soc->num_resets;
336 
337 	err = reset_controller_register(&mc->reset);
338 	if (err < 0)
339 		return err;
340 
341 	return 0;
342 }
343 
tegra_mc_write_emem_configuration(struct tegra_mc * mc,unsigned long rate)344 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
345 {
346 	unsigned int i;
347 	struct tegra_mc_timing *timing = NULL;
348 
349 	for (i = 0; i < mc->num_timings; i++) {
350 		if (mc->timings[i].rate == rate) {
351 			timing = &mc->timings[i];
352 			break;
353 		}
354 	}
355 
356 	if (!timing) {
357 		dev_err(mc->dev, "no memory timing registered for rate %lu\n",
358 			rate);
359 		return -EINVAL;
360 	}
361 
362 	for (i = 0; i < mc->soc->num_emem_regs; ++i)
363 		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
364 
365 	return 0;
366 }
367 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
368 
tegra_mc_get_emem_device_count(struct tegra_mc * mc)369 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
370 {
371 	u8 dram_count;
372 
373 	dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
374 	dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
375 	dram_count++;
376 
377 	return dram_count;
378 }
379 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
380 
381 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
382     defined(CONFIG_ARCH_TEGRA_114_SOC) || \
383     defined(CONFIG_ARCH_TEGRA_124_SOC) || \
384     defined(CONFIG_ARCH_TEGRA_132_SOC) || \
385     defined(CONFIG_ARCH_TEGRA_210_SOC)
tegra_mc_setup_latency_allowance(struct tegra_mc * mc)386 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
387 {
388 	unsigned long long tick;
389 	unsigned int i;
390 	u32 value;
391 
392 	/* compute the number of MC clock cycles per tick */
393 	tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
394 	do_div(tick, NSEC_PER_SEC);
395 
396 	value = mc_readl(mc, MC_EMEM_ARB_CFG);
397 	value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
398 	value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
399 	mc_writel(mc, value, MC_EMEM_ARB_CFG);
400 
401 	/* write latency allowance defaults */
402 	for (i = 0; i < mc->soc->num_clients; i++) {
403 		const struct tegra_mc_client *client = &mc->soc->clients[i];
404 		u32 value;
405 
406 		value = mc_readl(mc, client->regs.la.reg);
407 		value &= ~(client->regs.la.mask << client->regs.la.shift);
408 		value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
409 		mc_writel(mc, value, client->regs.la.reg);
410 	}
411 
412 	/* latch new values */
413 	mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
414 
415 	return 0;
416 }
417 
load_one_timing(struct tegra_mc * mc,struct tegra_mc_timing * timing,struct device_node * node)418 static int load_one_timing(struct tegra_mc *mc,
419 			   struct tegra_mc_timing *timing,
420 			   struct device_node *node)
421 {
422 	int err;
423 	u32 tmp;
424 
425 	err = of_property_read_u32(node, "clock-frequency", &tmp);
426 	if (err) {
427 		dev_err(mc->dev,
428 			"timing %pOFn: failed to read rate\n", node);
429 		return err;
430 	}
431 
432 	timing->rate = tmp;
433 	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
434 					 sizeof(u32), GFP_KERNEL);
435 	if (!timing->emem_data)
436 		return -ENOMEM;
437 
438 	err = of_property_read_u32_array(node, "nvidia,emem-configuration",
439 					 timing->emem_data,
440 					 mc->soc->num_emem_regs);
441 	if (err) {
442 		dev_err(mc->dev,
443 			"timing %pOFn: failed to read EMEM configuration\n",
444 			node);
445 		return err;
446 	}
447 
448 	return 0;
449 }
450 
load_timings(struct tegra_mc * mc,struct device_node * node)451 static int load_timings(struct tegra_mc *mc, struct device_node *node)
452 {
453 	struct device_node *child;
454 	struct tegra_mc_timing *timing;
455 	int child_count = of_get_child_count(node);
456 	int i = 0, err;
457 
458 	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
459 				   GFP_KERNEL);
460 	if (!mc->timings)
461 		return -ENOMEM;
462 
463 	mc->num_timings = child_count;
464 
465 	for_each_child_of_node(node, child) {
466 		timing = &mc->timings[i++];
467 
468 		err = load_one_timing(mc, timing, child);
469 		if (err) {
470 			of_node_put(child);
471 			return err;
472 		}
473 	}
474 
475 	return 0;
476 }
477 
tegra_mc_setup_timings(struct tegra_mc * mc)478 static int tegra_mc_setup_timings(struct tegra_mc *mc)
479 {
480 	struct device_node *node;
481 	u32 ram_code, node_ram_code;
482 	int err;
483 
484 	ram_code = tegra_read_ram_code();
485 
486 	mc->num_timings = 0;
487 
488 	for_each_child_of_node(mc->dev->of_node, node) {
489 		err = of_property_read_u32(node, "nvidia,ram-code",
490 					   &node_ram_code);
491 		if (err || (node_ram_code != ram_code))
492 			continue;
493 
494 		err = load_timings(mc, node);
495 		of_node_put(node);
496 		if (err)
497 			return err;
498 		break;
499 	}
500 
501 	if (mc->num_timings == 0)
502 		dev_warn(mc->dev,
503 			 "no memory timings for RAM code %u registered\n",
504 			 ram_code);
505 
506 	return 0;
507 }
508 
tegra30_mc_probe(struct tegra_mc * mc)509 int tegra30_mc_probe(struct tegra_mc *mc)
510 {
511 	int err;
512 
513 	mc->clk = devm_clk_get_optional(mc->dev, "mc");
514 	if (IS_ERR(mc->clk)) {
515 		dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
516 		return PTR_ERR(mc->clk);
517 	}
518 
519 	/* ensure that debug features are disabled */
520 	mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
521 
522 	err = tegra_mc_setup_latency_allowance(mc);
523 	if (err < 0) {
524 		dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
525 		return err;
526 	}
527 
528 	err = tegra_mc_setup_timings(mc);
529 	if (err < 0) {
530 		dev_err(mc->dev, "failed to setup timings: %d\n", err);
531 		return err;
532 	}
533 
534 	return 0;
535 }
536 
537 const struct tegra_mc_ops tegra30_mc_ops = {
538 	.probe = tegra30_mc_probe,
539 	.handle_irq = tegra30_mc_handle_irq,
540 };
541 #endif
542 
mc_global_intstatus_to_channel(const struct tegra_mc * mc,u32 status,unsigned int * mc_channel)543 static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
544 					  unsigned int *mc_channel)
545 {
546 	if ((status & mc->soc->ch_intmask) == 0)
547 		return -EINVAL;
548 
549 	*mc_channel = __ffs((status & mc->soc->ch_intmask) >>
550 			    mc->soc->global_intstatus_channel_shift);
551 
552 	return 0;
553 }
554 
mc_channel_to_global_intstatus(const struct tegra_mc * mc,unsigned int channel)555 static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
556 					  unsigned int channel)
557 {
558 	return BIT(channel) << mc->soc->global_intstatus_channel_shift;
559 }
560 
tegra30_mc_handle_irq(int irq,void * data)561 irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
562 {
563 	struct tegra_mc *mc = data;
564 	unsigned int bit, channel;
565 	unsigned long status;
566 
567 	if (mc->soc->num_channels) {
568 		u32 global_status;
569 		int err;
570 
571 		global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
572 		err = mc_global_intstatus_to_channel(mc, global_status, &channel);
573 		if (err < 0) {
574 			dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
575 					    global_status);
576 			return IRQ_NONE;
577 		}
578 
579 		/* mask all interrupts to avoid flooding */
580 		status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
581 	} else {
582 		status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
583 	}
584 
585 	if (!status)
586 		return IRQ_NONE;
587 
588 	for_each_set_bit(bit, &status, 32) {
589 		const char *error = tegra_mc_status_names[bit] ?: "unknown";
590 		const char *client = "unknown", *desc;
591 		const char *direction, *secure;
592 		u32 status_reg, addr_reg;
593 		u32 intmask = BIT(bit);
594 		phys_addr_t addr = 0;
595 #ifdef CONFIG_PHYS_ADDR_T_64BIT
596 		u32 addr_hi_reg = 0;
597 #endif
598 		unsigned int i;
599 		char perm[7];
600 		u8 id, type;
601 		u32 value;
602 
603 		switch (intmask) {
604 		case MC_INT_DECERR_VPR:
605 			status_reg = MC_ERR_VPR_STATUS;
606 			addr_reg = MC_ERR_VPR_ADR;
607 			break;
608 
609 		case MC_INT_SECERR_SEC:
610 			status_reg = MC_ERR_SEC_STATUS;
611 			addr_reg = MC_ERR_SEC_ADR;
612 			break;
613 
614 		case MC_INT_DECERR_MTS:
615 			status_reg = MC_ERR_MTS_STATUS;
616 			addr_reg = MC_ERR_MTS_ADR;
617 			break;
618 
619 		case MC_INT_DECERR_GENERALIZED_CARVEOUT:
620 			status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
621 			addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
622 			break;
623 
624 		case MC_INT_DECERR_ROUTE_SANITY:
625 			status_reg = MC_ERR_ROUTE_SANITY_STATUS;
626 			addr_reg = MC_ERR_ROUTE_SANITY_ADR;
627 			break;
628 
629 		default:
630 			status_reg = MC_ERR_STATUS;
631 			addr_reg = MC_ERR_ADR;
632 
633 #ifdef CONFIG_PHYS_ADDR_T_64BIT
634 			if (mc->soc->has_addr_hi_reg)
635 				addr_hi_reg = MC_ERR_ADR_HI;
636 #endif
637 			break;
638 		}
639 
640 		if (mc->soc->num_channels)
641 			value = mc_ch_readl(mc, channel, status_reg);
642 		else
643 			value = mc_readl(mc, status_reg);
644 
645 #ifdef CONFIG_PHYS_ADDR_T_64BIT
646 		if (mc->soc->num_address_bits > 32) {
647 			if (addr_hi_reg) {
648 				if (mc->soc->num_channels)
649 					addr = mc_ch_readl(mc, channel, addr_hi_reg);
650 				else
651 					addr = mc_readl(mc, addr_hi_reg);
652 			} else {
653 				addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
654 					MC_ERR_STATUS_ADR_HI_MASK);
655 			}
656 			addr <<= 32;
657 		}
658 #endif
659 
660 		if (value & MC_ERR_STATUS_RW)
661 			direction = "write";
662 		else
663 			direction = "read";
664 
665 		if (value & MC_ERR_STATUS_SECURITY)
666 			secure = "secure ";
667 		else
668 			secure = "";
669 
670 		id = value & mc->soc->client_id_mask;
671 
672 		for (i = 0; i < mc->soc->num_clients; i++) {
673 			if (mc->soc->clients[i].id == id) {
674 				client = mc->soc->clients[i].name;
675 				break;
676 			}
677 		}
678 
679 		type = (value & MC_ERR_STATUS_TYPE_MASK) >>
680 		       MC_ERR_STATUS_TYPE_SHIFT;
681 		desc = tegra_mc_error_names[type];
682 
683 		switch (value & MC_ERR_STATUS_TYPE_MASK) {
684 		case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
685 			perm[0] = ' ';
686 			perm[1] = '[';
687 
688 			if (value & MC_ERR_STATUS_READABLE)
689 				perm[2] = 'R';
690 			else
691 				perm[2] = '-';
692 
693 			if (value & MC_ERR_STATUS_WRITABLE)
694 				perm[3] = 'W';
695 			else
696 				perm[3] = '-';
697 
698 			if (value & MC_ERR_STATUS_NONSECURE)
699 				perm[4] = '-';
700 			else
701 				perm[4] = 'S';
702 
703 			perm[5] = ']';
704 			perm[6] = '\0';
705 			break;
706 
707 		default:
708 			perm[0] = '\0';
709 			break;
710 		}
711 
712 		if (mc->soc->num_channels)
713 			value = mc_ch_readl(mc, channel, addr_reg);
714 		else
715 			value = mc_readl(mc, addr_reg);
716 		addr |= value;
717 
718 		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
719 				    client, secure, direction, &addr, error,
720 				    desc, perm);
721 	}
722 
723 	/* clear interrupts */
724 	if (mc->soc->num_channels) {
725 		mc_ch_writel(mc, channel, status, MC_INTSTATUS);
726 		mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
727 			     mc_channel_to_global_intstatus(mc, channel),
728 			     MC_GLOBAL_INTSTATUS);
729 	} else {
730 		mc_writel(mc, status, MC_INTSTATUS);
731 	}
732 
733 	return IRQ_HANDLED;
734 }
735 
736 const char *const tegra_mc_status_names[32] = {
737 	[ 1] = "External interrupt",
738 	[ 6] = "EMEM address decode error",
739 	[ 7] = "GART page fault",
740 	[ 8] = "Security violation",
741 	[ 9] = "EMEM arbitration error",
742 	[10] = "Page fault",
743 	[11] = "Invalid APB ASID update",
744 	[12] = "VPR violation",
745 	[13] = "Secure carveout violation",
746 	[16] = "MTS carveout violation",
747 	[17] = "Generalized carveout violation",
748 	[20] = "Route Sanity error",
749 };
750 
751 const char *const tegra_mc_error_names[8] = {
752 	[2] = "EMEM decode error",
753 	[3] = "TrustZone violation",
754 	[4] = "Carveout violation",
755 	[6] = "SMMU translation error",
756 };
757 
tegra_mc_icc_xlate(struct of_phandle_args * spec,void * data)758 struct icc_node *tegra_mc_icc_xlate(struct of_phandle_args *spec, void *data)
759 {
760 	struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
761 	struct icc_node *node;
762 
763 	list_for_each_entry(node, &mc->provider.nodes, node_list) {
764 		if (node->id == spec->args[0])
765 			return node;
766 	}
767 
768 	/*
769 	 * If a client driver calls devm_of_icc_get() before the MC driver
770 	 * is probed, then return EPROBE_DEFER to the client driver.
771 	 */
772 	return ERR_PTR(-EPROBE_DEFER);
773 }
774 
tegra_mc_icc_get(struct icc_node * node,u32 * average,u32 * peak)775 static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak)
776 {
777 	*average = 0;
778 	*peak = 0;
779 
780 	return 0;
781 }
782 
tegra_mc_icc_set(struct icc_node * src,struct icc_node * dst)783 static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst)
784 {
785 	return 0;
786 }
787 
788 const struct tegra_mc_icc_ops tegra_mc_icc_ops = {
789 	.xlate = tegra_mc_icc_xlate,
790 	.aggregate = icc_std_aggregate,
791 	.get_bw = tegra_mc_icc_get,
792 	.set = tegra_mc_icc_set,
793 };
794 
795 /*
796  * Memory Controller (MC) has few Memory Clients that are issuing memory
797  * bandwidth allocation requests to the MC interconnect provider. The MC
798  * provider aggregates the requests and then sends the aggregated request
799  * up to the External Memory Controller (EMC) interconnect provider which
800  * re-configures hardware interface to External Memory (EMEM) in accordance
801  * to the required bandwidth. Each MC interconnect node represents an
802  * individual Memory Client.
803  *
804  * Memory interconnect topology:
805  *
806  *               +----+
807  * +--------+    |    |
808  * | TEXSRD +--->+    |
809  * +--------+    |    |
810  *               |    |    +-----+    +------+
811  *    ...        | MC +--->+ EMC +--->+ EMEM |
812  *               |    |    +-----+    +------+
813  * +--------+    |    |
814  * | DISP.. +--->+    |
815  * +--------+    |    |
816  *               +----+
817  */
tegra_mc_interconnect_setup(struct tegra_mc * mc)818 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
819 {
820 	struct icc_node *node;
821 	unsigned int i;
822 	int err;
823 
824 	/* older device-trees don't have interconnect properties */
825 	if (!device_property_present(mc->dev, "#interconnect-cells") ||
826 	    !mc->soc->icc_ops)
827 		return 0;
828 
829 	mc->provider.dev = mc->dev;
830 	mc->provider.data = &mc->provider;
831 	mc->provider.set = mc->soc->icc_ops->set;
832 	mc->provider.aggregate = mc->soc->icc_ops->aggregate;
833 	mc->provider.get_bw = mc->soc->icc_ops->get_bw;
834 	mc->provider.xlate = mc->soc->icc_ops->xlate;
835 	mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
836 
837 	icc_provider_init(&mc->provider);
838 
839 	/* create Memory Controller node */
840 	node = icc_node_create(TEGRA_ICC_MC);
841 	if (IS_ERR(node))
842 		return PTR_ERR(node);
843 
844 	node->name = "Memory Controller";
845 	icc_node_add(node, &mc->provider);
846 
847 	/* link Memory Controller to External Memory Controller */
848 	err = icc_link_create(node, TEGRA_ICC_EMC);
849 	if (err)
850 		goto remove_nodes;
851 
852 	for (i = 0; i < mc->soc->num_clients; i++) {
853 		/* create MC client node */
854 		node = icc_node_create(mc->soc->clients[i].id);
855 		if (IS_ERR(node)) {
856 			err = PTR_ERR(node);
857 			goto remove_nodes;
858 		}
859 
860 		node->name = mc->soc->clients[i].name;
861 		icc_node_add(node, &mc->provider);
862 
863 		/* link Memory Client to Memory Controller */
864 		err = icc_link_create(node, TEGRA_ICC_MC);
865 		if (err)
866 			goto remove_nodes;
867 
868 		node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]);
869 	}
870 
871 	err = icc_provider_register(&mc->provider);
872 	if (err)
873 		goto remove_nodes;
874 
875 	return 0;
876 
877 remove_nodes:
878 	icc_nodes_remove(&mc->provider);
879 
880 	return err;
881 }
882 
tegra_mc_num_channel_enabled(struct tegra_mc * mc)883 static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
884 {
885 	unsigned int i;
886 	u32 value;
887 
888 	value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE);
889 	if (value <= 0) {
890 		mc->num_channels = mc->soc->num_channels;
891 		return;
892 	}
893 
894 	for (i = 0; i < 32; i++) {
895 		if (value & BIT(i))
896 			mc->num_channels++;
897 	}
898 }
899 
tegra_mc_probe(struct platform_device * pdev)900 static int tegra_mc_probe(struct platform_device *pdev)
901 {
902 	struct tegra_mc *mc;
903 	u64 mask;
904 	int err;
905 
906 	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
907 	if (!mc)
908 		return -ENOMEM;
909 
910 	platform_set_drvdata(pdev, mc);
911 	spin_lock_init(&mc->lock);
912 	mc->soc = of_device_get_match_data(&pdev->dev);
913 	mc->dev = &pdev->dev;
914 
915 	mask = DMA_BIT_MASK(mc->soc->num_address_bits);
916 
917 	err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
918 	if (err < 0) {
919 		dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
920 		return err;
921 	}
922 
923 	/* length of MC tick in nanoseconds */
924 	mc->tick = 30;
925 
926 	mc->regs = devm_platform_ioremap_resource(pdev, 0);
927 	if (IS_ERR(mc->regs))
928 		return PTR_ERR(mc->regs);
929 
930 	mc->debugfs.root = debugfs_create_dir("mc", NULL);
931 
932 	if (mc->soc->ops && mc->soc->ops->probe) {
933 		err = mc->soc->ops->probe(mc);
934 		if (err < 0)
935 			return err;
936 	}
937 
938 	tegra_mc_num_channel_enabled(mc);
939 
940 	if (mc->soc->ops && mc->soc->ops->handle_irq) {
941 		mc->irq = platform_get_irq(pdev, 0);
942 		if (mc->irq < 0)
943 			return mc->irq;
944 
945 		WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
946 
947 		if (mc->soc->num_channels)
948 			mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
949 				     MC_INTMASK);
950 		else
951 			mc_writel(mc, mc->soc->intmask, MC_INTMASK);
952 
953 		err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
954 				       dev_name(&pdev->dev), mc);
955 		if (err < 0) {
956 			dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
957 				err);
958 			return err;
959 		}
960 	}
961 
962 	if (mc->soc->reset_ops) {
963 		err = tegra_mc_reset_setup(mc);
964 		if (err < 0)
965 			dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
966 	}
967 
968 	err = tegra_mc_interconnect_setup(mc);
969 	if (err < 0)
970 		dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
971 			err);
972 
973 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
974 		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
975 		if (IS_ERR(mc->smmu)) {
976 			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
977 				PTR_ERR(mc->smmu));
978 			mc->smmu = NULL;
979 		}
980 	}
981 
982 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
983 		mc->gart = tegra_gart_probe(&pdev->dev, mc);
984 		if (IS_ERR(mc->gart)) {
985 			dev_err(&pdev->dev, "failed to probe GART: %ld\n",
986 				PTR_ERR(mc->gart));
987 			mc->gart = NULL;
988 		}
989 	}
990 
991 	return 0;
992 }
993 
tegra_mc_suspend(struct device * dev)994 static int __maybe_unused tegra_mc_suspend(struct device *dev)
995 {
996 	struct tegra_mc *mc = dev_get_drvdata(dev);
997 
998 	if (mc->soc->ops && mc->soc->ops->suspend)
999 		return mc->soc->ops->suspend(mc);
1000 
1001 	return 0;
1002 }
1003 
tegra_mc_resume(struct device * dev)1004 static int __maybe_unused tegra_mc_resume(struct device *dev)
1005 {
1006 	struct tegra_mc *mc = dev_get_drvdata(dev);
1007 
1008 	if (mc->soc->ops && mc->soc->ops->resume)
1009 		return mc->soc->ops->resume(mc);
1010 
1011 	return 0;
1012 }
1013 
tegra_mc_sync_state(struct device * dev)1014 static void tegra_mc_sync_state(struct device *dev)
1015 {
1016 	struct tegra_mc *mc = dev_get_drvdata(dev);
1017 
1018 	/* check whether ICC provider is registered */
1019 	if (mc->provider.dev == dev)
1020 		icc_sync_state(dev);
1021 }
1022 
1023 static const struct dev_pm_ops tegra_mc_pm_ops = {
1024 	SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume)
1025 };
1026 
1027 static struct platform_driver tegra_mc_driver = {
1028 	.driver = {
1029 		.name = "tegra-mc",
1030 		.of_match_table = tegra_mc_of_match,
1031 		.pm = &tegra_mc_pm_ops,
1032 		.suppress_bind_attrs = true,
1033 		.sync_state = tegra_mc_sync_state,
1034 	},
1035 	.prevent_deferred_probe = true,
1036 	.probe = tegra_mc_probe,
1037 };
1038 
tegra_mc_init(void)1039 static int tegra_mc_init(void)
1040 {
1041 	return platform_driver_register(&tegra_mc_driver);
1042 }
1043 arch_initcall(tegra_mc_init);
1044 
1045 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
1046 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
1047