1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
289184651SThierry Reding /*
389184651SThierry Reding * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved.
489184651SThierry Reding */
589184651SThierry Reding
689184651SThierry Reding #include <linux/clk.h>
720e92462SDmitry Osipenko #include <linux/delay.h>
8c4c21f22SThierry Reding #include <linux/dma-mapping.h>
90c56eda8SDmitry Osipenko #include <linux/export.h>
1089184651SThierry Reding #include <linux/interrupt.h>
1189184651SThierry Reding #include <linux/kernel.h>
1289184651SThierry Reding #include <linux/module.h>
1389184651SThierry Reding #include <linux/of.h>
14*0b483871SRob Herring #include <linux/of_platform.h>
1589184651SThierry Reding #include <linux/platform_device.h>
1689184651SThierry Reding #include <linux/slab.h>
173d9dd6fdSMikko Perttunen #include <linux/sort.h>
189a38cb27SSumit Gupta #include <linux/tegra-icc.h>
193d9dd6fdSMikko Perttunen
203d9dd6fdSMikko Perttunen #include <soc/tegra/fuse.h>
2189184651SThierry Reding
2289184651SThierry Reding #include "mc.h"
2389184651SThierry Reding
2489184651SThierry Reding static const struct of_device_id tegra_mc_of_match[] = {
25a8d502fdSDmitry Osipenko #ifdef CONFIG_ARCH_TEGRA_2x_SOC
2696efa118SDmitry Osipenko { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
27a8d502fdSDmitry Osipenko #endif
2889184651SThierry Reding #ifdef CONFIG_ARCH_TEGRA_3x_SOC
2989184651SThierry Reding { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
3089184651SThierry Reding #endif
3189184651SThierry Reding #ifdef CONFIG_ARCH_TEGRA_114_SOC
3289184651SThierry Reding { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
3389184651SThierry Reding #endif
3489184651SThierry Reding #ifdef CONFIG_ARCH_TEGRA_124_SOC
3589184651SThierry Reding { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
3689184651SThierry Reding #endif
37242b1d71SThierry Reding #ifdef CONFIG_ARCH_TEGRA_132_SOC
38242b1d71SThierry Reding { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
39242b1d71SThierry Reding #endif
40588c43a7SThierry Reding #ifdef CONFIG_ARCH_TEGRA_210_SOC
41588c43a7SThierry Reding { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
42588c43a7SThierry Reding #endif
437355c7b9SThierry Reding #ifdef CONFIG_ARCH_TEGRA_186_SOC
447355c7b9SThierry Reding { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
457355c7b9SThierry Reding #endif
467355c7b9SThierry Reding #ifdef CONFIG_ARCH_TEGRA_194_SOC
477355c7b9SThierry Reding { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
487355c7b9SThierry Reding #endif
4972c81bb6SThierry Reding #ifdef CONFIG_ARCH_TEGRA_234_SOC
5072c81bb6SThierry Reding { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
5172c81bb6SThierry Reding #endif
527355c7b9SThierry Reding { /* sentinel */ }
5389184651SThierry Reding };
5489184651SThierry Reding MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
5589184651SThierry Reding
tegra_mc_devm_action_put_device(void * data)566c6bd207SDmitry Osipenko static void tegra_mc_devm_action_put_device(void *data)
576c6bd207SDmitry Osipenko {
586c6bd207SDmitry Osipenko struct tegra_mc *mc = data;
596c6bd207SDmitry Osipenko
606c6bd207SDmitry Osipenko put_device(mc->dev);
616c6bd207SDmitry Osipenko }
626c6bd207SDmitry Osipenko
636c6bd207SDmitry Osipenko /**
646c6bd207SDmitry Osipenko * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
656c6bd207SDmitry Osipenko * @dev: device pointer for the consumer device
666c6bd207SDmitry Osipenko *
676c6bd207SDmitry Osipenko * This function will search for the Memory Controller node in a device-tree
686c6bd207SDmitry Osipenko * and retrieve the Memory Controller handle.
696c6bd207SDmitry Osipenko *
706c6bd207SDmitry Osipenko * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
716c6bd207SDmitry Osipenko */
devm_tegra_memory_controller_get(struct device * dev)726c6bd207SDmitry Osipenko struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
736c6bd207SDmitry Osipenko {
746c6bd207SDmitry Osipenko struct platform_device *pdev;
756c6bd207SDmitry Osipenko struct device_node *np;
766c6bd207SDmitry Osipenko struct tegra_mc *mc;
776c6bd207SDmitry Osipenko int err;
786c6bd207SDmitry Osipenko
796c6bd207SDmitry Osipenko np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
806c6bd207SDmitry Osipenko if (!np)
816c6bd207SDmitry Osipenko return ERR_PTR(-ENOENT);
826c6bd207SDmitry Osipenko
836c6bd207SDmitry Osipenko pdev = of_find_device_by_node(np);
846c6bd207SDmitry Osipenko of_node_put(np);
856c6bd207SDmitry Osipenko if (!pdev)
866c6bd207SDmitry Osipenko return ERR_PTR(-ENODEV);
876c6bd207SDmitry Osipenko
886c6bd207SDmitry Osipenko mc = platform_get_drvdata(pdev);
896c6bd207SDmitry Osipenko if (!mc) {
906c6bd207SDmitry Osipenko put_device(&pdev->dev);
916c6bd207SDmitry Osipenko return ERR_PTR(-EPROBE_DEFER);
926c6bd207SDmitry Osipenko }
936c6bd207SDmitry Osipenko
941d8e0223SCai Huoqing err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
951d8e0223SCai Huoqing if (err)
966c6bd207SDmitry Osipenko return ERR_PTR(err);
976c6bd207SDmitry Osipenko
986c6bd207SDmitry Osipenko return mc;
996c6bd207SDmitry Osipenko }
1006c6bd207SDmitry Osipenko EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
1016c6bd207SDmitry Osipenko
tegra_mc_probe_device(struct tegra_mc * mc,struct device * dev)102393d66fdSThierry Reding int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
103393d66fdSThierry Reding {
104393d66fdSThierry Reding if (mc->soc->ops && mc->soc->ops->probe_device)
105393d66fdSThierry Reding return mc->soc->ops->probe_device(mc, dev);
106393d66fdSThierry Reding
107393d66fdSThierry Reding return 0;
108393d66fdSThierry Reding }
109393d66fdSThierry Reding EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
110393d66fdSThierry Reding
tegra_mc_get_carveout_info(struct tegra_mc * mc,unsigned int id,phys_addr_t * base,u64 * size)1117946920dSMikko Perttunen int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id,
1127946920dSMikko Perttunen phys_addr_t *base, u64 *size)
1137946920dSMikko Perttunen {
1147946920dSMikko Perttunen u32 offset;
1157946920dSMikko Perttunen
1167946920dSMikko Perttunen if (id < 1 || id >= mc->soc->num_carveouts)
1177946920dSMikko Perttunen return -EINVAL;
1187946920dSMikko Perttunen
1197946920dSMikko Perttunen if (id < 6)
1207946920dSMikko Perttunen offset = 0xc0c + 0x50 * (id - 1);
1217946920dSMikko Perttunen else
1227946920dSMikko Perttunen offset = 0x2004 + 0x50 * (id - 6);
1237946920dSMikko Perttunen
1247946920dSMikko Perttunen *base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0);
1257946920dSMikko Perttunen #ifdef CONFIG_PHYS_ADDR_T_64BIT
1267946920dSMikko Perttunen *base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32;
1277946920dSMikko Perttunen #endif
1287946920dSMikko Perttunen
1297946920dSMikko Perttunen if (size)
1307946920dSMikko Perttunen *size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17;
1317946920dSMikko Perttunen
1327946920dSMikko Perttunen return 0;
1337946920dSMikko Perttunen }
1347946920dSMikko Perttunen EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info);
1357946920dSMikko Perttunen
tegra_mc_block_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)136cb2b5839SThierry Reding static int tegra_mc_block_dma_common(struct tegra_mc *mc,
13720e92462SDmitry Osipenko const struct tegra_mc_reset *rst)
13820e92462SDmitry Osipenko {
13920e92462SDmitry Osipenko unsigned long flags;
14020e92462SDmitry Osipenko u32 value;
14120e92462SDmitry Osipenko
14220e92462SDmitry Osipenko spin_lock_irqsave(&mc->lock, flags);
14320e92462SDmitry Osipenko
14420e92462SDmitry Osipenko value = mc_readl(mc, rst->control) | BIT(rst->bit);
14520e92462SDmitry Osipenko mc_writel(mc, value, rst->control);
14620e92462SDmitry Osipenko
14720e92462SDmitry Osipenko spin_unlock_irqrestore(&mc->lock, flags);
14820e92462SDmitry Osipenko
14920e92462SDmitry Osipenko return 0;
15020e92462SDmitry Osipenko }
15120e92462SDmitry Osipenko
tegra_mc_dma_idling_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)152cb2b5839SThierry Reding static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
15320e92462SDmitry Osipenko const struct tegra_mc_reset *rst)
15420e92462SDmitry Osipenko {
15520e92462SDmitry Osipenko return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
15620e92462SDmitry Osipenko }
15720e92462SDmitry Osipenko
tegra_mc_unblock_dma_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)158cb2b5839SThierry Reding static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
15920e92462SDmitry Osipenko const struct tegra_mc_reset *rst)
16020e92462SDmitry Osipenko {
16120e92462SDmitry Osipenko unsigned long flags;
16220e92462SDmitry Osipenko u32 value;
16320e92462SDmitry Osipenko
16420e92462SDmitry Osipenko spin_lock_irqsave(&mc->lock, flags);
16520e92462SDmitry Osipenko
16620e92462SDmitry Osipenko value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
16720e92462SDmitry Osipenko mc_writel(mc, value, rst->control);
16820e92462SDmitry Osipenko
16920e92462SDmitry Osipenko spin_unlock_irqrestore(&mc->lock, flags);
17020e92462SDmitry Osipenko
17120e92462SDmitry Osipenko return 0;
17220e92462SDmitry Osipenko }
17320e92462SDmitry Osipenko
tegra_mc_reset_status_common(struct tegra_mc * mc,const struct tegra_mc_reset * rst)174cb2b5839SThierry Reding static int tegra_mc_reset_status_common(struct tegra_mc *mc,
17520e92462SDmitry Osipenko const struct tegra_mc_reset *rst)
17620e92462SDmitry Osipenko {
17720e92462SDmitry Osipenko return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
17820e92462SDmitry Osipenko }
17920e92462SDmitry Osipenko
180cb2b5839SThierry Reding const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
181cb2b5839SThierry Reding .block_dma = tegra_mc_block_dma_common,
182cb2b5839SThierry Reding .dma_idling = tegra_mc_dma_idling_common,
183cb2b5839SThierry Reding .unblock_dma = tegra_mc_unblock_dma_common,
184cb2b5839SThierry Reding .reset_status = tegra_mc_reset_status_common,
18520e92462SDmitry Osipenko };
18620e92462SDmitry Osipenko
reset_to_mc(struct reset_controller_dev * rcdev)18720e92462SDmitry Osipenko static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
18820e92462SDmitry Osipenko {
18920e92462SDmitry Osipenko return container_of(rcdev, struct tegra_mc, reset);
19020e92462SDmitry Osipenko }
19120e92462SDmitry Osipenko
tegra_mc_reset_find(struct tegra_mc * mc,unsigned long id)19220e92462SDmitry Osipenko static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
19320e92462SDmitry Osipenko unsigned long id)
19420e92462SDmitry Osipenko {
19520e92462SDmitry Osipenko unsigned int i;
19620e92462SDmitry Osipenko
19720e92462SDmitry Osipenko for (i = 0; i < mc->soc->num_resets; i++)
19820e92462SDmitry Osipenko if (mc->soc->resets[i].id == id)
19920e92462SDmitry Osipenko return &mc->soc->resets[i];
20020e92462SDmitry Osipenko
20120e92462SDmitry Osipenko return NULL;
20220e92462SDmitry Osipenko }
20320e92462SDmitry Osipenko
tegra_mc_hotreset_assert(struct reset_controller_dev * rcdev,unsigned long id)20420e92462SDmitry Osipenko static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
20520e92462SDmitry Osipenko unsigned long id)
20620e92462SDmitry Osipenko {
20720e92462SDmitry Osipenko struct tegra_mc *mc = reset_to_mc(rcdev);
20820e92462SDmitry Osipenko const struct tegra_mc_reset_ops *rst_ops;
20920e92462SDmitry Osipenko const struct tegra_mc_reset *rst;
21020e92462SDmitry Osipenko int retries = 500;
21120e92462SDmitry Osipenko int err;
21220e92462SDmitry Osipenko
21320e92462SDmitry Osipenko rst = tegra_mc_reset_find(mc, id);
21420e92462SDmitry Osipenko if (!rst)
21520e92462SDmitry Osipenko return -ENODEV;
21620e92462SDmitry Osipenko
21720e92462SDmitry Osipenko rst_ops = mc->soc->reset_ops;
21820e92462SDmitry Osipenko if (!rst_ops)
21920e92462SDmitry Osipenko return -ENODEV;
22020e92462SDmitry Osipenko
2216ce84ab6SDmitry Osipenko /* DMA flushing will fail if reset is already asserted */
2226ce84ab6SDmitry Osipenko if (rst_ops->reset_status) {
2236ce84ab6SDmitry Osipenko /* check whether reset is asserted */
2246ce84ab6SDmitry Osipenko if (rst_ops->reset_status(mc, rst))
2256ce84ab6SDmitry Osipenko return 0;
2266ce84ab6SDmitry Osipenko }
2276ce84ab6SDmitry Osipenko
22820e92462SDmitry Osipenko if (rst_ops->block_dma) {
22920e92462SDmitry Osipenko /* block clients DMA requests */
23020e92462SDmitry Osipenko err = rst_ops->block_dma(mc, rst);
23120e92462SDmitry Osipenko if (err) {
232f2dcded1SDmitry Osipenko dev_err(mc->dev, "failed to block %s DMA: %d\n",
23320e92462SDmitry Osipenko rst->name, err);
23420e92462SDmitry Osipenko return err;
23520e92462SDmitry Osipenko }
23620e92462SDmitry Osipenko }
23720e92462SDmitry Osipenko
23820e92462SDmitry Osipenko if (rst_ops->dma_idling) {
23920e92462SDmitry Osipenko /* wait for completion of the outstanding DMA requests */
24020e92462SDmitry Osipenko while (!rst_ops->dma_idling(mc, rst)) {
24120e92462SDmitry Osipenko if (!retries--) {
242f2dcded1SDmitry Osipenko dev_err(mc->dev, "failed to flush %s DMA\n",
24320e92462SDmitry Osipenko rst->name);
24420e92462SDmitry Osipenko return -EBUSY;
24520e92462SDmitry Osipenko }
24620e92462SDmitry Osipenko
24720e92462SDmitry Osipenko usleep_range(10, 100);
24820e92462SDmitry Osipenko }
24920e92462SDmitry Osipenko }
25020e92462SDmitry Osipenko
25120e92462SDmitry Osipenko if (rst_ops->hotreset_assert) {
25220e92462SDmitry Osipenko /* clear clients DMA requests sitting before arbitration */
25320e92462SDmitry Osipenko err = rst_ops->hotreset_assert(mc, rst);
25420e92462SDmitry Osipenko if (err) {
255f2dcded1SDmitry Osipenko dev_err(mc->dev, "failed to hot reset %s: %d\n",
25620e92462SDmitry Osipenko rst->name, err);
25720e92462SDmitry Osipenko return err;
25820e92462SDmitry Osipenko }
25920e92462SDmitry Osipenko }
26020e92462SDmitry Osipenko
26120e92462SDmitry Osipenko return 0;
26220e92462SDmitry Osipenko }
26320e92462SDmitry Osipenko
tegra_mc_hotreset_deassert(struct reset_controller_dev * rcdev,unsigned long id)26420e92462SDmitry Osipenko static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
26520e92462SDmitry Osipenko unsigned long id)
26620e92462SDmitry Osipenko {
26720e92462SDmitry Osipenko struct tegra_mc *mc = reset_to_mc(rcdev);
26820e92462SDmitry Osipenko const struct tegra_mc_reset_ops *rst_ops;
26920e92462SDmitry Osipenko const struct tegra_mc_reset *rst;
27020e92462SDmitry Osipenko int err;
27120e92462SDmitry Osipenko
27220e92462SDmitry Osipenko rst = tegra_mc_reset_find(mc, id);
27320e92462SDmitry Osipenko if (!rst)
27420e92462SDmitry Osipenko return -ENODEV;
27520e92462SDmitry Osipenko
27620e92462SDmitry Osipenko rst_ops = mc->soc->reset_ops;
27720e92462SDmitry Osipenko if (!rst_ops)
27820e92462SDmitry Osipenko return -ENODEV;
27920e92462SDmitry Osipenko
28020e92462SDmitry Osipenko if (rst_ops->hotreset_deassert) {
28120e92462SDmitry Osipenko /* take out client from hot reset */
28220e92462SDmitry Osipenko err = rst_ops->hotreset_deassert(mc, rst);
28320e92462SDmitry Osipenko if (err) {
284f2dcded1SDmitry Osipenko dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
28520e92462SDmitry Osipenko rst->name, err);
28620e92462SDmitry Osipenko return err;
28720e92462SDmitry Osipenko }
28820e92462SDmitry Osipenko }
28920e92462SDmitry Osipenko
29020e92462SDmitry Osipenko if (rst_ops->unblock_dma) {
29120e92462SDmitry Osipenko /* allow new DMA requests to proceed to arbitration */
29220e92462SDmitry Osipenko err = rst_ops->unblock_dma(mc, rst);
29320e92462SDmitry Osipenko if (err) {
294f2dcded1SDmitry Osipenko dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
29520e92462SDmitry Osipenko rst->name, err);
29620e92462SDmitry Osipenko return err;
29720e92462SDmitry Osipenko }
29820e92462SDmitry Osipenko }
29920e92462SDmitry Osipenko
30020e92462SDmitry Osipenko return 0;
30120e92462SDmitry Osipenko }
30220e92462SDmitry Osipenko
tegra_mc_hotreset_status(struct reset_controller_dev * rcdev,unsigned long id)30320e92462SDmitry Osipenko static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
30420e92462SDmitry Osipenko unsigned long id)
30520e92462SDmitry Osipenko {
30620e92462SDmitry Osipenko struct tegra_mc *mc = reset_to_mc(rcdev);
30720e92462SDmitry Osipenko const struct tegra_mc_reset_ops *rst_ops;
30820e92462SDmitry Osipenko const struct tegra_mc_reset *rst;
30920e92462SDmitry Osipenko
31020e92462SDmitry Osipenko rst = tegra_mc_reset_find(mc, id);
31120e92462SDmitry Osipenko if (!rst)
31220e92462SDmitry Osipenko return -ENODEV;
31320e92462SDmitry Osipenko
31420e92462SDmitry Osipenko rst_ops = mc->soc->reset_ops;
31520e92462SDmitry Osipenko if (!rst_ops)
31620e92462SDmitry Osipenko return -ENODEV;
31720e92462SDmitry Osipenko
31820e92462SDmitry Osipenko return rst_ops->reset_status(mc, rst);
31920e92462SDmitry Osipenko }
32020e92462SDmitry Osipenko
32120e92462SDmitry Osipenko static const struct reset_control_ops tegra_mc_reset_ops = {
32220e92462SDmitry Osipenko .assert = tegra_mc_hotreset_assert,
32320e92462SDmitry Osipenko .deassert = tegra_mc_hotreset_deassert,
32420e92462SDmitry Osipenko .status = tegra_mc_hotreset_status,
32520e92462SDmitry Osipenko };
32620e92462SDmitry Osipenko
tegra_mc_reset_setup(struct tegra_mc * mc)32720e92462SDmitry Osipenko static int tegra_mc_reset_setup(struct tegra_mc *mc)
32820e92462SDmitry Osipenko {
32920e92462SDmitry Osipenko int err;
33020e92462SDmitry Osipenko
33120e92462SDmitry Osipenko mc->reset.ops = &tegra_mc_reset_ops;
33220e92462SDmitry Osipenko mc->reset.owner = THIS_MODULE;
33320e92462SDmitry Osipenko mc->reset.of_node = mc->dev->of_node;
33420e92462SDmitry Osipenko mc->reset.of_reset_n_cells = 1;
33520e92462SDmitry Osipenko mc->reset.nr_resets = mc->soc->num_resets;
33620e92462SDmitry Osipenko
33720e92462SDmitry Osipenko err = reset_controller_register(&mc->reset);
33820e92462SDmitry Osipenko if (err < 0)
33920e92462SDmitry Osipenko return err;
34020e92462SDmitry Osipenko
34120e92462SDmitry Osipenko return 0;
34220e92462SDmitry Osipenko }
34320e92462SDmitry Osipenko
tegra_mc_write_emem_configuration(struct tegra_mc * mc,unsigned long rate)344e34212c7SDmitry Osipenko int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
3453d9dd6fdSMikko Perttunen {
3463d9dd6fdSMikko Perttunen unsigned int i;
3473d9dd6fdSMikko Perttunen struct tegra_mc_timing *timing = NULL;
3483d9dd6fdSMikko Perttunen
3493d9dd6fdSMikko Perttunen for (i = 0; i < mc->num_timings; i++) {
3503d9dd6fdSMikko Perttunen if (mc->timings[i].rate == rate) {
3513d9dd6fdSMikko Perttunen timing = &mc->timings[i];
3523d9dd6fdSMikko Perttunen break;
3533d9dd6fdSMikko Perttunen }
3543d9dd6fdSMikko Perttunen }
3553d9dd6fdSMikko Perttunen
3563d9dd6fdSMikko Perttunen if (!timing) {
3573d9dd6fdSMikko Perttunen dev_err(mc->dev, "no memory timing registered for rate %lu\n",
3583d9dd6fdSMikko Perttunen rate);
359e34212c7SDmitry Osipenko return -EINVAL;
3603d9dd6fdSMikko Perttunen }
3613d9dd6fdSMikko Perttunen
3623d9dd6fdSMikko Perttunen for (i = 0; i < mc->soc->num_emem_regs; ++i)
3633d9dd6fdSMikko Perttunen mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
364e34212c7SDmitry Osipenko
365e34212c7SDmitry Osipenko return 0;
3663d9dd6fdSMikko Perttunen }
3670c56eda8SDmitry Osipenko EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
3683d9dd6fdSMikko Perttunen
tegra_mc_get_emem_device_count(struct tegra_mc * mc)3693d9dd6fdSMikko Perttunen unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
3703d9dd6fdSMikko Perttunen {
3713d9dd6fdSMikko Perttunen u8 dram_count;
3723d9dd6fdSMikko Perttunen
3733d9dd6fdSMikko Perttunen dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
3743d9dd6fdSMikko Perttunen dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
3753d9dd6fdSMikko Perttunen dram_count++;
3763d9dd6fdSMikko Perttunen
3773d9dd6fdSMikko Perttunen return dram_count;
3783d9dd6fdSMikko Perttunen }
3790c56eda8SDmitry Osipenko EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
3803d9dd6fdSMikko Perttunen
381ddeceab0SThierry Reding #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
382ddeceab0SThierry Reding defined(CONFIG_ARCH_TEGRA_114_SOC) || \
383ddeceab0SThierry Reding defined(CONFIG_ARCH_TEGRA_124_SOC) || \
384ddeceab0SThierry Reding defined(CONFIG_ARCH_TEGRA_132_SOC) || \
385ddeceab0SThierry Reding defined(CONFIG_ARCH_TEGRA_210_SOC)
tegra_mc_setup_latency_allowance(struct tegra_mc * mc)386ddeceab0SThierry Reding static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
387ddeceab0SThierry Reding {
388ddeceab0SThierry Reding unsigned long long tick;
389ddeceab0SThierry Reding unsigned int i;
390ddeceab0SThierry Reding u32 value;
391ddeceab0SThierry Reding
392ddeceab0SThierry Reding /* compute the number of MC clock cycles per tick */
393ddeceab0SThierry Reding tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
394ddeceab0SThierry Reding do_div(tick, NSEC_PER_SEC);
395ddeceab0SThierry Reding
396ddeceab0SThierry Reding value = mc_readl(mc, MC_EMEM_ARB_CFG);
397ddeceab0SThierry Reding value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
398ddeceab0SThierry Reding value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
399ddeceab0SThierry Reding mc_writel(mc, value, MC_EMEM_ARB_CFG);
400ddeceab0SThierry Reding
401ddeceab0SThierry Reding /* write latency allowance defaults */
402ddeceab0SThierry Reding for (i = 0; i < mc->soc->num_clients; i++) {
403ddeceab0SThierry Reding const struct tegra_mc_client *client = &mc->soc->clients[i];
404ddeceab0SThierry Reding u32 value;
405ddeceab0SThierry Reding
406ddeceab0SThierry Reding value = mc_readl(mc, client->regs.la.reg);
407ddeceab0SThierry Reding value &= ~(client->regs.la.mask << client->regs.la.shift);
408ddeceab0SThierry Reding value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
409ddeceab0SThierry Reding mc_writel(mc, value, client->regs.la.reg);
410ddeceab0SThierry Reding }
411ddeceab0SThierry Reding
412ddeceab0SThierry Reding /* latch new values */
413ddeceab0SThierry Reding mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
414ddeceab0SThierry Reding
415ddeceab0SThierry Reding return 0;
416ddeceab0SThierry Reding }
417ddeceab0SThierry Reding
load_one_timing(struct tegra_mc * mc,struct tegra_mc_timing * timing,struct device_node * node)4183d9dd6fdSMikko Perttunen static int load_one_timing(struct tegra_mc *mc,
4193d9dd6fdSMikko Perttunen struct tegra_mc_timing *timing,
4203d9dd6fdSMikko Perttunen struct device_node *node)
4213d9dd6fdSMikko Perttunen {
4223d9dd6fdSMikko Perttunen int err;
4233d9dd6fdSMikko Perttunen u32 tmp;
4243d9dd6fdSMikko Perttunen
4253d9dd6fdSMikko Perttunen err = of_property_read_u32(node, "clock-frequency", &tmp);
4263d9dd6fdSMikko Perttunen if (err) {
4273d9dd6fdSMikko Perttunen dev_err(mc->dev,
428c86f9854SRob Herring "timing %pOFn: failed to read rate\n", node);
4293d9dd6fdSMikko Perttunen return err;
4303d9dd6fdSMikko Perttunen }
4313d9dd6fdSMikko Perttunen
4323d9dd6fdSMikko Perttunen timing->rate = tmp;
4333d9dd6fdSMikko Perttunen timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
4343d9dd6fdSMikko Perttunen sizeof(u32), GFP_KERNEL);
4353d9dd6fdSMikko Perttunen if (!timing->emem_data)
4363d9dd6fdSMikko Perttunen return -ENOMEM;
4373d9dd6fdSMikko Perttunen
4383d9dd6fdSMikko Perttunen err = of_property_read_u32_array(node, "nvidia,emem-configuration",
4393d9dd6fdSMikko Perttunen timing->emem_data,
4403d9dd6fdSMikko Perttunen mc->soc->num_emem_regs);
4413d9dd6fdSMikko Perttunen if (err) {
4423d9dd6fdSMikko Perttunen dev_err(mc->dev,
443c86f9854SRob Herring "timing %pOFn: failed to read EMEM configuration\n",
444c86f9854SRob Herring node);
4453d9dd6fdSMikko Perttunen return err;
4463d9dd6fdSMikko Perttunen }
4473d9dd6fdSMikko Perttunen
4483d9dd6fdSMikko Perttunen return 0;
4493d9dd6fdSMikko Perttunen }
4503d9dd6fdSMikko Perttunen
load_timings(struct tegra_mc * mc,struct device_node * node)4513d9dd6fdSMikko Perttunen static int load_timings(struct tegra_mc *mc, struct device_node *node)
4523d9dd6fdSMikko Perttunen {
4533d9dd6fdSMikko Perttunen struct device_node *child;
4543d9dd6fdSMikko Perttunen struct tegra_mc_timing *timing;
4553d9dd6fdSMikko Perttunen int child_count = of_get_child_count(node);
4563d9dd6fdSMikko Perttunen int i = 0, err;
4573d9dd6fdSMikko Perttunen
4583d9dd6fdSMikko Perttunen mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
4593d9dd6fdSMikko Perttunen GFP_KERNEL);
4603d9dd6fdSMikko Perttunen if (!mc->timings)
4613d9dd6fdSMikko Perttunen return -ENOMEM;
4623d9dd6fdSMikko Perttunen
4633d9dd6fdSMikko Perttunen mc->num_timings = child_count;
4643d9dd6fdSMikko Perttunen
4653d9dd6fdSMikko Perttunen for_each_child_of_node(node, child) {
4663d9dd6fdSMikko Perttunen timing = &mc->timings[i++];
4673d9dd6fdSMikko Perttunen
4683d9dd6fdSMikko Perttunen err = load_one_timing(mc, timing, child);
46955bb1d83SAmitoj Kaur Chawla if (err) {
47055bb1d83SAmitoj Kaur Chawla of_node_put(child);
4713d9dd6fdSMikko Perttunen return err;
4723d9dd6fdSMikko Perttunen }
47355bb1d83SAmitoj Kaur Chawla }
4743d9dd6fdSMikko Perttunen
4753d9dd6fdSMikko Perttunen return 0;
4763d9dd6fdSMikko Perttunen }
4773d9dd6fdSMikko Perttunen
tegra_mc_setup_timings(struct tegra_mc * mc)4783d9dd6fdSMikko Perttunen static int tegra_mc_setup_timings(struct tegra_mc *mc)
4793d9dd6fdSMikko Perttunen {
4803d9dd6fdSMikko Perttunen struct device_node *node;
4813d9dd6fdSMikko Perttunen u32 ram_code, node_ram_code;
4823d9dd6fdSMikko Perttunen int err;
4833d9dd6fdSMikko Perttunen
4843d9dd6fdSMikko Perttunen ram_code = tegra_read_ram_code();
4853d9dd6fdSMikko Perttunen
4863d9dd6fdSMikko Perttunen mc->num_timings = 0;
4873d9dd6fdSMikko Perttunen
4883d9dd6fdSMikko Perttunen for_each_child_of_node(mc->dev->of_node, node) {
4893d9dd6fdSMikko Perttunen err = of_property_read_u32(node, "nvidia,ram-code",
4903d9dd6fdSMikko Perttunen &node_ram_code);
491d1122e4bSJulia Lawall if (err || (node_ram_code != ram_code))
4923d9dd6fdSMikko Perttunen continue;
4933d9dd6fdSMikko Perttunen
4943d9dd6fdSMikko Perttunen err = load_timings(mc, node);
49555bb1d83SAmitoj Kaur Chawla of_node_put(node);
4963d9dd6fdSMikko Perttunen if (err)
4973d9dd6fdSMikko Perttunen return err;
4983d9dd6fdSMikko Perttunen break;
4993d9dd6fdSMikko Perttunen }
5003d9dd6fdSMikko Perttunen
5013d9dd6fdSMikko Perttunen if (mc->num_timings == 0)
5023d9dd6fdSMikko Perttunen dev_warn(mc->dev,
5033d9dd6fdSMikko Perttunen "no memory timings for RAM code %u registered\n",
5043d9dd6fdSMikko Perttunen ram_code);
5053d9dd6fdSMikko Perttunen
5063d9dd6fdSMikko Perttunen return 0;
5073d9dd6fdSMikko Perttunen }
5083d9dd6fdSMikko Perttunen
tegra30_mc_probe(struct tegra_mc * mc)509ddeceab0SThierry Reding int tegra30_mc_probe(struct tegra_mc *mc)
510ddeceab0SThierry Reding {
511ddeceab0SThierry Reding int err;
512ddeceab0SThierry Reding
513ddeceab0SThierry Reding mc->clk = devm_clk_get_optional(mc->dev, "mc");
514ddeceab0SThierry Reding if (IS_ERR(mc->clk)) {
515ddeceab0SThierry Reding dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
516ddeceab0SThierry Reding return PTR_ERR(mc->clk);
517ddeceab0SThierry Reding }
518ddeceab0SThierry Reding
519ddeceab0SThierry Reding /* ensure that debug features are disabled */
520ddeceab0SThierry Reding mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
521ddeceab0SThierry Reding
522ddeceab0SThierry Reding err = tegra_mc_setup_latency_allowance(mc);
523ddeceab0SThierry Reding if (err < 0) {
524ddeceab0SThierry Reding dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
525ddeceab0SThierry Reding return err;
526ddeceab0SThierry Reding }
527ddeceab0SThierry Reding
528ddeceab0SThierry Reding err = tegra_mc_setup_timings(mc);
529ddeceab0SThierry Reding if (err < 0) {
530ddeceab0SThierry Reding dev_err(mc->dev, "failed to setup timings: %d\n", err);
531ddeceab0SThierry Reding return err;
532ddeceab0SThierry Reding }
533ddeceab0SThierry Reding
534ddeceab0SThierry Reding return 0;
535ddeceab0SThierry Reding }
536ddeceab0SThierry Reding
53754a85e09SAshish Mhetre const struct tegra_mc_ops tegra30_mc_ops = {
53854a85e09SAshish Mhetre .probe = tegra30_mc_probe,
53954a85e09SAshish Mhetre .handle_irq = tegra30_mc_handle_irq,
54054a85e09SAshish Mhetre };
54154a85e09SAshish Mhetre #endif
54254a85e09SAshish Mhetre
mc_global_intstatus_to_channel(const struct tegra_mc * mc,u32 status,unsigned int * mc_channel)54354a85e09SAshish Mhetre static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
54454a85e09SAshish Mhetre unsigned int *mc_channel)
54554a85e09SAshish Mhetre {
54654a85e09SAshish Mhetre if ((status & mc->soc->ch_intmask) == 0)
54754a85e09SAshish Mhetre return -EINVAL;
54854a85e09SAshish Mhetre
54954a85e09SAshish Mhetre *mc_channel = __ffs((status & mc->soc->ch_intmask) >>
55054a85e09SAshish Mhetre mc->soc->global_intstatus_channel_shift);
55154a85e09SAshish Mhetre
55254a85e09SAshish Mhetre return 0;
55354a85e09SAshish Mhetre }
55454a85e09SAshish Mhetre
mc_channel_to_global_intstatus(const struct tegra_mc * mc,unsigned int channel)55554a85e09SAshish Mhetre static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
55654a85e09SAshish Mhetre unsigned int channel)
55754a85e09SAshish Mhetre {
55854a85e09SAshish Mhetre return BIT(channel) << mc->soc->global_intstatus_channel_shift;
55954a85e09SAshish Mhetre }
56054a85e09SAshish Mhetre
tegra30_mc_handle_irq(int irq,void * data)56154a85e09SAshish Mhetre irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
56289184651SThierry Reding {
56389184651SThierry Reding struct tegra_mc *mc = data;
56454a85e09SAshish Mhetre unsigned int bit, channel;
5651c74d5c0SDmitry Osipenko unsigned long status;
56654a85e09SAshish Mhetre
56754a85e09SAshish Mhetre if (mc->soc->num_channels) {
56854a85e09SAshish Mhetre u32 global_status;
56954a85e09SAshish Mhetre int err;
57054a85e09SAshish Mhetre
57154a85e09SAshish Mhetre global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
57254a85e09SAshish Mhetre err = mc_global_intstatus_to_channel(mc, global_status, &channel);
57354a85e09SAshish Mhetre if (err < 0) {
57454a85e09SAshish Mhetre dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
57554a85e09SAshish Mhetre global_status);
57654a85e09SAshish Mhetre return IRQ_NONE;
57754a85e09SAshish Mhetre }
57889184651SThierry Reding
57989184651SThierry Reding /* mask all interrupts to avoid flooding */
58054a85e09SAshish Mhetre status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
58154a85e09SAshish Mhetre } else {
5821c74d5c0SDmitry Osipenko status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
58354a85e09SAshish Mhetre }
58454a85e09SAshish Mhetre
585bf3fbdfbSDmitry Osipenko if (!status)
586bf3fbdfbSDmitry Osipenko return IRQ_NONE;
58789184651SThierry Reding
58889184651SThierry Reding for_each_set_bit(bit, &status, 32) {
5891079a66bSThierry Reding const char *error = tegra_mc_status_names[bit] ?: "unknown";
59089184651SThierry Reding const char *client = "unknown", *desc;
59189184651SThierry Reding const char *direction, *secure;
59254a85e09SAshish Mhetre u32 status_reg, addr_reg;
59354a85e09SAshish Mhetre u32 intmask = BIT(bit);
59489184651SThierry Reding phys_addr_t addr = 0;
59554a85e09SAshish Mhetre #ifdef CONFIG_PHYS_ADDR_T_64BIT
59654a85e09SAshish Mhetre u32 addr_hi_reg = 0;
59754a85e09SAshish Mhetre #endif
59889184651SThierry Reding unsigned int i;
59989184651SThierry Reding char perm[7];
60089184651SThierry Reding u8 id, type;
60189184651SThierry Reding u32 value;
60289184651SThierry Reding
60354a85e09SAshish Mhetre switch (intmask) {
60454a85e09SAshish Mhetre case MC_INT_DECERR_VPR:
60554a85e09SAshish Mhetre status_reg = MC_ERR_VPR_STATUS;
60654a85e09SAshish Mhetre addr_reg = MC_ERR_VPR_ADR;
60754a85e09SAshish Mhetre break;
60854a85e09SAshish Mhetre
60954a85e09SAshish Mhetre case MC_INT_SECERR_SEC:
61054a85e09SAshish Mhetre status_reg = MC_ERR_SEC_STATUS;
61154a85e09SAshish Mhetre addr_reg = MC_ERR_SEC_ADR;
61254a85e09SAshish Mhetre break;
61354a85e09SAshish Mhetre
61454a85e09SAshish Mhetre case MC_INT_DECERR_MTS:
61554a85e09SAshish Mhetre status_reg = MC_ERR_MTS_STATUS;
61654a85e09SAshish Mhetre addr_reg = MC_ERR_MTS_ADR;
61754a85e09SAshish Mhetre break;
61854a85e09SAshish Mhetre
61954a85e09SAshish Mhetre case MC_INT_DECERR_GENERALIZED_CARVEOUT:
62054a85e09SAshish Mhetre status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
62154a85e09SAshish Mhetre addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
62254a85e09SAshish Mhetre break;
62354a85e09SAshish Mhetre
62454a85e09SAshish Mhetre case MC_INT_DECERR_ROUTE_SANITY:
62554a85e09SAshish Mhetre status_reg = MC_ERR_ROUTE_SANITY_STATUS;
62654a85e09SAshish Mhetre addr_reg = MC_ERR_ROUTE_SANITY_ADR;
62754a85e09SAshish Mhetre break;
62854a85e09SAshish Mhetre
62954a85e09SAshish Mhetre default:
63054a85e09SAshish Mhetre status_reg = MC_ERR_STATUS;
63154a85e09SAshish Mhetre addr_reg = MC_ERR_ADR;
63254a85e09SAshish Mhetre
63354a85e09SAshish Mhetre #ifdef CONFIG_PHYS_ADDR_T_64BIT
63454a85e09SAshish Mhetre if (mc->soc->has_addr_hi_reg)
63554a85e09SAshish Mhetre addr_hi_reg = MC_ERR_ADR_HI;
63654a85e09SAshish Mhetre #endif
63754a85e09SAshish Mhetre break;
63854a85e09SAshish Mhetre }
63954a85e09SAshish Mhetre
64054a85e09SAshish Mhetre if (mc->soc->num_channels)
64154a85e09SAshish Mhetre value = mc_ch_readl(mc, channel, status_reg);
64254a85e09SAshish Mhetre else
64354a85e09SAshish Mhetre value = mc_readl(mc, status_reg);
64489184651SThierry Reding
64589184651SThierry Reding #ifdef CONFIG_PHYS_ADDR_T_64BIT
64689184651SThierry Reding if (mc->soc->num_address_bits > 32) {
64754a85e09SAshish Mhetre if (addr_hi_reg) {
64854a85e09SAshish Mhetre if (mc->soc->num_channels)
64954a85e09SAshish Mhetre addr = mc_ch_readl(mc, channel, addr_hi_reg);
65054a85e09SAshish Mhetre else
65154a85e09SAshish Mhetre addr = mc_readl(mc, addr_hi_reg);
65254a85e09SAshish Mhetre } else {
65389184651SThierry Reding addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
65489184651SThierry Reding MC_ERR_STATUS_ADR_HI_MASK);
65554a85e09SAshish Mhetre }
65689184651SThierry Reding addr <<= 32;
65789184651SThierry Reding }
65889184651SThierry Reding #endif
65989184651SThierry Reding
66089184651SThierry Reding if (value & MC_ERR_STATUS_RW)
66189184651SThierry Reding direction = "write";
66289184651SThierry Reding else
66389184651SThierry Reding direction = "read";
66489184651SThierry Reding
66589184651SThierry Reding if (value & MC_ERR_STATUS_SECURITY)
66689184651SThierry Reding secure = "secure ";
66789184651SThierry Reding else
66889184651SThierry Reding secure = "";
66989184651SThierry Reding
6703c01cf3bSPaul Walmsley id = value & mc->soc->client_id_mask;
67189184651SThierry Reding
67289184651SThierry Reding for (i = 0; i < mc->soc->num_clients; i++) {
67389184651SThierry Reding if (mc->soc->clients[i].id == id) {
67489184651SThierry Reding client = mc->soc->clients[i].name;
67589184651SThierry Reding break;
67689184651SThierry Reding }
67789184651SThierry Reding }
67889184651SThierry Reding
67989184651SThierry Reding type = (value & MC_ERR_STATUS_TYPE_MASK) >>
68089184651SThierry Reding MC_ERR_STATUS_TYPE_SHIFT;
6811079a66bSThierry Reding desc = tegra_mc_error_names[type];
68289184651SThierry Reding
68389184651SThierry Reding switch (value & MC_ERR_STATUS_TYPE_MASK) {
68489184651SThierry Reding case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
68589184651SThierry Reding perm[0] = ' ';
68689184651SThierry Reding perm[1] = '[';
68789184651SThierry Reding
68889184651SThierry Reding if (value & MC_ERR_STATUS_READABLE)
68989184651SThierry Reding perm[2] = 'R';
69089184651SThierry Reding else
69189184651SThierry Reding perm[2] = '-';
69289184651SThierry Reding
69389184651SThierry Reding if (value & MC_ERR_STATUS_WRITABLE)
69489184651SThierry Reding perm[3] = 'W';
69589184651SThierry Reding else
69689184651SThierry Reding perm[3] = '-';
69789184651SThierry Reding
69889184651SThierry Reding if (value & MC_ERR_STATUS_NONSECURE)
69989184651SThierry Reding perm[4] = '-';
70089184651SThierry Reding else
70189184651SThierry Reding perm[4] = 'S';
70289184651SThierry Reding
70389184651SThierry Reding perm[5] = ']';
70489184651SThierry Reding perm[6] = '\0';
70589184651SThierry Reding break;
70689184651SThierry Reding
70789184651SThierry Reding default:
70889184651SThierry Reding perm[0] = '\0';
70989184651SThierry Reding break;
71089184651SThierry Reding }
71189184651SThierry Reding
71254a85e09SAshish Mhetre if (mc->soc->num_channels)
71354a85e09SAshish Mhetre value = mc_ch_readl(mc, channel, addr_reg);
71454a85e09SAshish Mhetre else
71554a85e09SAshish Mhetre value = mc_readl(mc, addr_reg);
71689184651SThierry Reding addr |= value;
71789184651SThierry Reding
71889184651SThierry Reding dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
71989184651SThierry Reding client, secure, direction, &addr, error,
72089184651SThierry Reding desc, perm);
72189184651SThierry Reding }
72289184651SThierry Reding
72389184651SThierry Reding /* clear interrupts */
72454a85e09SAshish Mhetre if (mc->soc->num_channels) {
72554a85e09SAshish Mhetre mc_ch_writel(mc, channel, status, MC_INTSTATUS);
72654a85e09SAshish Mhetre mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
72754a85e09SAshish Mhetre mc_channel_to_global_intstatus(mc, channel),
72854a85e09SAshish Mhetre MC_GLOBAL_INTSTATUS);
72954a85e09SAshish Mhetre } else {
73089184651SThierry Reding mc_writel(mc, status, MC_INTSTATUS);
73154a85e09SAshish Mhetre }
73289184651SThierry Reding
73389184651SThierry Reding return IRQ_HANDLED;
73489184651SThierry Reding }
73589184651SThierry Reding
7361079a66bSThierry Reding const char *const tegra_mc_status_names[32] = {
7371079a66bSThierry Reding [ 1] = "External interrupt",
7381079a66bSThierry Reding [ 6] = "EMEM address decode error",
7391079a66bSThierry Reding [ 7] = "GART page fault",
7401079a66bSThierry Reding [ 8] = "Security violation",
7411079a66bSThierry Reding [ 9] = "EMEM arbitration error",
7421079a66bSThierry Reding [10] = "Page fault",
7431079a66bSThierry Reding [11] = "Invalid APB ASID update",
7441079a66bSThierry Reding [12] = "VPR violation",
7451079a66bSThierry Reding [13] = "Secure carveout violation",
7461079a66bSThierry Reding [16] = "MTS carveout violation",
74754a85e09SAshish Mhetre [17] = "Generalized carveout violation",
74854a85e09SAshish Mhetre [20] = "Route Sanity error",
7491079a66bSThierry Reding };
750a8d502fdSDmitry Osipenko
7511079a66bSThierry Reding const char *const tegra_mc_error_names[8] = {
7521079a66bSThierry Reding [2] = "EMEM decode error",
7531079a66bSThierry Reding [3] = "TrustZone violation",
7541079a66bSThierry Reding [4] = "Carveout violation",
7551079a66bSThierry Reding [6] = "SMMU translation error",
7561079a66bSThierry Reding };
757a8d502fdSDmitry Osipenko
tegra_mc_icc_xlate(struct of_phandle_args * spec,void * data)758d1478aeaSThierry Reding struct icc_node *tegra_mc_icc_xlate(struct of_phandle_args *spec, void *data)
759d1478aeaSThierry Reding {
760d1478aeaSThierry Reding struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
761d1478aeaSThierry Reding struct icc_node *node;
762d1478aeaSThierry Reding
763d1478aeaSThierry Reding list_for_each_entry(node, &mc->provider.nodes, node_list) {
764d1478aeaSThierry Reding if (node->id == spec->args[0])
765d1478aeaSThierry Reding return node;
766d1478aeaSThierry Reding }
767d1478aeaSThierry Reding
768d1478aeaSThierry Reding /*
769d1478aeaSThierry Reding * If a client driver calls devm_of_icc_get() before the MC driver
770d1478aeaSThierry Reding * is probed, then return EPROBE_DEFER to the client driver.
771d1478aeaSThierry Reding */
772d1478aeaSThierry Reding return ERR_PTR(-EPROBE_DEFER);
773d1478aeaSThierry Reding }
774d1478aeaSThierry Reding
tegra_mc_icc_get(struct icc_node * node,u32 * average,u32 * peak)775d1478aeaSThierry Reding static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak)
776d1478aeaSThierry Reding {
777d1478aeaSThierry Reding *average = 0;
778d1478aeaSThierry Reding *peak = 0;
779d1478aeaSThierry Reding
780d1478aeaSThierry Reding return 0;
781d1478aeaSThierry Reding }
782d1478aeaSThierry Reding
tegra_mc_icc_set(struct icc_node * src,struct icc_node * dst)783d1478aeaSThierry Reding static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst)
784d1478aeaSThierry Reding {
785d1478aeaSThierry Reding return 0;
786d1478aeaSThierry Reding }
787d1478aeaSThierry Reding
788d1478aeaSThierry Reding const struct tegra_mc_icc_ops tegra_mc_icc_ops = {
789d1478aeaSThierry Reding .xlate = tegra_mc_icc_xlate,
790d1478aeaSThierry Reding .aggregate = icc_std_aggregate,
791d1478aeaSThierry Reding .get_bw = tegra_mc_icc_get,
792d1478aeaSThierry Reding .set = tegra_mc_icc_set,
793d1478aeaSThierry Reding };
794d1478aeaSThierry Reding
79506f07981SDmitry Osipenko /*
79606f07981SDmitry Osipenko * Memory Controller (MC) has few Memory Clients that are issuing memory
79706f07981SDmitry Osipenko * bandwidth allocation requests to the MC interconnect provider. The MC
79806f07981SDmitry Osipenko * provider aggregates the requests and then sends the aggregated request
79906f07981SDmitry Osipenko * up to the External Memory Controller (EMC) interconnect provider which
80006f07981SDmitry Osipenko * re-configures hardware interface to External Memory (EMEM) in accordance
80106f07981SDmitry Osipenko * to the required bandwidth. Each MC interconnect node represents an
80206f07981SDmitry Osipenko * individual Memory Client.
80306f07981SDmitry Osipenko *
80406f07981SDmitry Osipenko * Memory interconnect topology:
80506f07981SDmitry Osipenko *
80606f07981SDmitry Osipenko * +----+
80706f07981SDmitry Osipenko * +--------+ | |
80806f07981SDmitry Osipenko * | TEXSRD +--->+ |
80906f07981SDmitry Osipenko * +--------+ | |
81006f07981SDmitry Osipenko * | | +-----+ +------+
81106f07981SDmitry Osipenko * ... | MC +--->+ EMC +--->+ EMEM |
81206f07981SDmitry Osipenko * | | +-----+ +------+
81306f07981SDmitry Osipenko * +--------+ | |
81406f07981SDmitry Osipenko * | DISP.. +--->+ |
81506f07981SDmitry Osipenko * +--------+ | |
81606f07981SDmitry Osipenko * +----+
81706f07981SDmitry Osipenko */
tegra_mc_interconnect_setup(struct tegra_mc * mc)81806f07981SDmitry Osipenko static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
81906f07981SDmitry Osipenko {
82006f07981SDmitry Osipenko struct icc_node *node;
82106f07981SDmitry Osipenko unsigned int i;
82206f07981SDmitry Osipenko int err;
82306f07981SDmitry Osipenko
82406f07981SDmitry Osipenko /* older device-trees don't have interconnect properties */
82506f07981SDmitry Osipenko if (!device_property_present(mc->dev, "#interconnect-cells") ||
82606f07981SDmitry Osipenko !mc->soc->icc_ops)
82706f07981SDmitry Osipenko return 0;
82806f07981SDmitry Osipenko
82906f07981SDmitry Osipenko mc->provider.dev = mc->dev;
83006f07981SDmitry Osipenko mc->provider.data = &mc->provider;
83106f07981SDmitry Osipenko mc->provider.set = mc->soc->icc_ops->set;
83206f07981SDmitry Osipenko mc->provider.aggregate = mc->soc->icc_ops->aggregate;
8339a38cb27SSumit Gupta mc->provider.get_bw = mc->soc->icc_ops->get_bw;
8349a38cb27SSumit Gupta mc->provider.xlate = mc->soc->icc_ops->xlate;
83506f07981SDmitry Osipenko mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
83606f07981SDmitry Osipenko
8375553055cSJohan Hovold icc_provider_init(&mc->provider);
83806f07981SDmitry Osipenko
83906f07981SDmitry Osipenko /* create Memory Controller node */
84006f07981SDmitry Osipenko node = icc_node_create(TEGRA_ICC_MC);
8415553055cSJohan Hovold if (IS_ERR(node))
8425553055cSJohan Hovold return PTR_ERR(node);
84306f07981SDmitry Osipenko
84406f07981SDmitry Osipenko node->name = "Memory Controller";
84506f07981SDmitry Osipenko icc_node_add(node, &mc->provider);
84606f07981SDmitry Osipenko
84706f07981SDmitry Osipenko /* link Memory Controller to External Memory Controller */
84806f07981SDmitry Osipenko err = icc_link_create(node, TEGRA_ICC_EMC);
84906f07981SDmitry Osipenko if (err)
85006f07981SDmitry Osipenko goto remove_nodes;
85106f07981SDmitry Osipenko
85206f07981SDmitry Osipenko for (i = 0; i < mc->soc->num_clients; i++) {
85306f07981SDmitry Osipenko /* create MC client node */
85406f07981SDmitry Osipenko node = icc_node_create(mc->soc->clients[i].id);
85506f07981SDmitry Osipenko if (IS_ERR(node)) {
85606f07981SDmitry Osipenko err = PTR_ERR(node);
85706f07981SDmitry Osipenko goto remove_nodes;
85806f07981SDmitry Osipenko }
85906f07981SDmitry Osipenko
86006f07981SDmitry Osipenko node->name = mc->soc->clients[i].name;
86106f07981SDmitry Osipenko icc_node_add(node, &mc->provider);
86206f07981SDmitry Osipenko
86306f07981SDmitry Osipenko /* link Memory Client to Memory Controller */
86406f07981SDmitry Osipenko err = icc_link_create(node, TEGRA_ICC_MC);
86506f07981SDmitry Osipenko if (err)
86606f07981SDmitry Osipenko goto remove_nodes;
8679a38cb27SSumit Gupta
8689a38cb27SSumit Gupta node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]);
86906f07981SDmitry Osipenko }
87006f07981SDmitry Osipenko
8715553055cSJohan Hovold err = icc_provider_register(&mc->provider);
8725553055cSJohan Hovold if (err)
8735553055cSJohan Hovold goto remove_nodes;
8745553055cSJohan Hovold
87506f07981SDmitry Osipenko return 0;
87606f07981SDmitry Osipenko
87706f07981SDmitry Osipenko remove_nodes:
87806f07981SDmitry Osipenko icc_nodes_remove(&mc->provider);
87906f07981SDmitry Osipenko
88006f07981SDmitry Osipenko return err;
88106f07981SDmitry Osipenko }
88206f07981SDmitry Osipenko
tegra_mc_num_channel_enabled(struct tegra_mc * mc)883e852af72SSumit Gupta static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
884e852af72SSumit Gupta {
885e852af72SSumit Gupta unsigned int i;
886e852af72SSumit Gupta u32 value;
887e852af72SSumit Gupta
888e852af72SSumit Gupta value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE);
889e852af72SSumit Gupta if (value <= 0) {
890e852af72SSumit Gupta mc->num_channels = mc->soc->num_channels;
891e852af72SSumit Gupta return;
892e852af72SSumit Gupta }
893e852af72SSumit Gupta
894e852af72SSumit Gupta for (i = 0; i < 32; i++) {
895e852af72SSumit Gupta if (value & BIT(i))
896e852af72SSumit Gupta mc->num_channels++;
897e852af72SSumit Gupta }
898e852af72SSumit Gupta }
899e852af72SSumit Gupta
tegra_mc_probe(struct platform_device * pdev)90089184651SThierry Reding static int tegra_mc_probe(struct platform_device *pdev)
90189184651SThierry Reding {
90289184651SThierry Reding struct tegra_mc *mc;
903c4c21f22SThierry Reding u64 mask;
90489184651SThierry Reding int err;
90589184651SThierry Reding
90689184651SThierry Reding mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
90789184651SThierry Reding if (!mc)
90889184651SThierry Reding return -ENOMEM;
90989184651SThierry Reding
91089184651SThierry Reding platform_set_drvdata(pdev, mc);
91120e92462SDmitry Osipenko spin_lock_init(&mc->lock);
91259cd046fSDmitry Osipenko mc->soc = of_device_get_match_data(&pdev->dev);
91389184651SThierry Reding mc->dev = &pdev->dev;
91489184651SThierry Reding
915c4c21f22SThierry Reding mask = DMA_BIT_MASK(mc->soc->num_address_bits);
916c4c21f22SThierry Reding
917c4c21f22SThierry Reding err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
918c4c21f22SThierry Reding if (err < 0) {
919c4c21f22SThierry Reding dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
920c4c21f22SThierry Reding return err;
921c4c21f22SThierry Reding }
922c4c21f22SThierry Reding
92389184651SThierry Reding /* length of MC tick in nanoseconds */
92489184651SThierry Reding mc->tick = 30;
92589184651SThierry Reding
926dab022f2SKrzysztof Kozlowski mc->regs = devm_platform_ioremap_resource(pdev, 0);
92789184651SThierry Reding if (IS_ERR(mc->regs))
92889184651SThierry Reding return PTR_ERR(mc->regs);
92989184651SThierry Reding
930c64738e9SThierry Reding mc->debugfs.root = debugfs_create_dir("mc", NULL);
931c64738e9SThierry Reding
932c64738e9SThierry Reding if (mc->soc->ops && mc->soc->ops->probe) {
933c64738e9SThierry Reding err = mc->soc->ops->probe(mc);
934c64738e9SThierry Reding if (err < 0)
935c64738e9SThierry Reding return err;
936c64738e9SThierry Reding }
937c64738e9SThierry Reding
938e852af72SSumit Gupta tegra_mc_num_channel_enabled(mc);
939e852af72SSumit Gupta
940e474b3a1SThierry Reding if (mc->soc->ops && mc->soc->ops->handle_irq) {
94189184651SThierry Reding mc->irq = platform_get_irq(pdev, 0);
942162641a6SDmitry Osipenko if (mc->irq < 0)
94389184651SThierry Reding return mc->irq;
94489184651SThierry Reding
945f2dcded1SDmitry Osipenko WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
9463c01cf3bSPaul Walmsley
94754a85e09SAshish Mhetre if (mc->soc->num_channels)
94854a85e09SAshish Mhetre mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
94954a85e09SAshish Mhetre MC_INTMASK);
95054a85e09SAshish Mhetre else
9511c74d5c0SDmitry Osipenko mc_writel(mc, mc->soc->intmask, MC_INTMASK);
95289184651SThierry Reding
9531079a66bSThierry Reding err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
954db4a9c19SDmitry Osipenko dev_name(&pdev->dev), mc);
955db4a9c19SDmitry Osipenko if (err < 0) {
956db4a9c19SDmitry Osipenko dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
957db4a9c19SDmitry Osipenko err);
958db4a9c19SDmitry Osipenko return err;
959db4a9c19SDmitry Osipenko }
960e474b3a1SThierry Reding }
961db4a9c19SDmitry Osipenko
9620de93c69SThierry Reding if (mc->soc->reset_ops) {
9631662dd64SDmitry Osipenko err = tegra_mc_reset_setup(mc);
9641662dd64SDmitry Osipenko if (err < 0)
9650de93c69SThierry Reding dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
9660de93c69SThierry Reding }
9671662dd64SDmitry Osipenko
96806f07981SDmitry Osipenko err = tegra_mc_interconnect_setup(mc);
96906f07981SDmitry Osipenko if (err < 0)
97006f07981SDmitry Osipenko dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
97106f07981SDmitry Osipenko err);
97206f07981SDmitry Osipenko
973568ece5bSDmitry Osipenko if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
97445a81df0SDmitry Osipenko mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
975568ece5bSDmitry Osipenko if (IS_ERR(mc->smmu)) {
97645a81df0SDmitry Osipenko dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
97745a81df0SDmitry Osipenko PTR_ERR(mc->smmu));
978568ece5bSDmitry Osipenko mc->smmu = NULL;
979568ece5bSDmitry Osipenko }
98045a81df0SDmitry Osipenko }
98145a81df0SDmitry Osipenko
982ce2785a7SDmitry Osipenko if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
983ce2785a7SDmitry Osipenko mc->gart = tegra_gart_probe(&pdev->dev, mc);
984ce2785a7SDmitry Osipenko if (IS_ERR(mc->gart)) {
985ce2785a7SDmitry Osipenko dev_err(&pdev->dev, "failed to probe GART: %ld\n",
986ce2785a7SDmitry Osipenko PTR_ERR(mc->gart));
987ce2785a7SDmitry Osipenko mc->gart = NULL;
988ce2785a7SDmitry Osipenko }
989ce2785a7SDmitry Osipenko }
990ce2785a7SDmitry Osipenko
99189184651SThierry Reding return 0;
99289184651SThierry Reding }
99389184651SThierry Reding
tegra_mc_suspend(struct device * dev)9945c9016f0SThierry Reding static int __maybe_unused tegra_mc_suspend(struct device *dev)
995ce2785a7SDmitry Osipenko {
996ce2785a7SDmitry Osipenko struct tegra_mc *mc = dev_get_drvdata(dev);
997ce2785a7SDmitry Osipenko
9985c9016f0SThierry Reding if (mc->soc->ops && mc->soc->ops->suspend)
9995c9016f0SThierry Reding return mc->soc->ops->suspend(mc);
1000ce2785a7SDmitry Osipenko
1001ce2785a7SDmitry Osipenko return 0;
1002ce2785a7SDmitry Osipenko }
1003ce2785a7SDmitry Osipenko
tegra_mc_resume(struct device * dev)10045c9016f0SThierry Reding static int __maybe_unused tegra_mc_resume(struct device *dev)
1005ce2785a7SDmitry Osipenko {
1006ce2785a7SDmitry Osipenko struct tegra_mc *mc = dev_get_drvdata(dev);
1007ce2785a7SDmitry Osipenko
10085c9016f0SThierry Reding if (mc->soc->ops && mc->soc->ops->resume)
10095c9016f0SThierry Reding return mc->soc->ops->resume(mc);
1010ce2785a7SDmitry Osipenko
1011ce2785a7SDmitry Osipenko return 0;
1012ce2785a7SDmitry Osipenko }
1013ce2785a7SDmitry Osipenko
tegra_mc_sync_state(struct device * dev)101477b14c9dSDmitry Osipenko static void tegra_mc_sync_state(struct device *dev)
101577b14c9dSDmitry Osipenko {
101677b14c9dSDmitry Osipenko struct tegra_mc *mc = dev_get_drvdata(dev);
101777b14c9dSDmitry Osipenko
101877b14c9dSDmitry Osipenko /* check whether ICC provider is registered */
101977b14c9dSDmitry Osipenko if (mc->provider.dev == dev)
102077b14c9dSDmitry Osipenko icc_sync_state(dev);
102177b14c9dSDmitry Osipenko }
102277b14c9dSDmitry Osipenko
1023ce2785a7SDmitry Osipenko static const struct dev_pm_ops tegra_mc_pm_ops = {
10245c9016f0SThierry Reding SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume)
1025ce2785a7SDmitry Osipenko };
1026ce2785a7SDmitry Osipenko
102789184651SThierry Reding static struct platform_driver tegra_mc_driver = {
102889184651SThierry Reding .driver = {
102989184651SThierry Reding .name = "tegra-mc",
103089184651SThierry Reding .of_match_table = tegra_mc_of_match,
1031ce2785a7SDmitry Osipenko .pm = &tegra_mc_pm_ops,
103289184651SThierry Reding .suppress_bind_attrs = true,
103377b14c9dSDmitry Osipenko .sync_state = tegra_mc_sync_state,
103489184651SThierry Reding },
103589184651SThierry Reding .prevent_deferred_probe = true,
103689184651SThierry Reding .probe = tegra_mc_probe,
103789184651SThierry Reding };
103889184651SThierry Reding
tegra_mc_init(void)103989184651SThierry Reding static int tegra_mc_init(void)
104089184651SThierry Reding {
104189184651SThierry Reding return platform_driver_register(&tegra_mc_driver);
104289184651SThierry Reding }
104389184651SThierry Reding arch_initcall(tegra_mc_init);
104489184651SThierry Reding
104589184651SThierry Reding MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
104689184651SThierry Reding MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
1047