1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved. 4 */ 5 6 #include <linux/clk.h> 7 #include <linux/delay.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/interrupt.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_device.h> 14 #include <linux/platform_device.h> 15 #include <linux/slab.h> 16 #include <linux/sort.h> 17 18 #include <soc/tegra/fuse.h> 19 20 #include "mc.h" 21 22 static const struct of_device_id tegra_mc_of_match[] = { 23 #ifdef CONFIG_ARCH_TEGRA_2x_SOC 24 { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc }, 25 #endif 26 #ifdef CONFIG_ARCH_TEGRA_3x_SOC 27 { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc }, 28 #endif 29 #ifdef CONFIG_ARCH_TEGRA_114_SOC 30 { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc }, 31 #endif 32 #ifdef CONFIG_ARCH_TEGRA_124_SOC 33 { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc }, 34 #endif 35 #ifdef CONFIG_ARCH_TEGRA_132_SOC 36 { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc }, 37 #endif 38 #ifdef CONFIG_ARCH_TEGRA_210_SOC 39 { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc }, 40 #endif 41 { } 42 }; 43 MODULE_DEVICE_TABLE(of, tegra_mc_of_match); 44 45 static int tegra_mc_block_dma_common(struct tegra_mc *mc, 46 const struct tegra_mc_reset *rst) 47 { 48 unsigned long flags; 49 u32 value; 50 51 spin_lock_irqsave(&mc->lock, flags); 52 53 value = mc_readl(mc, rst->control) | BIT(rst->bit); 54 mc_writel(mc, value, rst->control); 55 56 spin_unlock_irqrestore(&mc->lock, flags); 57 58 return 0; 59 } 60 61 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc, 62 const struct tegra_mc_reset *rst) 63 { 64 return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0; 65 } 66 67 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc, 68 const struct tegra_mc_reset *rst) 69 { 70 unsigned long flags; 71 u32 value; 72 73 spin_lock_irqsave(&mc->lock, flags); 74 75 value = mc_readl(mc, rst->control) & ~BIT(rst->bit); 76 mc_writel(mc, value, rst->control); 77 78 spin_unlock_irqrestore(&mc->lock, flags); 79 80 return 0; 81 } 82 83 static int tegra_mc_reset_status_common(struct tegra_mc *mc, 84 const struct tegra_mc_reset *rst) 85 { 86 return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0; 87 } 88 89 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = { 90 .block_dma = tegra_mc_block_dma_common, 91 .dma_idling = tegra_mc_dma_idling_common, 92 .unblock_dma = tegra_mc_unblock_dma_common, 93 .reset_status = tegra_mc_reset_status_common, 94 }; 95 96 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev) 97 { 98 return container_of(rcdev, struct tegra_mc, reset); 99 } 100 101 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc, 102 unsigned long id) 103 { 104 unsigned int i; 105 106 for (i = 0; i < mc->soc->num_resets; i++) 107 if (mc->soc->resets[i].id == id) 108 return &mc->soc->resets[i]; 109 110 return NULL; 111 } 112 113 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, 114 unsigned long id) 115 { 116 struct tegra_mc *mc = reset_to_mc(rcdev); 117 const struct tegra_mc_reset_ops *rst_ops; 118 const struct tegra_mc_reset *rst; 119 int retries = 500; 120 int err; 121 122 rst = tegra_mc_reset_find(mc, id); 123 if (!rst) 124 return -ENODEV; 125 126 rst_ops = mc->soc->reset_ops; 127 if (!rst_ops) 128 return -ENODEV; 129 130 if (rst_ops->block_dma) { 131 /* block clients DMA requests */ 132 err = rst_ops->block_dma(mc, rst); 133 if (err) { 134 dev_err(mc->dev, "failed to block %s DMA: %d\n", 135 rst->name, err); 136 return err; 137 } 138 } 139 140 if (rst_ops->dma_idling) { 141 /* wait for completion of the outstanding DMA requests */ 142 while (!rst_ops->dma_idling(mc, rst)) { 143 if (!retries--) { 144 dev_err(mc->dev, "failed to flush %s DMA\n", 145 rst->name); 146 return -EBUSY; 147 } 148 149 usleep_range(10, 100); 150 } 151 } 152 153 if (rst_ops->hotreset_assert) { 154 /* clear clients DMA requests sitting before arbitration */ 155 err = rst_ops->hotreset_assert(mc, rst); 156 if (err) { 157 dev_err(mc->dev, "failed to hot reset %s: %d\n", 158 rst->name, err); 159 return err; 160 } 161 } 162 163 return 0; 164 } 165 166 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev, 167 unsigned long id) 168 { 169 struct tegra_mc *mc = reset_to_mc(rcdev); 170 const struct tegra_mc_reset_ops *rst_ops; 171 const struct tegra_mc_reset *rst; 172 int err; 173 174 rst = tegra_mc_reset_find(mc, id); 175 if (!rst) 176 return -ENODEV; 177 178 rst_ops = mc->soc->reset_ops; 179 if (!rst_ops) 180 return -ENODEV; 181 182 if (rst_ops->hotreset_deassert) { 183 /* take out client from hot reset */ 184 err = rst_ops->hotreset_deassert(mc, rst); 185 if (err) { 186 dev_err(mc->dev, "failed to deassert hot reset %s: %d\n", 187 rst->name, err); 188 return err; 189 } 190 } 191 192 if (rst_ops->unblock_dma) { 193 /* allow new DMA requests to proceed to arbitration */ 194 err = rst_ops->unblock_dma(mc, rst); 195 if (err) { 196 dev_err(mc->dev, "failed to unblock %s DMA : %d\n", 197 rst->name, err); 198 return err; 199 } 200 } 201 202 return 0; 203 } 204 205 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev, 206 unsigned long id) 207 { 208 struct tegra_mc *mc = reset_to_mc(rcdev); 209 const struct tegra_mc_reset_ops *rst_ops; 210 const struct tegra_mc_reset *rst; 211 212 rst = tegra_mc_reset_find(mc, id); 213 if (!rst) 214 return -ENODEV; 215 216 rst_ops = mc->soc->reset_ops; 217 if (!rst_ops) 218 return -ENODEV; 219 220 return rst_ops->reset_status(mc, rst); 221 } 222 223 static const struct reset_control_ops tegra_mc_reset_ops = { 224 .assert = tegra_mc_hotreset_assert, 225 .deassert = tegra_mc_hotreset_deassert, 226 .status = tegra_mc_hotreset_status, 227 }; 228 229 static int tegra_mc_reset_setup(struct tegra_mc *mc) 230 { 231 int err; 232 233 mc->reset.ops = &tegra_mc_reset_ops; 234 mc->reset.owner = THIS_MODULE; 235 mc->reset.of_node = mc->dev->of_node; 236 mc->reset.of_reset_n_cells = 1; 237 mc->reset.nr_resets = mc->soc->num_resets; 238 239 err = reset_controller_register(&mc->reset); 240 if (err < 0) 241 return err; 242 243 return 0; 244 } 245 246 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) 247 { 248 unsigned long long tick; 249 unsigned int i; 250 u32 value; 251 252 /* compute the number of MC clock cycles per tick */ 253 tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); 254 do_div(tick, NSEC_PER_SEC); 255 256 value = mc_readl(mc, MC_EMEM_ARB_CFG); 257 value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; 258 value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); 259 mc_writel(mc, value, MC_EMEM_ARB_CFG); 260 261 /* write latency allowance defaults */ 262 for (i = 0; i < mc->soc->num_clients; i++) { 263 const struct tegra_mc_la *la = &mc->soc->clients[i].la; 264 u32 value; 265 266 value = mc_readl(mc, la->reg); 267 value &= ~(la->mask << la->shift); 268 value |= (la->def & la->mask) << la->shift; 269 mc_writel(mc, value, la->reg); 270 } 271 272 /* latch new values */ 273 mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); 274 275 return 0; 276 } 277 278 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) 279 { 280 unsigned int i; 281 struct tegra_mc_timing *timing = NULL; 282 283 for (i = 0; i < mc->num_timings; i++) { 284 if (mc->timings[i].rate == rate) { 285 timing = &mc->timings[i]; 286 break; 287 } 288 } 289 290 if (!timing) { 291 dev_err(mc->dev, "no memory timing registered for rate %lu\n", 292 rate); 293 return -EINVAL; 294 } 295 296 for (i = 0; i < mc->soc->num_emem_regs; ++i) 297 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]); 298 299 return 0; 300 } 301 302 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) 303 { 304 u8 dram_count; 305 306 dram_count = mc_readl(mc, MC_EMEM_ADR_CFG); 307 dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV; 308 dram_count++; 309 310 return dram_count; 311 } 312 313 static int load_one_timing(struct tegra_mc *mc, 314 struct tegra_mc_timing *timing, 315 struct device_node *node) 316 { 317 int err; 318 u32 tmp; 319 320 err = of_property_read_u32(node, "clock-frequency", &tmp); 321 if (err) { 322 dev_err(mc->dev, 323 "timing %pOFn: failed to read rate\n", node); 324 return err; 325 } 326 327 timing->rate = tmp; 328 timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs, 329 sizeof(u32), GFP_KERNEL); 330 if (!timing->emem_data) 331 return -ENOMEM; 332 333 err = of_property_read_u32_array(node, "nvidia,emem-configuration", 334 timing->emem_data, 335 mc->soc->num_emem_regs); 336 if (err) { 337 dev_err(mc->dev, 338 "timing %pOFn: failed to read EMEM configuration\n", 339 node); 340 return err; 341 } 342 343 return 0; 344 } 345 346 static int load_timings(struct tegra_mc *mc, struct device_node *node) 347 { 348 struct device_node *child; 349 struct tegra_mc_timing *timing; 350 int child_count = of_get_child_count(node); 351 int i = 0, err; 352 353 mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing), 354 GFP_KERNEL); 355 if (!mc->timings) 356 return -ENOMEM; 357 358 mc->num_timings = child_count; 359 360 for_each_child_of_node(node, child) { 361 timing = &mc->timings[i++]; 362 363 err = load_one_timing(mc, timing, child); 364 if (err) { 365 of_node_put(child); 366 return err; 367 } 368 } 369 370 return 0; 371 } 372 373 static int tegra_mc_setup_timings(struct tegra_mc *mc) 374 { 375 struct device_node *node; 376 u32 ram_code, node_ram_code; 377 int err; 378 379 ram_code = tegra_read_ram_code(); 380 381 mc->num_timings = 0; 382 383 for_each_child_of_node(mc->dev->of_node, node) { 384 err = of_property_read_u32(node, "nvidia,ram-code", 385 &node_ram_code); 386 if (err || (node_ram_code != ram_code)) 387 continue; 388 389 err = load_timings(mc, node); 390 of_node_put(node); 391 if (err) 392 return err; 393 break; 394 } 395 396 if (mc->num_timings == 0) 397 dev_warn(mc->dev, 398 "no memory timings for RAM code %u registered\n", 399 ram_code); 400 401 return 0; 402 } 403 404 static const char *const status_names[32] = { 405 [ 1] = "External interrupt", 406 [ 6] = "EMEM address decode error", 407 [ 7] = "GART page fault", 408 [ 8] = "Security violation", 409 [ 9] = "EMEM arbitration error", 410 [10] = "Page fault", 411 [11] = "Invalid APB ASID update", 412 [12] = "VPR violation", 413 [13] = "Secure carveout violation", 414 [16] = "MTS carveout violation", 415 }; 416 417 static const char *const error_names[8] = { 418 [2] = "EMEM decode error", 419 [3] = "TrustZone violation", 420 [4] = "Carveout violation", 421 [6] = "SMMU translation error", 422 }; 423 424 static irqreturn_t tegra_mc_irq(int irq, void *data) 425 { 426 struct tegra_mc *mc = data; 427 unsigned long status; 428 unsigned int bit; 429 430 /* mask all interrupts to avoid flooding */ 431 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; 432 if (!status) 433 return IRQ_NONE; 434 435 for_each_set_bit(bit, &status, 32) { 436 const char *error = status_names[bit] ?: "unknown"; 437 const char *client = "unknown", *desc; 438 const char *direction, *secure; 439 phys_addr_t addr = 0; 440 unsigned int i; 441 char perm[7]; 442 u8 id, type; 443 u32 value; 444 445 value = mc_readl(mc, MC_ERR_STATUS); 446 447 #ifdef CONFIG_PHYS_ADDR_T_64BIT 448 if (mc->soc->num_address_bits > 32) { 449 addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & 450 MC_ERR_STATUS_ADR_HI_MASK); 451 addr <<= 32; 452 } 453 #endif 454 455 if (value & MC_ERR_STATUS_RW) 456 direction = "write"; 457 else 458 direction = "read"; 459 460 if (value & MC_ERR_STATUS_SECURITY) 461 secure = "secure "; 462 else 463 secure = ""; 464 465 id = value & mc->soc->client_id_mask; 466 467 for (i = 0; i < mc->soc->num_clients; i++) { 468 if (mc->soc->clients[i].id == id) { 469 client = mc->soc->clients[i].name; 470 break; 471 } 472 } 473 474 type = (value & MC_ERR_STATUS_TYPE_MASK) >> 475 MC_ERR_STATUS_TYPE_SHIFT; 476 desc = error_names[type]; 477 478 switch (value & MC_ERR_STATUS_TYPE_MASK) { 479 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE: 480 perm[0] = ' '; 481 perm[1] = '['; 482 483 if (value & MC_ERR_STATUS_READABLE) 484 perm[2] = 'R'; 485 else 486 perm[2] = '-'; 487 488 if (value & MC_ERR_STATUS_WRITABLE) 489 perm[3] = 'W'; 490 else 491 perm[3] = '-'; 492 493 if (value & MC_ERR_STATUS_NONSECURE) 494 perm[4] = '-'; 495 else 496 perm[4] = 'S'; 497 498 perm[5] = ']'; 499 perm[6] = '\0'; 500 break; 501 502 default: 503 perm[0] = '\0'; 504 break; 505 } 506 507 value = mc_readl(mc, MC_ERR_ADR); 508 addr |= value; 509 510 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n", 511 client, secure, direction, &addr, error, 512 desc, perm); 513 } 514 515 /* clear interrupts */ 516 mc_writel(mc, status, MC_INTSTATUS); 517 518 return IRQ_HANDLED; 519 } 520 521 static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data) 522 { 523 struct tegra_mc *mc = data; 524 unsigned long status; 525 unsigned int bit; 526 527 /* mask all interrupts to avoid flooding */ 528 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; 529 if (!status) 530 return IRQ_NONE; 531 532 for_each_set_bit(bit, &status, 32) { 533 const char *direction = "read", *secure = ""; 534 const char *error = status_names[bit]; 535 const char *client, *desc; 536 phys_addr_t addr; 537 u32 value, reg; 538 u8 id, type; 539 540 switch (BIT(bit)) { 541 case MC_INT_DECERR_EMEM: 542 reg = MC_DECERR_EMEM_OTHERS_STATUS; 543 value = mc_readl(mc, reg); 544 545 id = value & mc->soc->client_id_mask; 546 desc = error_names[2]; 547 548 if (value & BIT(31)) 549 direction = "write"; 550 break; 551 552 case MC_INT_INVALID_GART_PAGE: 553 reg = MC_GART_ERROR_REQ; 554 value = mc_readl(mc, reg); 555 556 id = (value >> 1) & mc->soc->client_id_mask; 557 desc = error_names[2]; 558 559 if (value & BIT(0)) 560 direction = "write"; 561 break; 562 563 case MC_INT_SECURITY_VIOLATION: 564 reg = MC_SECURITY_VIOLATION_STATUS; 565 value = mc_readl(mc, reg); 566 567 id = value & mc->soc->client_id_mask; 568 type = (value & BIT(30)) ? 4 : 3; 569 desc = error_names[type]; 570 secure = "secure "; 571 572 if (value & BIT(31)) 573 direction = "write"; 574 break; 575 576 default: 577 continue; 578 } 579 580 client = mc->soc->clients[id].name; 581 addr = mc_readl(mc, reg + sizeof(u32)); 582 583 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n", 584 client, secure, direction, &addr, error, 585 desc); 586 } 587 588 /* clear interrupts */ 589 mc_writel(mc, status, MC_INTSTATUS); 590 591 return IRQ_HANDLED; 592 } 593 594 static int tegra_mc_probe(struct platform_device *pdev) 595 { 596 struct resource *res; 597 struct tegra_mc *mc; 598 void *isr; 599 u64 mask; 600 int err; 601 602 mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL); 603 if (!mc) 604 return -ENOMEM; 605 606 platform_set_drvdata(pdev, mc); 607 spin_lock_init(&mc->lock); 608 mc->soc = of_device_get_match_data(&pdev->dev); 609 mc->dev = &pdev->dev; 610 611 mask = DMA_BIT_MASK(mc->soc->num_address_bits); 612 613 err = dma_coerce_mask_and_coherent(&pdev->dev, mask); 614 if (err < 0) { 615 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err); 616 return err; 617 } 618 619 /* length of MC tick in nanoseconds */ 620 mc->tick = 30; 621 622 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 623 mc->regs = devm_ioremap_resource(&pdev->dev, res); 624 if (IS_ERR(mc->regs)) 625 return PTR_ERR(mc->regs); 626 627 mc->clk = devm_clk_get(&pdev->dev, "mc"); 628 if (IS_ERR(mc->clk)) { 629 dev_err(&pdev->dev, "failed to get MC clock: %ld\n", 630 PTR_ERR(mc->clk)); 631 return PTR_ERR(mc->clk); 632 } 633 634 #ifdef CONFIG_ARCH_TEGRA_2x_SOC 635 if (mc->soc == &tegra20_mc_soc) { 636 isr = tegra20_mc_irq; 637 } else 638 #endif 639 { 640 /* ensure that debug features are disabled */ 641 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); 642 643 err = tegra_mc_setup_latency_allowance(mc); 644 if (err < 0) { 645 dev_err(&pdev->dev, 646 "failed to setup latency allowance: %d\n", 647 err); 648 return err; 649 } 650 651 isr = tegra_mc_irq; 652 653 err = tegra_mc_setup_timings(mc); 654 if (err < 0) { 655 dev_err(&pdev->dev, "failed to setup timings: %d\n", 656 err); 657 return err; 658 } 659 } 660 661 mc->irq = platform_get_irq(pdev, 0); 662 if (mc->irq < 0) { 663 dev_err(&pdev->dev, "interrupt not specified\n"); 664 return mc->irq; 665 } 666 667 WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n"); 668 669 mc_writel(mc, mc->soc->intmask, MC_INTMASK); 670 671 err = devm_request_irq(&pdev->dev, mc->irq, isr, 0, 672 dev_name(&pdev->dev), mc); 673 if (err < 0) { 674 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq, 675 err); 676 return err; 677 } 678 679 err = tegra_mc_reset_setup(mc); 680 if (err < 0) 681 dev_err(&pdev->dev, "failed to register reset controller: %d\n", 682 err); 683 684 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) { 685 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc); 686 if (IS_ERR(mc->smmu)) { 687 dev_err(&pdev->dev, "failed to probe SMMU: %ld\n", 688 PTR_ERR(mc->smmu)); 689 mc->smmu = NULL; 690 } 691 } 692 693 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) { 694 mc->gart = tegra_gart_probe(&pdev->dev, mc); 695 if (IS_ERR(mc->gart)) { 696 dev_err(&pdev->dev, "failed to probe GART: %ld\n", 697 PTR_ERR(mc->gart)); 698 mc->gart = NULL; 699 } 700 } 701 702 return 0; 703 } 704 705 static int tegra_mc_suspend(struct device *dev) 706 { 707 struct tegra_mc *mc = dev_get_drvdata(dev); 708 int err; 709 710 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) { 711 err = tegra_gart_suspend(mc->gart); 712 if (err) 713 return err; 714 } 715 716 return 0; 717 } 718 719 static int tegra_mc_resume(struct device *dev) 720 { 721 struct tegra_mc *mc = dev_get_drvdata(dev); 722 int err; 723 724 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) { 725 err = tegra_gart_resume(mc->gart); 726 if (err) 727 return err; 728 } 729 730 return 0; 731 } 732 733 static const struct dev_pm_ops tegra_mc_pm_ops = { 734 .suspend = tegra_mc_suspend, 735 .resume = tegra_mc_resume, 736 }; 737 738 static struct platform_driver tegra_mc_driver = { 739 .driver = { 740 .name = "tegra-mc", 741 .of_match_table = tegra_mc_of_match, 742 .pm = &tegra_mc_pm_ops, 743 .suppress_bind_attrs = true, 744 }, 745 .prevent_deferred_probe = true, 746 .probe = tegra_mc_probe, 747 }; 748 749 static int tegra_mc_init(void) 750 { 751 return platform_driver_register(&tegra_mc_driver); 752 } 753 arch_initcall(tegra_mc_init); 754 755 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); 756 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver"); 757 MODULE_LICENSE("GPL v2"); 758