1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/dma-direct.h> 3 #include <linux/dma-debug.h> 4 #include <linux/dmar.h> 5 #include <linux/export.h> 6 #include <linux/memblock.h> 7 #include <linux/gfp.h> 8 #include <linux/pci.h> 9 10 #include <asm/proto.h> 11 #include <asm/dma.h> 12 #include <asm/iommu.h> 13 #include <asm/gart.h> 14 #include <asm/calgary.h> 15 #include <asm/x86_init.h> 16 #include <asm/iommu_table.h> 17 18 static bool disable_dac_quirk __read_mostly; 19 20 const struct dma_map_ops *dma_ops; 21 EXPORT_SYMBOL(dma_ops); 22 23 #ifdef CONFIG_IOMMU_DEBUG 24 int panic_on_overflow __read_mostly = 1; 25 int force_iommu __read_mostly = 1; 26 #else 27 int panic_on_overflow __read_mostly = 0; 28 int force_iommu __read_mostly = 0; 29 #endif 30 31 int iommu_merge __read_mostly = 0; 32 33 int no_iommu __read_mostly; 34 /* Set this to 1 if there is a HW IOMMU in the system */ 35 int iommu_detected __read_mostly = 0; 36 37 /* 38 * This variable becomes 1 if iommu=pt is passed on the kernel command line. 39 * If this variable is 1, IOMMU implementations do no DMA translation for 40 * devices and allow every device to access to whole physical memory. This is 41 * useful if a user wants to use an IOMMU only for KVM device assignment to 42 * guests and not for driver dma translation. 43 * It is also possible to disable by default in kernel config, and enable with 44 * iommu=nopt at boot time. 45 */ 46 #ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH 47 int iommu_pass_through __read_mostly = 1; 48 #else 49 int iommu_pass_through __read_mostly; 50 #endif 51 52 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; 53 54 void __init pci_iommu_alloc(void) 55 { 56 struct iommu_table_entry *p; 57 58 sort_iommu_table(__iommu_table, __iommu_table_end); 59 check_iommu_entries(__iommu_table, __iommu_table_end); 60 61 for (p = __iommu_table; p < __iommu_table_end; p++) { 62 if (p && p->detect && p->detect() > 0) { 63 p->flags |= IOMMU_DETECTED; 64 if (p->early_init) 65 p->early_init(); 66 if (p->flags & IOMMU_FINISH_IF_DETECTED) 67 break; 68 } 69 } 70 } 71 72 /* 73 * See <Documentation/x86/x86_64/boot-options.rst> for the iommu kernel 74 * parameter documentation. 75 */ 76 static __init int iommu_setup(char *p) 77 { 78 iommu_merge = 1; 79 80 if (!p) 81 return -EINVAL; 82 83 while (*p) { 84 if (!strncmp(p, "off", 3)) 85 no_iommu = 1; 86 /* gart_parse_options has more force support */ 87 if (!strncmp(p, "force", 5)) 88 force_iommu = 1; 89 if (!strncmp(p, "noforce", 7)) { 90 iommu_merge = 0; 91 force_iommu = 0; 92 } 93 94 if (!strncmp(p, "biomerge", 8)) { 95 iommu_merge = 1; 96 force_iommu = 1; 97 } 98 if (!strncmp(p, "panic", 5)) 99 panic_on_overflow = 1; 100 if (!strncmp(p, "nopanic", 7)) 101 panic_on_overflow = 0; 102 if (!strncmp(p, "merge", 5)) { 103 iommu_merge = 1; 104 force_iommu = 1; 105 } 106 if (!strncmp(p, "nomerge", 7)) 107 iommu_merge = 0; 108 if (!strncmp(p, "forcesac", 8)) 109 pr_warn("forcesac option ignored.\n"); 110 if (!strncmp(p, "allowdac", 8)) 111 pr_warn("allowdac option ignored.\n"); 112 if (!strncmp(p, "nodac", 5)) 113 pr_warn("nodac option ignored.\n"); 114 if (!strncmp(p, "usedac", 6)) { 115 disable_dac_quirk = true; 116 return 1; 117 } 118 #ifdef CONFIG_SWIOTLB 119 if (!strncmp(p, "soft", 4)) 120 swiotlb = 1; 121 #endif 122 if (!strncmp(p, "pt", 2)) 123 iommu_pass_through = 1; 124 if (!strncmp(p, "nopt", 4)) 125 iommu_pass_through = 0; 126 127 gart_parse_options(p); 128 129 #ifdef CONFIG_CALGARY_IOMMU 130 if (!strncmp(p, "calgary", 7)) 131 use_calgary = 1; 132 #endif /* CONFIG_CALGARY_IOMMU */ 133 134 p += strcspn(p, ","); 135 if (*p == ',') 136 ++p; 137 } 138 return 0; 139 } 140 early_param("iommu", iommu_setup); 141 142 static int __init pci_iommu_init(void) 143 { 144 struct iommu_table_entry *p; 145 146 x86_init.iommu.iommu_init(); 147 148 for (p = __iommu_table; p < __iommu_table_end; p++) { 149 if (p && (p->flags & IOMMU_DETECTED) && p->late_init) 150 p->late_init(); 151 } 152 153 return 0; 154 } 155 /* Must execute after PCI subsystem */ 156 rootfs_initcall(pci_iommu_init); 157 158 #ifdef CONFIG_PCI 159 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 160 161 static int via_no_dac_cb(struct pci_dev *pdev, void *data) 162 { 163 pdev->dev.bus_dma_mask = DMA_BIT_MASK(32); 164 return 0; 165 } 166 167 static void via_no_dac(struct pci_dev *dev) 168 { 169 if (!disable_dac_quirk) { 170 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 171 pci_walk_bus(dev->subordinate, via_no_dac_cb, NULL); 172 } 173 } 174 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, 175 PCI_CLASS_BRIDGE_PCI, 8, via_no_dac); 176 #endif 177