1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/dma-direct.h> 3 #include <linux/dma-debug.h> 4 #include <linux/dmar.h> 5 #include <linux/export.h> 6 #include <linux/bootmem.h> 7 #include <linux/gfp.h> 8 #include <linux/pci.h> 9 10 #include <asm/proto.h> 11 #include <asm/dma.h> 12 #include <asm/iommu.h> 13 #include <asm/gart.h> 14 #include <asm/calgary.h> 15 #include <asm/x86_init.h> 16 #include <asm/iommu_table.h> 17 18 static bool disable_dac_quirk __read_mostly; 19 20 const struct dma_map_ops *dma_ops = &dma_direct_ops; 21 EXPORT_SYMBOL(dma_ops); 22 23 #ifdef CONFIG_IOMMU_DEBUG 24 int panic_on_overflow __read_mostly = 1; 25 int force_iommu __read_mostly = 1; 26 #else 27 int panic_on_overflow __read_mostly = 0; 28 int force_iommu __read_mostly = 0; 29 #endif 30 31 int iommu_merge __read_mostly = 0; 32 33 int no_iommu __read_mostly; 34 /* Set this to 1 if there is a HW IOMMU in the system */ 35 int iommu_detected __read_mostly = 0; 36 37 /* 38 * This variable becomes 1 if iommu=pt is passed on the kernel command line. 39 * If this variable is 1, IOMMU implementations do no DMA translation for 40 * devices and allow every device to access to whole physical memory. This is 41 * useful if a user wants to use an IOMMU only for KVM device assignment to 42 * guests and not for driver dma translation. 43 */ 44 int iommu_pass_through __read_mostly; 45 46 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; 47 48 /* Dummy device used for NULL arguments (normally ISA). */ 49 struct device x86_dma_fallback_dev = { 50 .init_name = "fallback device", 51 .coherent_dma_mask = ISA_DMA_BIT_MASK, 52 .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 53 }; 54 EXPORT_SYMBOL(x86_dma_fallback_dev); 55 56 void __init pci_iommu_alloc(void) 57 { 58 struct iommu_table_entry *p; 59 60 sort_iommu_table(__iommu_table, __iommu_table_end); 61 check_iommu_entries(__iommu_table, __iommu_table_end); 62 63 for (p = __iommu_table; p < __iommu_table_end; p++) { 64 if (p && p->detect && p->detect() > 0) { 65 p->flags |= IOMMU_DETECTED; 66 if (p->early_init) 67 p->early_init(); 68 if (p->flags & IOMMU_FINISH_IF_DETECTED) 69 break; 70 } 71 } 72 } 73 74 bool arch_dma_alloc_attrs(struct device **dev) 75 { 76 if (!*dev) 77 *dev = &x86_dma_fallback_dev; 78 79 if (!is_device_dma_capable(*dev)) 80 return false; 81 return true; 82 83 } 84 EXPORT_SYMBOL(arch_dma_alloc_attrs); 85 86 /* 87 * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel 88 * parameter documentation. 89 */ 90 static __init int iommu_setup(char *p) 91 { 92 iommu_merge = 1; 93 94 if (!p) 95 return -EINVAL; 96 97 while (*p) { 98 if (!strncmp(p, "off", 3)) 99 no_iommu = 1; 100 /* gart_parse_options has more force support */ 101 if (!strncmp(p, "force", 5)) 102 force_iommu = 1; 103 if (!strncmp(p, "noforce", 7)) { 104 iommu_merge = 0; 105 force_iommu = 0; 106 } 107 108 if (!strncmp(p, "biomerge", 8)) { 109 iommu_merge = 1; 110 force_iommu = 1; 111 } 112 if (!strncmp(p, "panic", 5)) 113 panic_on_overflow = 1; 114 if (!strncmp(p, "nopanic", 7)) 115 panic_on_overflow = 0; 116 if (!strncmp(p, "merge", 5)) { 117 iommu_merge = 1; 118 force_iommu = 1; 119 } 120 if (!strncmp(p, "nomerge", 7)) 121 iommu_merge = 0; 122 if (!strncmp(p, "forcesac", 8)) 123 pr_warn("forcesac option ignored.\n"); 124 if (!strncmp(p, "allowdac", 8)) 125 pr_warn("allowdac option ignored.\n"); 126 if (!strncmp(p, "nodac", 5)) 127 pr_warn("nodac option ignored.\n"); 128 if (!strncmp(p, "usedac", 6)) { 129 disable_dac_quirk = true; 130 return 1; 131 } 132 #ifdef CONFIG_SWIOTLB 133 if (!strncmp(p, "soft", 4)) 134 swiotlb = 1; 135 #endif 136 if (!strncmp(p, "pt", 2)) 137 iommu_pass_through = 1; 138 139 gart_parse_options(p); 140 141 #ifdef CONFIG_CALGARY_IOMMU 142 if (!strncmp(p, "calgary", 7)) 143 use_calgary = 1; 144 #endif /* CONFIG_CALGARY_IOMMU */ 145 146 p += strcspn(p, ","); 147 if (*p == ',') 148 ++p; 149 } 150 return 0; 151 } 152 early_param("iommu", iommu_setup); 153 154 static int __init pci_iommu_init(void) 155 { 156 struct iommu_table_entry *p; 157 158 #ifdef CONFIG_PCI 159 dma_debug_add_bus(&pci_bus_type); 160 #endif 161 x86_init.iommu.iommu_init(); 162 163 for (p = __iommu_table; p < __iommu_table_end; p++) { 164 if (p && (p->flags & IOMMU_DETECTED) && p->late_init) 165 p->late_init(); 166 } 167 168 return 0; 169 } 170 /* Must execute after PCI subsystem */ 171 rootfs_initcall(pci_iommu_init); 172 173 #ifdef CONFIG_PCI 174 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 175 176 static int via_no_dac_cb(struct pci_dev *pdev, void *data) 177 { 178 pdev->dev.dma_32bit_limit = true; 179 return 0; 180 } 181 182 static void via_no_dac(struct pci_dev *dev) 183 { 184 if (!disable_dac_quirk) { 185 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 186 pci_walk_bus(dev->subordinate, via_no_dac_cb, NULL); 187 } 188 } 189 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, 190 PCI_CLASS_BRIDGE_PCI, 8, via_no_dac); 191 #endif 192