1 /* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2007 - 2009 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * The full GNU General Public License is included in this distribution in 15 * the file called "COPYING". 16 * 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/pci.h> 21 #include <linux/smp.h> 22 #include <linux/interrupt.h> 23 #include <linux/dca.h> 24 25 /* either a kernel change is needed, or we need something like this in kernel */ 26 #ifndef CONFIG_SMP 27 #include <asm/smp.h> 28 #undef cpu_physical_id 29 #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24) 30 #endif 31 32 #include "dma.h" 33 #include "registers.h" 34 35 /* 36 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6 37 * contain the bit number of the APIC ID to map into the DCA tag. If the valid 38 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag. 39 */ 40 #define DCA_TAG_MAP_VALID 0x80 41 42 #define DCA3_TAG_MAP_BIT_TO_INV 0x80 43 #define DCA3_TAG_MAP_BIT_TO_SEL 0x40 44 #define DCA3_TAG_MAP_LITERAL_VAL 0x1 45 46 #define DCA_TAG_MAP_MASK 0xDF 47 48 /* expected tag map bytes for I/OAT ver.2 */ 49 #define DCA2_TAG_MAP_BYTE0 0x80 50 #define DCA2_TAG_MAP_BYTE1 0x0 51 #define DCA2_TAG_MAP_BYTE2 0x81 52 #define DCA2_TAG_MAP_BYTE3 0x82 53 #define DCA2_TAG_MAP_BYTE4 0x82 54 55 /* verify if tag map matches expected values */ 56 static inline int dca2_tag_map_valid(u8 *tag_map) 57 { 58 return ((tag_map[0] == DCA2_TAG_MAP_BYTE0) && 59 (tag_map[1] == DCA2_TAG_MAP_BYTE1) && 60 (tag_map[2] == DCA2_TAG_MAP_BYTE2) && 61 (tag_map[3] == DCA2_TAG_MAP_BYTE3) && 62 (tag_map[4] == DCA2_TAG_MAP_BYTE4)); 63 } 64 65 /* 66 * "Legacy" DCA systems do not implement the DCA register set in the 67 * I/OAT device. Software needs direct support for their tag mappings. 68 */ 69 70 #define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x)) 71 #define IOAT_TAG_MAP_LEN 8 72 73 /* pack PCI B/D/F into a u16 */ 74 static inline u16 dcaid_from_pcidev(struct pci_dev *pci) 75 { 76 return (pci->bus->number << 8) | pci->devfn; 77 } 78 79 static int dca_enabled_in_bios(struct pci_dev *pdev) 80 { 81 /* CPUID level 9 returns DCA configuration */ 82 /* Bit 0 indicates DCA enabled by the BIOS */ 83 unsigned long cpuid_level_9; 84 int res; 85 86 cpuid_level_9 = cpuid_eax(9); 87 res = test_bit(0, &cpuid_level_9); 88 if (!res) 89 dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n"); 90 91 return res; 92 } 93 94 int system_has_dca_enabled(struct pci_dev *pdev) 95 { 96 if (boot_cpu_has(X86_FEATURE_DCA)) 97 return dca_enabled_in_bios(pdev); 98 99 dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n"); 100 return 0; 101 } 102 103 struct ioat_dca_slot { 104 struct pci_dev *pdev; /* requester device */ 105 u16 rid; /* requester id, as used by IOAT */ 106 }; 107 108 #define IOAT_DCA_MAX_REQ 6 109 #define IOAT3_DCA_MAX_REQ 2 110 111 struct ioat_dca_priv { 112 void __iomem *iobase; 113 void __iomem *dca_base; 114 int max_requesters; 115 int requester_count; 116 u8 tag_map[IOAT_TAG_MAP_LEN]; 117 struct ioat_dca_slot req_slots[0]; 118 }; 119 120 static int ioat_dca_dev_managed(struct dca_provider *dca, 121 struct device *dev) 122 { 123 struct ioat_dca_priv *ioatdca = dca_priv(dca); 124 struct pci_dev *pdev; 125 int i; 126 127 pdev = to_pci_dev(dev); 128 for (i = 0; i < ioatdca->max_requesters; i++) { 129 if (ioatdca->req_slots[i].pdev == pdev) 130 return 1; 131 } 132 return 0; 133 } 134 135 static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev) 136 { 137 struct ioat_dca_priv *ioatdca = dca_priv(dca); 138 struct pci_dev *pdev; 139 int i; 140 u16 id; 141 u16 global_req_table; 142 143 /* This implementation only supports PCI-Express */ 144 if (!dev_is_pci(dev)) 145 return -ENODEV; 146 pdev = to_pci_dev(dev); 147 id = dcaid_from_pcidev(pdev); 148 149 if (ioatdca->requester_count == ioatdca->max_requesters) 150 return -ENODEV; 151 152 for (i = 0; i < ioatdca->max_requesters; i++) { 153 if (ioatdca->req_slots[i].pdev == NULL) { 154 /* found an empty slot */ 155 ioatdca->requester_count++; 156 ioatdca->req_slots[i].pdev = pdev; 157 ioatdca->req_slots[i].rid = id; 158 global_req_table = 159 readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET); 160 writel(id | IOAT_DCA_GREQID_VALID, 161 ioatdca->iobase + global_req_table + (i * 4)); 162 return i; 163 } 164 } 165 /* Error, ioatdma->requester_count is out of whack */ 166 return -EFAULT; 167 } 168 169 static int ioat_dca_remove_requester(struct dca_provider *dca, 170 struct device *dev) 171 { 172 struct ioat_dca_priv *ioatdca = dca_priv(dca); 173 struct pci_dev *pdev; 174 int i; 175 u16 global_req_table; 176 177 /* This implementation only supports PCI-Express */ 178 if (!dev_is_pci(dev)) 179 return -ENODEV; 180 pdev = to_pci_dev(dev); 181 182 for (i = 0; i < ioatdca->max_requesters; i++) { 183 if (ioatdca->req_slots[i].pdev == pdev) { 184 global_req_table = 185 readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET); 186 writel(0, ioatdca->iobase + global_req_table + (i * 4)); 187 ioatdca->req_slots[i].pdev = NULL; 188 ioatdca->req_slots[i].rid = 0; 189 ioatdca->requester_count--; 190 return i; 191 } 192 } 193 return -ENODEV; 194 } 195 196 static u8 ioat_dca_get_tag(struct dca_provider *dca, 197 struct device *dev, 198 int cpu) 199 { 200 u8 tag; 201 202 struct ioat_dca_priv *ioatdca = dca_priv(dca); 203 int i, apic_id, bit, value; 204 u8 entry; 205 206 tag = 0; 207 apic_id = cpu_physical_id(cpu); 208 209 for (i = 0; i < IOAT_TAG_MAP_LEN; i++) { 210 entry = ioatdca->tag_map[i]; 211 if (entry & DCA3_TAG_MAP_BIT_TO_SEL) { 212 bit = entry & 213 ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV); 214 value = (apic_id & (1 << bit)) ? 1 : 0; 215 } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) { 216 bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV; 217 value = (apic_id & (1 << bit)) ? 0 : 1; 218 } else { 219 value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0; 220 } 221 tag |= (value << i); 222 } 223 224 return tag; 225 } 226 227 static const struct dca_ops ioat_dca_ops = { 228 .add_requester = ioat_dca_add_requester, 229 .remove_requester = ioat_dca_remove_requester, 230 .get_tag = ioat_dca_get_tag, 231 .dev_managed = ioat_dca_dev_managed, 232 }; 233 234 static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset) 235 { 236 int slots = 0; 237 u32 req; 238 u16 global_req_table; 239 240 global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET); 241 if (global_req_table == 0) 242 return 0; 243 244 do { 245 req = readl(iobase + global_req_table + (slots * sizeof(u32))); 246 slots++; 247 } while ((req & IOAT_DCA_GREQID_LASTID) == 0); 248 249 return slots; 250 } 251 252 static inline int dca3_tag_map_invalid(u8 *tag_map) 253 { 254 /* 255 * If the tag map is not programmed by the BIOS the default is: 256 * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00 257 * 258 * This an invalid map and will result in only 2 possible tags 259 * 0x1F and 0x00. 0x00 is an invalid DCA tag so we know that 260 * this entire definition is invalid. 261 */ 262 return ((tag_map[0] == DCA_TAG_MAP_VALID) && 263 (tag_map[1] == DCA_TAG_MAP_VALID) && 264 (tag_map[2] == DCA_TAG_MAP_VALID) && 265 (tag_map[3] == DCA_TAG_MAP_VALID) && 266 (tag_map[4] == DCA_TAG_MAP_VALID)); 267 } 268 269 struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) 270 { 271 struct dca_provider *dca; 272 struct ioat_dca_priv *ioatdca; 273 int slots; 274 int i; 275 int err; 276 u16 dca_offset; 277 u16 csi_fsb_control; 278 u16 pcie_control; 279 u8 bit; 280 281 union { 282 u64 full; 283 struct { 284 u32 low; 285 u32 high; 286 }; 287 } tag_map; 288 289 if (!system_has_dca_enabled(pdev)) 290 return NULL; 291 292 dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET); 293 if (dca_offset == 0) 294 return NULL; 295 296 slots = ioat_dca_count_dca_slots(iobase, dca_offset); 297 if (slots == 0) 298 return NULL; 299 300 dca = alloc_dca_provider(&ioat_dca_ops, 301 sizeof(*ioatdca) 302 + (sizeof(struct ioat_dca_slot) * slots)); 303 if (!dca) 304 return NULL; 305 306 ioatdca = dca_priv(dca); 307 ioatdca->iobase = iobase; 308 ioatdca->dca_base = iobase + dca_offset; 309 ioatdca->max_requesters = slots; 310 311 /* some bios might not know to turn these on */ 312 csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET); 313 if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) { 314 csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH; 315 writew(csi_fsb_control, 316 ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET); 317 } 318 pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET); 319 if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) { 320 pcie_control |= IOAT3_PCI_CONTROL_MEMWR; 321 writew(pcie_control, 322 ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET); 323 } 324 325 326 /* TODO version, compatibility and configuration checks */ 327 328 /* copy out the APIC to DCA tag map */ 329 tag_map.low = 330 readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW); 331 tag_map.high = 332 readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH); 333 for (i = 0; i < 8; i++) { 334 bit = tag_map.full >> (8 * i); 335 ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK; 336 } 337 338 if (dca3_tag_map_invalid(ioatdca->tag_map)) { 339 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 340 pr_warn_once("%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n", 341 dev_driver_string(&pdev->dev), 342 dev_name(&pdev->dev)); 343 free_dca_provider(dca); 344 return NULL; 345 } 346 347 err = register_dca_provider(dca, &pdev->dev); 348 if (err) { 349 free_dca_provider(dca); 350 return NULL; 351 } 352 353 return dca; 354 } 355