1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2006, Intel Corporation.
4 *
5 * Copyright (C) 2006-2008 Intel Corporation
6 * Author: Ashok Raj <ashok.raj@intel.com>
7 * Author: Shaohua Li <shaohua.li@intel.com>
8 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
9 *
10 * This file implements early detection/parsing of Remapping Devices
11 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
12 * tables.
13 *
14 * These routines are used by both DMA-remapping and Interrupt-remapping
15 */
16
17 #define pr_fmt(fmt) "DMAR: " fmt
18
19 #include <linux/pci.h>
20 #include <linux/dmar.h>
21 #include <linux/iova.h>
22 #include <linux/timer.h>
23 #include <linux/irq.h>
24 #include <linux/interrupt.h>
25 #include <linux/tboot.h>
26 #include <linux/dmi.h>
27 #include <linux/slab.h>
28 #include <linux/iommu.h>
29 #include <linux/numa.h>
30 #include <linux/limits.h>
31 #include <asm/irq_remapping.h>
32
33 #include "iommu.h"
34 #include "../irq_remapping.h"
35 #include "perf.h"
36 #include "trace.h"
37 #include "perfmon.h"
38
39 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
40 struct dmar_res_callback {
41 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
42 void *arg[ACPI_DMAR_TYPE_RESERVED];
43 bool ignore_unhandled;
44 bool print_entry;
45 };
46
47 /*
48 * Assumptions:
49 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
50 * before IO devices managed by that unit.
51 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
52 * after IO devices managed by that unit.
53 * 3) Hotplug events are rare.
54 *
55 * Locking rules for DMA and interrupt remapping related global data structures:
56 * 1) Use dmar_global_lock in process context
57 * 2) Use RCU in interrupt context
58 */
59 DECLARE_RWSEM(dmar_global_lock);
60 LIST_HEAD(dmar_drhd_units);
61
62 struct acpi_table_header * __initdata dmar_tbl;
63 static int dmar_dev_scope_status = 1;
64 static DEFINE_IDA(dmar_seq_ids);
65
66 static int alloc_iommu(struct dmar_drhd_unit *drhd);
67 static void free_iommu(struct intel_iommu *iommu);
68
dmar_register_drhd_unit(struct dmar_drhd_unit * drhd)69 static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
70 {
71 /*
72 * add INCLUDE_ALL at the tail, so scan the list will find it at
73 * the very end.
74 */
75 if (drhd->include_all)
76 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
77 else
78 list_add_rcu(&drhd->list, &dmar_drhd_units);
79 }
80
dmar_alloc_dev_scope(void * start,void * end,int * cnt)81 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
82 {
83 struct acpi_dmar_device_scope *scope;
84
85 *cnt = 0;
86 while (start < end) {
87 scope = start;
88 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
89 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
90 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
91 (*cnt)++;
92 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
93 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
94 pr_warn("Unsupported device scope\n");
95 }
96 start += scope->length;
97 }
98 if (*cnt == 0)
99 return NULL;
100
101 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
102 }
103
dmar_free_dev_scope(struct dmar_dev_scope ** devices,int * cnt)104 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
105 {
106 int i;
107 struct device *tmp_dev;
108
109 if (*devices && *cnt) {
110 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
111 put_device(tmp_dev);
112 kfree(*devices);
113 }
114
115 *devices = NULL;
116 *cnt = 0;
117 }
118
119 /* Optimize out kzalloc()/kfree() for normal cases */
120 static char dmar_pci_notify_info_buf[64];
121
122 static struct dmar_pci_notify_info *
dmar_alloc_pci_notify_info(struct pci_dev * dev,unsigned long event)123 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
124 {
125 int level = 0;
126 size_t size;
127 struct pci_dev *tmp;
128 struct dmar_pci_notify_info *info;
129
130 /*
131 * Ignore devices that have a domain number higher than what can
132 * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
133 */
134 if (pci_domain_nr(dev->bus) > U16_MAX)
135 return NULL;
136
137 /* Only generate path[] for device addition event */
138 if (event == BUS_NOTIFY_ADD_DEVICE)
139 for (tmp = dev; tmp; tmp = tmp->bus->self)
140 level++;
141
142 size = struct_size(info, path, level);
143 if (size <= sizeof(dmar_pci_notify_info_buf)) {
144 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
145 } else {
146 info = kzalloc(size, GFP_KERNEL);
147 if (!info) {
148 if (dmar_dev_scope_status == 0)
149 dmar_dev_scope_status = -ENOMEM;
150 return NULL;
151 }
152 }
153
154 info->event = event;
155 info->dev = dev;
156 info->seg = pci_domain_nr(dev->bus);
157 info->level = level;
158 if (event == BUS_NOTIFY_ADD_DEVICE) {
159 for (tmp = dev; tmp; tmp = tmp->bus->self) {
160 level--;
161 info->path[level].bus = tmp->bus->number;
162 info->path[level].device = PCI_SLOT(tmp->devfn);
163 info->path[level].function = PCI_FUNC(tmp->devfn);
164 if (pci_is_root_bus(tmp->bus))
165 info->bus = tmp->bus->number;
166 }
167 }
168
169 return info;
170 }
171
dmar_free_pci_notify_info(struct dmar_pci_notify_info * info)172 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
173 {
174 if ((void *)info != dmar_pci_notify_info_buf)
175 kfree(info);
176 }
177
dmar_match_pci_path(struct dmar_pci_notify_info * info,int bus,struct acpi_dmar_pci_path * path,int count)178 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
179 struct acpi_dmar_pci_path *path, int count)
180 {
181 int i;
182
183 if (info->bus != bus)
184 goto fallback;
185 if (info->level != count)
186 goto fallback;
187
188 for (i = 0; i < count; i++) {
189 if (path[i].device != info->path[i].device ||
190 path[i].function != info->path[i].function)
191 goto fallback;
192 }
193
194 return true;
195
196 fallback:
197
198 if (count != 1)
199 return false;
200
201 i = info->level - 1;
202 if (bus == info->path[i].bus &&
203 path[0].device == info->path[i].device &&
204 path[0].function == info->path[i].function) {
205 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
206 bus, path[0].device, path[0].function);
207 return true;
208 }
209
210 return false;
211 }
212
213 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
dmar_insert_dev_scope(struct dmar_pci_notify_info * info,void * start,void * end,u16 segment,struct dmar_dev_scope * devices,int devices_cnt)214 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
215 void *start, void*end, u16 segment,
216 struct dmar_dev_scope *devices,
217 int devices_cnt)
218 {
219 int i, level;
220 struct device *tmp, *dev = &info->dev->dev;
221 struct acpi_dmar_device_scope *scope;
222 struct acpi_dmar_pci_path *path;
223
224 if (segment != info->seg)
225 return 0;
226
227 for (; start < end; start += scope->length) {
228 scope = start;
229 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
230 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
231 continue;
232
233 path = (struct acpi_dmar_pci_path *)(scope + 1);
234 level = (scope->length - sizeof(*scope)) / sizeof(*path);
235 if (!dmar_match_pci_path(info, scope->bus, path, level))
236 continue;
237
238 /*
239 * We expect devices with endpoint scope to have normal PCI
240 * headers, and devices with bridge scope to have bridge PCI
241 * headers. However PCI NTB devices may be listed in the
242 * DMAR table with bridge scope, even though they have a
243 * normal PCI header. NTB devices are identified by class
244 * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
245 * for this special case.
246 */
247 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
248 info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
249 (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
250 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
251 info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
252 pr_warn("Device scope type does not match for %s\n",
253 pci_name(info->dev));
254 return -EINVAL;
255 }
256
257 for_each_dev_scope(devices, devices_cnt, i, tmp)
258 if (tmp == NULL) {
259 devices[i].bus = info->dev->bus->number;
260 devices[i].devfn = info->dev->devfn;
261 rcu_assign_pointer(devices[i].dev,
262 get_device(dev));
263 return 1;
264 }
265 if (WARN_ON(i >= devices_cnt))
266 return -EINVAL;
267 }
268
269 return 0;
270 }
271
dmar_remove_dev_scope(struct dmar_pci_notify_info * info,u16 segment,struct dmar_dev_scope * devices,int count)272 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
273 struct dmar_dev_scope *devices, int count)
274 {
275 int index;
276 struct device *tmp;
277
278 if (info->seg != segment)
279 return 0;
280
281 for_each_active_dev_scope(devices, count, index, tmp)
282 if (tmp == &info->dev->dev) {
283 RCU_INIT_POINTER(devices[index].dev, NULL);
284 synchronize_rcu();
285 put_device(tmp);
286 return 1;
287 }
288
289 return 0;
290 }
291
dmar_pci_bus_add_dev(struct dmar_pci_notify_info * info)292 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
293 {
294 int ret = 0;
295 struct dmar_drhd_unit *dmaru;
296 struct acpi_dmar_hardware_unit *drhd;
297
298 for_each_drhd_unit(dmaru) {
299 if (dmaru->include_all)
300 continue;
301
302 drhd = container_of(dmaru->hdr,
303 struct acpi_dmar_hardware_unit, header);
304 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
305 ((void *)drhd) + drhd->header.length,
306 dmaru->segment,
307 dmaru->devices, dmaru->devices_cnt);
308 if (ret)
309 break;
310 }
311 if (ret >= 0)
312 ret = dmar_iommu_notify_scope_dev(info);
313 if (ret < 0 && dmar_dev_scope_status == 0)
314 dmar_dev_scope_status = ret;
315
316 if (ret >= 0)
317 intel_irq_remap_add_device(info);
318
319 return ret;
320 }
321
dmar_pci_bus_del_dev(struct dmar_pci_notify_info * info)322 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
323 {
324 struct dmar_drhd_unit *dmaru;
325
326 for_each_drhd_unit(dmaru)
327 if (dmar_remove_dev_scope(info, dmaru->segment,
328 dmaru->devices, dmaru->devices_cnt))
329 break;
330 dmar_iommu_notify_scope_dev(info);
331 }
332
vf_inherit_msi_domain(struct pci_dev * pdev)333 static inline void vf_inherit_msi_domain(struct pci_dev *pdev)
334 {
335 struct pci_dev *physfn = pci_physfn(pdev);
336
337 dev_set_msi_domain(&pdev->dev, dev_get_msi_domain(&physfn->dev));
338 }
339
dmar_pci_bus_notifier(struct notifier_block * nb,unsigned long action,void * data)340 static int dmar_pci_bus_notifier(struct notifier_block *nb,
341 unsigned long action, void *data)
342 {
343 struct pci_dev *pdev = to_pci_dev(data);
344 struct dmar_pci_notify_info *info;
345
346 /* Only care about add/remove events for physical functions.
347 * For VFs we actually do the lookup based on the corresponding
348 * PF in device_to_iommu() anyway. */
349 if (pdev->is_virtfn) {
350 /*
351 * Ensure that the VF device inherits the irq domain of the
352 * PF device. Ideally the device would inherit the domain
353 * from the bus, but DMAR can have multiple units per bus
354 * which makes this impossible. The VF 'bus' could inherit
355 * from the PF device, but that's yet another x86'sism to
356 * inflict on everybody else.
357 */
358 if (action == BUS_NOTIFY_ADD_DEVICE)
359 vf_inherit_msi_domain(pdev);
360 return NOTIFY_DONE;
361 }
362
363 if (action != BUS_NOTIFY_ADD_DEVICE &&
364 action != BUS_NOTIFY_REMOVED_DEVICE)
365 return NOTIFY_DONE;
366
367 info = dmar_alloc_pci_notify_info(pdev, action);
368 if (!info)
369 return NOTIFY_DONE;
370
371 down_write(&dmar_global_lock);
372 if (action == BUS_NOTIFY_ADD_DEVICE)
373 dmar_pci_bus_add_dev(info);
374 else if (action == BUS_NOTIFY_REMOVED_DEVICE)
375 dmar_pci_bus_del_dev(info);
376 up_write(&dmar_global_lock);
377
378 dmar_free_pci_notify_info(info);
379
380 return NOTIFY_OK;
381 }
382
383 static struct notifier_block dmar_pci_bus_nb = {
384 .notifier_call = dmar_pci_bus_notifier,
385 .priority = 1,
386 };
387
388 static struct dmar_drhd_unit *
dmar_find_dmaru(struct acpi_dmar_hardware_unit * drhd)389 dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
390 {
391 struct dmar_drhd_unit *dmaru;
392
393 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
394 dmar_rcu_check())
395 if (dmaru->segment == drhd->segment &&
396 dmaru->reg_base_addr == drhd->address)
397 return dmaru;
398
399 return NULL;
400 }
401
402 /*
403 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
404 * structure which uniquely represent one DMA remapping hardware unit
405 * present in the platform
406 */
dmar_parse_one_drhd(struct acpi_dmar_header * header,void * arg)407 static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
408 {
409 struct acpi_dmar_hardware_unit *drhd;
410 struct dmar_drhd_unit *dmaru;
411 int ret;
412
413 drhd = (struct acpi_dmar_hardware_unit *)header;
414 dmaru = dmar_find_dmaru(drhd);
415 if (dmaru)
416 goto out;
417
418 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
419 if (!dmaru)
420 return -ENOMEM;
421
422 /*
423 * If header is allocated from slab by ACPI _DSM method, we need to
424 * copy the content because the memory buffer will be freed on return.
425 */
426 dmaru->hdr = (void *)(dmaru + 1);
427 memcpy(dmaru->hdr, header, header->length);
428 dmaru->reg_base_addr = drhd->address;
429 dmaru->segment = drhd->segment;
430 /* The size of the register set is 2 ^ N 4 KB pages. */
431 dmaru->reg_size = 1UL << (drhd->size + 12);
432 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
433 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
434 ((void *)drhd) + drhd->header.length,
435 &dmaru->devices_cnt);
436 if (dmaru->devices_cnt && dmaru->devices == NULL) {
437 kfree(dmaru);
438 return -ENOMEM;
439 }
440
441 ret = alloc_iommu(dmaru);
442 if (ret) {
443 dmar_free_dev_scope(&dmaru->devices,
444 &dmaru->devices_cnt);
445 kfree(dmaru);
446 return ret;
447 }
448 dmar_register_drhd_unit(dmaru);
449
450 out:
451 if (arg)
452 (*(int *)arg)++;
453
454 return 0;
455 }
456
dmar_free_drhd(struct dmar_drhd_unit * dmaru)457 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
458 {
459 if (dmaru->devices && dmaru->devices_cnt)
460 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
461 if (dmaru->iommu)
462 free_iommu(dmaru->iommu);
463 kfree(dmaru);
464 }
465
dmar_parse_one_andd(struct acpi_dmar_header * header,void * arg)466 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
467 void *arg)
468 {
469 struct acpi_dmar_andd *andd = (void *)header;
470
471 /* Check for NUL termination within the designated length */
472 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
473 pr_warn(FW_BUG
474 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
475 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
476 dmi_get_system_info(DMI_BIOS_VENDOR),
477 dmi_get_system_info(DMI_BIOS_VERSION),
478 dmi_get_system_info(DMI_PRODUCT_VERSION));
479 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
480 return -EINVAL;
481 }
482 pr_info("ANDD device: %x name: %s\n", andd->device_number,
483 andd->device_name);
484
485 return 0;
486 }
487
488 #ifdef CONFIG_ACPI_NUMA
dmar_parse_one_rhsa(struct acpi_dmar_header * header,void * arg)489 static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
490 {
491 struct acpi_dmar_rhsa *rhsa;
492 struct dmar_drhd_unit *drhd;
493
494 rhsa = (struct acpi_dmar_rhsa *)header;
495 for_each_drhd_unit(drhd) {
496 if (drhd->reg_base_addr == rhsa->base_address) {
497 int node = pxm_to_node(rhsa->proximity_domain);
498
499 if (node != NUMA_NO_NODE && !node_online(node))
500 node = NUMA_NO_NODE;
501 drhd->iommu->node = node;
502 return 0;
503 }
504 }
505 pr_warn(FW_BUG
506 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
507 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
508 rhsa->base_address,
509 dmi_get_system_info(DMI_BIOS_VENDOR),
510 dmi_get_system_info(DMI_BIOS_VERSION),
511 dmi_get_system_info(DMI_PRODUCT_VERSION));
512 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
513
514 return 0;
515 }
516 #else
517 #define dmar_parse_one_rhsa dmar_res_noop
518 #endif
519
520 static void
dmar_table_print_dmar_entry(struct acpi_dmar_header * header)521 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
522 {
523 struct acpi_dmar_hardware_unit *drhd;
524 struct acpi_dmar_reserved_memory *rmrr;
525 struct acpi_dmar_atsr *atsr;
526 struct acpi_dmar_rhsa *rhsa;
527 struct acpi_dmar_satc *satc;
528
529 switch (header->type) {
530 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
531 drhd = container_of(header, struct acpi_dmar_hardware_unit,
532 header);
533 pr_info("DRHD base: %#016Lx flags: %#x\n",
534 (unsigned long long)drhd->address, drhd->flags);
535 break;
536 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
537 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
538 header);
539 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
540 (unsigned long long)rmrr->base_address,
541 (unsigned long long)rmrr->end_address);
542 break;
543 case ACPI_DMAR_TYPE_ROOT_ATS:
544 atsr = container_of(header, struct acpi_dmar_atsr, header);
545 pr_info("ATSR flags: %#x\n", atsr->flags);
546 break;
547 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
548 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
549 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
550 (unsigned long long)rhsa->base_address,
551 rhsa->proximity_domain);
552 break;
553 case ACPI_DMAR_TYPE_NAMESPACE:
554 /* We don't print this here because we need to sanity-check
555 it first. So print it in dmar_parse_one_andd() instead. */
556 break;
557 case ACPI_DMAR_TYPE_SATC:
558 satc = container_of(header, struct acpi_dmar_satc, header);
559 pr_info("SATC flags: 0x%x\n", satc->flags);
560 break;
561 }
562 }
563
564 /**
565 * dmar_table_detect - checks to see if the platform supports DMAR devices
566 */
dmar_table_detect(void)567 static int __init dmar_table_detect(void)
568 {
569 acpi_status status = AE_OK;
570
571 /* if we could find DMAR table, then there are DMAR devices */
572 status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
573
574 if (ACPI_SUCCESS(status) && !dmar_tbl) {
575 pr_warn("Unable to map DMAR\n");
576 status = AE_NOT_FOUND;
577 }
578
579 return ACPI_SUCCESS(status) ? 0 : -ENOENT;
580 }
581
dmar_walk_remapping_entries(struct acpi_dmar_header * start,size_t len,struct dmar_res_callback * cb)582 static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
583 size_t len, struct dmar_res_callback *cb)
584 {
585 struct acpi_dmar_header *iter, *next;
586 struct acpi_dmar_header *end = ((void *)start) + len;
587
588 for (iter = start; iter < end; iter = next) {
589 next = (void *)iter + iter->length;
590 if (iter->length == 0) {
591 /* Avoid looping forever on bad ACPI tables */
592 pr_debug(FW_BUG "Invalid 0-length structure\n");
593 break;
594 } else if (next > end) {
595 /* Avoid passing table end */
596 pr_warn(FW_BUG "Record passes table end\n");
597 return -EINVAL;
598 }
599
600 if (cb->print_entry)
601 dmar_table_print_dmar_entry(iter);
602
603 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
604 /* continue for forward compatibility */
605 pr_debug("Unknown DMAR structure type %d\n",
606 iter->type);
607 } else if (cb->cb[iter->type]) {
608 int ret;
609
610 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
611 if (ret)
612 return ret;
613 } else if (!cb->ignore_unhandled) {
614 pr_warn("No handler for DMAR structure type %d\n",
615 iter->type);
616 return -EINVAL;
617 }
618 }
619
620 return 0;
621 }
622
dmar_walk_dmar_table(struct acpi_table_dmar * dmar,struct dmar_res_callback * cb)623 static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
624 struct dmar_res_callback *cb)
625 {
626 return dmar_walk_remapping_entries((void *)(dmar + 1),
627 dmar->header.length - sizeof(*dmar), cb);
628 }
629
630 /**
631 * parse_dmar_table - parses the DMA reporting table
632 */
633 static int __init
parse_dmar_table(void)634 parse_dmar_table(void)
635 {
636 struct acpi_table_dmar *dmar;
637 int drhd_count = 0;
638 int ret;
639 struct dmar_res_callback cb = {
640 .print_entry = true,
641 .ignore_unhandled = true,
642 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
643 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
644 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
645 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
646 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
647 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
648 .cb[ACPI_DMAR_TYPE_SATC] = &dmar_parse_one_satc,
649 };
650
651 /*
652 * Do it again, earlier dmar_tbl mapping could be mapped with
653 * fixed map.
654 */
655 dmar_table_detect();
656
657 /*
658 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
659 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
660 */
661 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
662
663 dmar = (struct acpi_table_dmar *)dmar_tbl;
664 if (!dmar)
665 return -ENODEV;
666
667 if (dmar->width < PAGE_SHIFT - 1) {
668 pr_warn("Invalid DMAR haw\n");
669 return -EINVAL;
670 }
671
672 pr_info("Host address width %d\n", dmar->width + 1);
673 ret = dmar_walk_dmar_table(dmar, &cb);
674 if (ret == 0 && drhd_count == 0)
675 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
676
677 return ret;
678 }
679
dmar_pci_device_match(struct dmar_dev_scope devices[],int cnt,struct pci_dev * dev)680 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
681 int cnt, struct pci_dev *dev)
682 {
683 int index;
684 struct device *tmp;
685
686 while (dev) {
687 for_each_active_dev_scope(devices, cnt, index, tmp)
688 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
689 return 1;
690
691 /* Check our parent */
692 dev = dev->bus->self;
693 }
694
695 return 0;
696 }
697
698 struct dmar_drhd_unit *
dmar_find_matched_drhd_unit(struct pci_dev * dev)699 dmar_find_matched_drhd_unit(struct pci_dev *dev)
700 {
701 struct dmar_drhd_unit *dmaru;
702 struct acpi_dmar_hardware_unit *drhd;
703
704 dev = pci_physfn(dev);
705
706 rcu_read_lock();
707 for_each_drhd_unit(dmaru) {
708 drhd = container_of(dmaru->hdr,
709 struct acpi_dmar_hardware_unit,
710 header);
711
712 if (dmaru->include_all &&
713 drhd->segment == pci_domain_nr(dev->bus))
714 goto out;
715
716 if (dmar_pci_device_match(dmaru->devices,
717 dmaru->devices_cnt, dev))
718 goto out;
719 }
720 dmaru = NULL;
721 out:
722 rcu_read_unlock();
723
724 return dmaru;
725 }
726
dmar_acpi_insert_dev_scope(u8 device_number,struct acpi_device * adev)727 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
728 struct acpi_device *adev)
729 {
730 struct dmar_drhd_unit *dmaru;
731 struct acpi_dmar_hardware_unit *drhd;
732 struct acpi_dmar_device_scope *scope;
733 struct device *tmp;
734 int i;
735 struct acpi_dmar_pci_path *path;
736
737 for_each_drhd_unit(dmaru) {
738 drhd = container_of(dmaru->hdr,
739 struct acpi_dmar_hardware_unit,
740 header);
741
742 for (scope = (void *)(drhd + 1);
743 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
744 scope = ((void *)scope) + scope->length) {
745 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
746 continue;
747 if (scope->enumeration_id != device_number)
748 continue;
749
750 path = (void *)(scope + 1);
751 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
752 dev_name(&adev->dev), dmaru->reg_base_addr,
753 scope->bus, path->device, path->function);
754 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
755 if (tmp == NULL) {
756 dmaru->devices[i].bus = scope->bus;
757 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
758 path->function);
759 rcu_assign_pointer(dmaru->devices[i].dev,
760 get_device(&adev->dev));
761 return;
762 }
763 BUG_ON(i >= dmaru->devices_cnt);
764 }
765 }
766 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
767 device_number, dev_name(&adev->dev));
768 }
769
dmar_acpi_dev_scope_init(void)770 static int __init dmar_acpi_dev_scope_init(void)
771 {
772 struct acpi_dmar_andd *andd;
773
774 if (dmar_tbl == NULL)
775 return -ENODEV;
776
777 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
778 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
779 andd = ((void *)andd) + andd->header.length) {
780 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
781 acpi_handle h;
782 struct acpi_device *adev;
783
784 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
785 andd->device_name,
786 &h))) {
787 pr_err("Failed to find handle for ACPI object %s\n",
788 andd->device_name);
789 continue;
790 }
791 adev = acpi_fetch_acpi_dev(h);
792 if (!adev) {
793 pr_err("Failed to get device for ACPI object %s\n",
794 andd->device_name);
795 continue;
796 }
797 dmar_acpi_insert_dev_scope(andd->device_number, adev);
798 }
799 }
800 return 0;
801 }
802
dmar_dev_scope_init(void)803 int __init dmar_dev_scope_init(void)
804 {
805 struct pci_dev *dev = NULL;
806 struct dmar_pci_notify_info *info;
807
808 if (dmar_dev_scope_status != 1)
809 return dmar_dev_scope_status;
810
811 if (list_empty(&dmar_drhd_units)) {
812 dmar_dev_scope_status = -ENODEV;
813 } else {
814 dmar_dev_scope_status = 0;
815
816 dmar_acpi_dev_scope_init();
817
818 for_each_pci_dev(dev) {
819 if (dev->is_virtfn)
820 continue;
821
822 info = dmar_alloc_pci_notify_info(dev,
823 BUS_NOTIFY_ADD_DEVICE);
824 if (!info) {
825 pci_dev_put(dev);
826 return dmar_dev_scope_status;
827 } else {
828 dmar_pci_bus_add_dev(info);
829 dmar_free_pci_notify_info(info);
830 }
831 }
832 }
833
834 return dmar_dev_scope_status;
835 }
836
dmar_register_bus_notifier(void)837 void __init dmar_register_bus_notifier(void)
838 {
839 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
840 }
841
842
dmar_table_init(void)843 int __init dmar_table_init(void)
844 {
845 static int dmar_table_initialized;
846 int ret;
847
848 if (dmar_table_initialized == 0) {
849 ret = parse_dmar_table();
850 if (ret < 0) {
851 if (ret != -ENODEV)
852 pr_info("Parse DMAR table failure.\n");
853 } else if (list_empty(&dmar_drhd_units)) {
854 pr_info("No DMAR devices found\n");
855 ret = -ENODEV;
856 }
857
858 if (ret < 0)
859 dmar_table_initialized = ret;
860 else
861 dmar_table_initialized = 1;
862 }
863
864 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
865 }
866
warn_invalid_dmar(u64 addr,const char * message)867 static void warn_invalid_dmar(u64 addr, const char *message)
868 {
869 pr_warn_once(FW_BUG
870 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
871 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
872 addr, message,
873 dmi_get_system_info(DMI_BIOS_VENDOR),
874 dmi_get_system_info(DMI_BIOS_VERSION),
875 dmi_get_system_info(DMI_PRODUCT_VERSION));
876 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
877 }
878
879 static int __ref
dmar_validate_one_drhd(struct acpi_dmar_header * entry,void * arg)880 dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
881 {
882 struct acpi_dmar_hardware_unit *drhd;
883 void __iomem *addr;
884 u64 cap, ecap;
885
886 drhd = (void *)entry;
887 if (!drhd->address) {
888 warn_invalid_dmar(0, "");
889 return -EINVAL;
890 }
891
892 if (arg)
893 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
894 else
895 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
896 if (!addr) {
897 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
898 return -EINVAL;
899 }
900
901 cap = dmar_readq(addr + DMAR_CAP_REG);
902 ecap = dmar_readq(addr + DMAR_ECAP_REG);
903
904 if (arg)
905 iounmap(addr);
906 else
907 early_iounmap(addr, VTD_PAGE_SIZE);
908
909 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
910 warn_invalid_dmar(drhd->address, " returns all ones");
911 return -EINVAL;
912 }
913
914 return 0;
915 }
916
detect_intel_iommu(void)917 void __init detect_intel_iommu(void)
918 {
919 int ret;
920 struct dmar_res_callback validate_drhd_cb = {
921 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
922 .ignore_unhandled = true,
923 };
924
925 down_write(&dmar_global_lock);
926 ret = dmar_table_detect();
927 if (!ret)
928 ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
929 &validate_drhd_cb);
930 if (!ret && !no_iommu && !iommu_detected &&
931 (!dmar_disabled || dmar_platform_optin())) {
932 iommu_detected = 1;
933 /* Make sure ACS will be enabled */
934 pci_request_acs();
935 }
936
937 #ifdef CONFIG_X86
938 if (!ret) {
939 x86_init.iommu.iommu_init = intel_iommu_init;
940 x86_platform.iommu_shutdown = intel_iommu_shutdown;
941 }
942
943 #endif
944
945 if (dmar_tbl) {
946 acpi_put_table(dmar_tbl);
947 dmar_tbl = NULL;
948 }
949 up_write(&dmar_global_lock);
950 }
951
unmap_iommu(struct intel_iommu * iommu)952 static void unmap_iommu(struct intel_iommu *iommu)
953 {
954 iounmap(iommu->reg);
955 release_mem_region(iommu->reg_phys, iommu->reg_size);
956 }
957
958 /**
959 * map_iommu: map the iommu's registers
960 * @iommu: the iommu to map
961 * @drhd: DMA remapping hardware definition structure
962 *
963 * Memory map the iommu's registers. Start w/ a single page, and
964 * possibly expand if that turns out to be insufficent.
965 */
map_iommu(struct intel_iommu * iommu,struct dmar_drhd_unit * drhd)966 static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)
967 {
968 u64 phys_addr = drhd->reg_base_addr;
969 int map_size, err=0;
970
971 iommu->reg_phys = phys_addr;
972 iommu->reg_size = drhd->reg_size;
973
974 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
975 pr_err("Can't reserve memory\n");
976 err = -EBUSY;
977 goto out;
978 }
979
980 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
981 if (!iommu->reg) {
982 pr_err("Can't map the region\n");
983 err = -ENOMEM;
984 goto release;
985 }
986
987 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
988 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
989
990 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
991 err = -EINVAL;
992 warn_invalid_dmar(phys_addr, " returns all ones");
993 goto unmap;
994 }
995
996 /* the registers might be more than one page */
997 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
998 cap_max_fault_reg_offset(iommu->cap));
999 map_size = VTD_PAGE_ALIGN(map_size);
1000 if (map_size > iommu->reg_size) {
1001 iounmap(iommu->reg);
1002 release_mem_region(iommu->reg_phys, iommu->reg_size);
1003 iommu->reg_size = map_size;
1004 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
1005 iommu->name)) {
1006 pr_err("Can't reserve memory\n");
1007 err = -EBUSY;
1008 goto out;
1009 }
1010 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
1011 if (!iommu->reg) {
1012 pr_err("Can't map the region\n");
1013 err = -ENOMEM;
1014 goto release;
1015 }
1016 }
1017
1018 if (cap_ecmds(iommu->cap)) {
1019 int i;
1020
1021 for (i = 0; i < DMA_MAX_NUM_ECMDCAP; i++) {
1022 iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG +
1023 i * DMA_ECMD_REG_STEP);
1024 }
1025 }
1026
1027 err = 0;
1028 goto out;
1029
1030 unmap:
1031 iounmap(iommu->reg);
1032 release:
1033 release_mem_region(iommu->reg_phys, iommu->reg_size);
1034 out:
1035 return err;
1036 }
1037
alloc_iommu(struct dmar_drhd_unit * drhd)1038 static int alloc_iommu(struct dmar_drhd_unit *drhd)
1039 {
1040 struct intel_iommu *iommu;
1041 u32 ver, sts;
1042 int agaw = -1;
1043 int msagaw = -1;
1044 int err;
1045
1046 if (!drhd->reg_base_addr) {
1047 warn_invalid_dmar(0, "");
1048 return -EINVAL;
1049 }
1050
1051 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1052 if (!iommu)
1053 return -ENOMEM;
1054
1055 iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
1056 DMAR_UNITS_SUPPORTED - 1, GFP_KERNEL);
1057 if (iommu->seq_id < 0) {
1058 pr_err("Failed to allocate seq_id\n");
1059 err = iommu->seq_id;
1060 goto error;
1061 }
1062 sprintf(iommu->name, "dmar%d", iommu->seq_id);
1063
1064 err = map_iommu(iommu, drhd);
1065 if (err) {
1066 pr_err("Failed to map %s\n", iommu->name);
1067 goto error_free_seq_id;
1068 }
1069
1070 err = -EINVAL;
1071 if (!cap_sagaw(iommu->cap) &&
1072 (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {
1073 pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1074 iommu->name);
1075 drhd->ignored = 1;
1076 }
1077
1078 if (!drhd->ignored) {
1079 agaw = iommu_calculate_agaw(iommu);
1080 if (agaw < 0) {
1081 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1082 iommu->seq_id);
1083 drhd->ignored = 1;
1084 }
1085 }
1086 if (!drhd->ignored) {
1087 msagaw = iommu_calculate_max_sagaw(iommu);
1088 if (msagaw < 0) {
1089 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1090 iommu->seq_id);
1091 drhd->ignored = 1;
1092 agaw = -1;
1093 }
1094 }
1095 iommu->agaw = agaw;
1096 iommu->msagaw = msagaw;
1097 iommu->segment = drhd->segment;
1098
1099 iommu->node = NUMA_NO_NODE;
1100
1101 ver = readl(iommu->reg + DMAR_VER_REG);
1102 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1103 iommu->name,
1104 (unsigned long long)drhd->reg_base_addr,
1105 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1106 (unsigned long long)iommu->cap,
1107 (unsigned long long)iommu->ecap);
1108
1109 /* Reflect status in gcmd */
1110 sts = readl(iommu->reg + DMAR_GSTS_REG);
1111 if (sts & DMA_GSTS_IRES)
1112 iommu->gcmd |= DMA_GCMD_IRE;
1113 if (sts & DMA_GSTS_TES)
1114 iommu->gcmd |= DMA_GCMD_TE;
1115 if (sts & DMA_GSTS_QIES)
1116 iommu->gcmd |= DMA_GCMD_QIE;
1117
1118 if (alloc_iommu_pmu(iommu))
1119 pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);
1120
1121 raw_spin_lock_init(&iommu->register_lock);
1122
1123 /*
1124 * A value of N in PSS field of eCap register indicates hardware
1125 * supports PASID field of N+1 bits.
1126 */
1127 if (pasid_supported(iommu))
1128 iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
1129
1130 /*
1131 * This is only for hotplug; at boot time intel_iommu_enabled won't
1132 * be set yet. When intel_iommu_init() runs, it registers the units
1133 * present at boot time, then sets intel_iommu_enabled.
1134 */
1135 if (intel_iommu_enabled && !drhd->ignored) {
1136 err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1137 intel_iommu_groups,
1138 "%s", iommu->name);
1139 if (err)
1140 goto err_unmap;
1141
1142 err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
1143 if (err)
1144 goto err_sysfs;
1145
1146 iommu_pmu_register(iommu);
1147 }
1148
1149 drhd->iommu = iommu;
1150 iommu->drhd = drhd;
1151
1152 return 0;
1153
1154 err_sysfs:
1155 iommu_device_sysfs_remove(&iommu->iommu);
1156 err_unmap:
1157 free_iommu_pmu(iommu);
1158 unmap_iommu(iommu);
1159 error_free_seq_id:
1160 ida_free(&dmar_seq_ids, iommu->seq_id);
1161 error:
1162 kfree(iommu);
1163 return err;
1164 }
1165
free_iommu(struct intel_iommu * iommu)1166 static void free_iommu(struct intel_iommu *iommu)
1167 {
1168 if (intel_iommu_enabled && !iommu->drhd->ignored) {
1169 iommu_pmu_unregister(iommu);
1170 iommu_device_unregister(&iommu->iommu);
1171 iommu_device_sysfs_remove(&iommu->iommu);
1172 }
1173
1174 free_iommu_pmu(iommu);
1175
1176 if (iommu->irq) {
1177 if (iommu->pr_irq) {
1178 free_irq(iommu->pr_irq, iommu);
1179 dmar_free_hwirq(iommu->pr_irq);
1180 iommu->pr_irq = 0;
1181 }
1182 free_irq(iommu->irq, iommu);
1183 dmar_free_hwirq(iommu->irq);
1184 iommu->irq = 0;
1185 }
1186
1187 if (iommu->qi) {
1188 free_page((unsigned long)iommu->qi->desc);
1189 kfree(iommu->qi->desc_status);
1190 kfree(iommu->qi);
1191 }
1192
1193 if (iommu->reg)
1194 unmap_iommu(iommu);
1195
1196 ida_free(&dmar_seq_ids, iommu->seq_id);
1197 kfree(iommu);
1198 }
1199
1200 /*
1201 * Reclaim all the submitted descriptors which have completed its work.
1202 */
reclaim_free_desc(struct q_inval * qi)1203 static inline void reclaim_free_desc(struct q_inval *qi)
1204 {
1205 while (qi->desc_status[qi->free_tail] == QI_FREE && qi->free_tail != qi->free_head) {
1206 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1207 qi->free_cnt++;
1208 }
1209 }
1210
qi_type_string(u8 type)1211 static const char *qi_type_string(u8 type)
1212 {
1213 switch (type) {
1214 case QI_CC_TYPE:
1215 return "Context-cache Invalidation";
1216 case QI_IOTLB_TYPE:
1217 return "IOTLB Invalidation";
1218 case QI_DIOTLB_TYPE:
1219 return "Device-TLB Invalidation";
1220 case QI_IEC_TYPE:
1221 return "Interrupt Entry Cache Invalidation";
1222 case QI_IWD_TYPE:
1223 return "Invalidation Wait";
1224 case QI_EIOTLB_TYPE:
1225 return "PASID-based IOTLB Invalidation";
1226 case QI_PC_TYPE:
1227 return "PASID-cache Invalidation";
1228 case QI_DEIOTLB_TYPE:
1229 return "PASID-based Device-TLB Invalidation";
1230 case QI_PGRP_RESP_TYPE:
1231 return "Page Group Response";
1232 default:
1233 return "UNKNOWN";
1234 }
1235 }
1236
qi_dump_fault(struct intel_iommu * iommu,u32 fault)1237 static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)
1238 {
1239 unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG);
1240 u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1241 struct qi_desc *desc = iommu->qi->desc + head;
1242
1243 if (fault & DMA_FSTS_IQE)
1244 pr_err("VT-d detected Invalidation Queue Error: Reason %llx",
1245 DMAR_IQER_REG_IQEI(iqe_err));
1246 if (fault & DMA_FSTS_ITE)
1247 pr_err("VT-d detected Invalidation Time-out Error: SID %llx",
1248 DMAR_IQER_REG_ITESID(iqe_err));
1249 if (fault & DMA_FSTS_ICE)
1250 pr_err("VT-d detected Invalidation Completion Error: SID %llx",
1251 DMAR_IQER_REG_ICESID(iqe_err));
1252
1253 pr_err("QI HEAD: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1254 qi_type_string(desc->qw0 & 0xf),
1255 (unsigned long long)desc->qw0,
1256 (unsigned long long)desc->qw1);
1257
1258 head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;
1259 head <<= qi_shift(iommu);
1260 desc = iommu->qi->desc + head;
1261
1262 pr_err("QI PRIOR: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1263 qi_type_string(desc->qw0 & 0xf),
1264 (unsigned long long)desc->qw0,
1265 (unsigned long long)desc->qw1);
1266 }
1267
qi_check_fault(struct intel_iommu * iommu,int index,int wait_index)1268 static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
1269 {
1270 u32 fault;
1271 int head, tail;
1272 struct q_inval *qi = iommu->qi;
1273 int shift = qi_shift(iommu);
1274
1275 if (qi->desc_status[wait_index] == QI_ABORT)
1276 return -EAGAIN;
1277
1278 fault = readl(iommu->reg + DMAR_FSTS_REG);
1279 if (fault & (DMA_FSTS_IQE | DMA_FSTS_ITE | DMA_FSTS_ICE))
1280 qi_dump_fault(iommu, fault);
1281
1282 /*
1283 * If IQE happens, the head points to the descriptor associated
1284 * with the error. No new descriptors are fetched until the IQE
1285 * is cleared.
1286 */
1287 if (fault & DMA_FSTS_IQE) {
1288 head = readl(iommu->reg + DMAR_IQH_REG);
1289 if ((head >> shift) == index) {
1290 struct qi_desc *desc = qi->desc + head;
1291
1292 /*
1293 * desc->qw2 and desc->qw3 are either reserved or
1294 * used by software as private data. We won't print
1295 * out these two qw's for security consideration.
1296 */
1297 memcpy(desc, qi->desc + (wait_index << shift),
1298 1 << shift);
1299 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1300 pr_info("Invalidation Queue Error (IQE) cleared\n");
1301 return -EINVAL;
1302 }
1303 }
1304
1305 /*
1306 * If ITE happens, all pending wait_desc commands are aborted.
1307 * No new descriptors are fetched until the ITE is cleared.
1308 */
1309 if (fault & DMA_FSTS_ITE) {
1310 head = readl(iommu->reg + DMAR_IQH_REG);
1311 head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1312 head |= 1;
1313 tail = readl(iommu->reg + DMAR_IQT_REG);
1314 tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1315
1316 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1317 pr_info("Invalidation Time-out Error (ITE) cleared\n");
1318
1319 do {
1320 if (qi->desc_status[head] == QI_IN_USE)
1321 qi->desc_status[head] = QI_ABORT;
1322 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1323 } while (head != tail);
1324
1325 if (qi->desc_status[wait_index] == QI_ABORT)
1326 return -EAGAIN;
1327 }
1328
1329 if (fault & DMA_FSTS_ICE) {
1330 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1331 pr_info("Invalidation Completion Error (ICE) cleared\n");
1332 }
1333
1334 return 0;
1335 }
1336
1337 /*
1338 * Function to submit invalidation descriptors of all types to the queued
1339 * invalidation interface(QI). Multiple descriptors can be submitted at a
1340 * time, a wait descriptor will be appended to each submission to ensure
1341 * hardware has completed the invalidation before return. Wait descriptors
1342 * can be part of the submission but it will not be polled for completion.
1343 */
qi_submit_sync(struct intel_iommu * iommu,struct qi_desc * desc,unsigned int count,unsigned long options)1344 int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1345 unsigned int count, unsigned long options)
1346 {
1347 struct q_inval *qi = iommu->qi;
1348 s64 devtlb_start_ktime = 0;
1349 s64 iotlb_start_ktime = 0;
1350 s64 iec_start_ktime = 0;
1351 struct qi_desc wait_desc;
1352 int wait_index, index;
1353 unsigned long flags;
1354 int offset, shift;
1355 int rc, i;
1356 u64 type;
1357
1358 if (!qi)
1359 return 0;
1360
1361 type = desc->qw0 & GENMASK_ULL(3, 0);
1362
1363 if ((type == QI_IOTLB_TYPE || type == QI_EIOTLB_TYPE) &&
1364 dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))
1365 iotlb_start_ktime = ktime_to_ns(ktime_get());
1366
1367 if ((type == QI_DIOTLB_TYPE || type == QI_DEIOTLB_TYPE) &&
1368 dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))
1369 devtlb_start_ktime = ktime_to_ns(ktime_get());
1370
1371 if (type == QI_IEC_TYPE &&
1372 dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))
1373 iec_start_ktime = ktime_to_ns(ktime_get());
1374
1375 restart:
1376 rc = 0;
1377
1378 raw_spin_lock_irqsave(&qi->q_lock, flags);
1379 /*
1380 * Check if we have enough empty slots in the queue to submit,
1381 * the calculation is based on:
1382 * # of desc + 1 wait desc + 1 space between head and tail
1383 */
1384 while (qi->free_cnt < count + 2) {
1385 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1386 cpu_relax();
1387 raw_spin_lock_irqsave(&qi->q_lock, flags);
1388 }
1389
1390 index = qi->free_head;
1391 wait_index = (index + count) % QI_LENGTH;
1392 shift = qi_shift(iommu);
1393
1394 for (i = 0; i < count; i++) {
1395 offset = ((index + i) % QI_LENGTH) << shift;
1396 memcpy(qi->desc + offset, &desc[i], 1 << shift);
1397 qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1398 trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,
1399 desc[i].qw2, desc[i].qw3);
1400 }
1401 qi->desc_status[wait_index] = QI_IN_USE;
1402
1403 wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1404 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1405 if (options & QI_OPT_WAIT_DRAIN)
1406 wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
1407 wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1408 wait_desc.qw2 = 0;
1409 wait_desc.qw3 = 0;
1410
1411 offset = wait_index << shift;
1412 memcpy(qi->desc + offset, &wait_desc, 1 << shift);
1413
1414 qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1415 qi->free_cnt -= count + 1;
1416
1417 /*
1418 * update the HW tail register indicating the presence of
1419 * new descriptors.
1420 */
1421 writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1422
1423 while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) {
1424 /*
1425 * We will leave the interrupts disabled, to prevent interrupt
1426 * context to queue another cmd while a cmd is already submitted
1427 * and waiting for completion on this cpu. This is to avoid
1428 * a deadlock where the interrupt context can wait indefinitely
1429 * for free slots in the queue.
1430 */
1431 rc = qi_check_fault(iommu, index, wait_index);
1432 if (rc)
1433 break;
1434
1435 raw_spin_unlock(&qi->q_lock);
1436 cpu_relax();
1437 raw_spin_lock(&qi->q_lock);
1438 }
1439
1440 /*
1441 * The reclaim code can free descriptors from multiple submissions
1442 * starting from the tail of the queue. When count == 0, the
1443 * status of the standalone wait descriptor at the tail of the queue
1444 * must be set to QI_FREE to allow the reclaim code to proceed.
1445 * It is also possible that descriptors from one of the previous
1446 * submissions has to be reclaimed by a subsequent submission.
1447 */
1448 for (i = 0; i <= count; i++)
1449 qi->desc_status[(index + i) % QI_LENGTH] = QI_FREE;
1450
1451 reclaim_free_desc(qi);
1452 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1453
1454 if (rc == -EAGAIN)
1455 goto restart;
1456
1457 if (iotlb_start_ktime)
1458 dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,
1459 ktime_to_ns(ktime_get()) - iotlb_start_ktime);
1460
1461 if (devtlb_start_ktime)
1462 dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,
1463 ktime_to_ns(ktime_get()) - devtlb_start_ktime);
1464
1465 if (iec_start_ktime)
1466 dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,
1467 ktime_to_ns(ktime_get()) - iec_start_ktime);
1468
1469 return rc;
1470 }
1471
1472 /*
1473 * Flush the global interrupt entry cache.
1474 */
qi_global_iec(struct intel_iommu * iommu)1475 void qi_global_iec(struct intel_iommu *iommu)
1476 {
1477 struct qi_desc desc;
1478
1479 desc.qw0 = QI_IEC_TYPE;
1480 desc.qw1 = 0;
1481 desc.qw2 = 0;
1482 desc.qw3 = 0;
1483
1484 /* should never fail */
1485 qi_submit_sync(iommu, &desc, 1, 0);
1486 }
1487
qi_flush_context(struct intel_iommu * iommu,u16 did,u16 sid,u8 fm,u64 type)1488 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1489 u64 type)
1490 {
1491 struct qi_desc desc;
1492
1493 desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1494 | QI_CC_GRAN(type) | QI_CC_TYPE;
1495 desc.qw1 = 0;
1496 desc.qw2 = 0;
1497 desc.qw3 = 0;
1498
1499 qi_submit_sync(iommu, &desc, 1, 0);
1500 }
1501
qi_flush_iotlb(struct intel_iommu * iommu,u16 did,u64 addr,unsigned int size_order,u64 type)1502 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1503 unsigned int size_order, u64 type)
1504 {
1505 u8 dw = 0, dr = 0;
1506
1507 struct qi_desc desc;
1508 int ih = 0;
1509
1510 if (cap_write_drain(iommu->cap))
1511 dw = 1;
1512
1513 if (cap_read_drain(iommu->cap))
1514 dr = 1;
1515
1516 desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1517 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1518 desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1519 | QI_IOTLB_AM(size_order);
1520 desc.qw2 = 0;
1521 desc.qw3 = 0;
1522
1523 qi_submit_sync(iommu, &desc, 1, 0);
1524 }
1525
qi_flush_dev_iotlb(struct intel_iommu * iommu,u16 sid,u16 pfsid,u16 qdep,u64 addr,unsigned mask)1526 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1527 u16 qdep, u64 addr, unsigned mask)
1528 {
1529 struct qi_desc desc;
1530
1531 /*
1532 * VT-d spec, section 4.3:
1533 *
1534 * Software is recommended to not submit any Device-TLB invalidation
1535 * requests while address remapping hardware is disabled.
1536 */
1537 if (!(iommu->gcmd & DMA_GCMD_TE))
1538 return;
1539
1540 if (mask) {
1541 addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1542 desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1543 } else
1544 desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1545
1546 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1547 qdep = 0;
1548
1549 desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1550 QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1551 desc.qw2 = 0;
1552 desc.qw3 = 0;
1553
1554 qi_submit_sync(iommu, &desc, 1, 0);
1555 }
1556
1557 /* PASID-based IOTLB invalidation */
qi_flush_piotlb(struct intel_iommu * iommu,u16 did,u32 pasid,u64 addr,unsigned long npages,bool ih)1558 void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1559 unsigned long npages, bool ih)
1560 {
1561 struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1562
1563 /*
1564 * npages == -1 means a PASID-selective invalidation, otherwise,
1565 * a positive value for Page-selective-within-PASID invalidation.
1566 * 0 is not a valid input.
1567 */
1568 if (WARN_ON(!npages)) {
1569 pr_err("Invalid input npages = %ld\n", npages);
1570 return;
1571 }
1572
1573 if (npages == -1) {
1574 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1575 QI_EIOTLB_DID(did) |
1576 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
1577 QI_EIOTLB_TYPE;
1578 desc.qw1 = 0;
1579 } else {
1580 int mask = ilog2(__roundup_pow_of_two(npages));
1581 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1582
1583 if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1584 addr = ALIGN_DOWN(addr, align);
1585
1586 desc.qw0 = QI_EIOTLB_PASID(pasid) |
1587 QI_EIOTLB_DID(did) |
1588 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
1589 QI_EIOTLB_TYPE;
1590 desc.qw1 = QI_EIOTLB_ADDR(addr) |
1591 QI_EIOTLB_IH(ih) |
1592 QI_EIOTLB_AM(mask);
1593 }
1594
1595 qi_submit_sync(iommu, &desc, 1, 0);
1596 }
1597
1598 /* PASID-based device IOTLB Invalidate */
qi_flush_dev_iotlb_pasid(struct intel_iommu * iommu,u16 sid,u16 pfsid,u32 pasid,u16 qdep,u64 addr,unsigned int size_order)1599 void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1600 u32 pasid, u16 qdep, u64 addr, unsigned int size_order)
1601 {
1602 unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
1603 struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1604
1605 /*
1606 * VT-d spec, section 4.3:
1607 *
1608 * Software is recommended to not submit any Device-TLB invalidation
1609 * requests while address remapping hardware is disabled.
1610 */
1611 if (!(iommu->gcmd & DMA_GCMD_TE))
1612 return;
1613
1614 desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
1615 QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
1616 QI_DEV_IOTLB_PFSID(pfsid);
1617
1618 /*
1619 * If S bit is 0, we only flush a single page. If S bit is set,
1620 * The least significant zero bit indicates the invalidation address
1621 * range. VT-d spec 6.5.2.6.
1622 * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
1623 * size order = 0 is PAGE_SIZE 4KB
1624 * Max Invs Pending (MIP) is set to 0 for now until we have DIT in
1625 * ECAP.
1626 */
1627 if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
1628 pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
1629 addr, size_order);
1630
1631 /* Take page address */
1632 desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);
1633
1634 if (size_order) {
1635 /*
1636 * Existing 0s in address below size_order may be the least
1637 * significant bit, we must set them to 1s to avoid having
1638 * smaller size than desired.
1639 */
1640 desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
1641 VTD_PAGE_SHIFT);
1642 /* Clear size_order bit to indicate size */
1643 desc.qw1 &= ~mask;
1644 /* Set the S bit to indicate flushing more than 1 page */
1645 desc.qw1 |= QI_DEV_EIOTLB_SIZE;
1646 }
1647
1648 qi_submit_sync(iommu, &desc, 1, 0);
1649 }
1650
qi_flush_pasid_cache(struct intel_iommu * iommu,u16 did,u64 granu,u32 pasid)1651 void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
1652 u64 granu, u32 pasid)
1653 {
1654 struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1655
1656 desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
1657 QI_PC_GRAN(granu) | QI_PC_TYPE;
1658 qi_submit_sync(iommu, &desc, 1, 0);
1659 }
1660
1661 /*
1662 * Disable Queued Invalidation interface.
1663 */
dmar_disable_qi(struct intel_iommu * iommu)1664 void dmar_disable_qi(struct intel_iommu *iommu)
1665 {
1666 unsigned long flags;
1667 u32 sts;
1668 cycles_t start_time = get_cycles();
1669
1670 if (!ecap_qis(iommu->ecap))
1671 return;
1672
1673 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1674
1675 sts = readl(iommu->reg + DMAR_GSTS_REG);
1676 if (!(sts & DMA_GSTS_QIES))
1677 goto end;
1678
1679 /*
1680 * Give a chance to HW to complete the pending invalidation requests.
1681 */
1682 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1683 readl(iommu->reg + DMAR_IQH_REG)) &&
1684 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1685 cpu_relax();
1686
1687 iommu->gcmd &= ~DMA_GCMD_QIE;
1688 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1689
1690 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1691 !(sts & DMA_GSTS_QIES), sts);
1692 end:
1693 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1694 }
1695
1696 /*
1697 * Enable queued invalidation.
1698 */
__dmar_enable_qi(struct intel_iommu * iommu)1699 static void __dmar_enable_qi(struct intel_iommu *iommu)
1700 {
1701 u32 sts;
1702 unsigned long flags;
1703 struct q_inval *qi = iommu->qi;
1704 u64 val = virt_to_phys(qi->desc);
1705
1706 qi->free_head = qi->free_tail = 0;
1707 qi->free_cnt = QI_LENGTH;
1708
1709 /*
1710 * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1711 * is present.
1712 */
1713 if (ecap_smts(iommu->ecap))
1714 val |= BIT_ULL(11) | BIT_ULL(0);
1715
1716 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1717
1718 /* write zero to the tail reg */
1719 writel(0, iommu->reg + DMAR_IQT_REG);
1720
1721 dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1722
1723 iommu->gcmd |= DMA_GCMD_QIE;
1724 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1725
1726 /* Make sure hardware complete it */
1727 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1728
1729 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1730 }
1731
1732 /*
1733 * Enable Queued Invalidation interface. This is a must to support
1734 * interrupt-remapping. Also used by DMA-remapping, which replaces
1735 * register based IOTLB invalidation.
1736 */
dmar_enable_qi(struct intel_iommu * iommu)1737 int dmar_enable_qi(struct intel_iommu *iommu)
1738 {
1739 struct q_inval *qi;
1740 struct page *desc_page;
1741
1742 if (!ecap_qis(iommu->ecap))
1743 return -ENOENT;
1744
1745 /*
1746 * queued invalidation is already setup and enabled.
1747 */
1748 if (iommu->qi)
1749 return 0;
1750
1751 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1752 if (!iommu->qi)
1753 return -ENOMEM;
1754
1755 qi = iommu->qi;
1756
1757 /*
1758 * Need two pages to accommodate 256 descriptors of 256 bits each
1759 * if the remapping hardware supports scalable mode translation.
1760 */
1761 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
1762 !!ecap_smts(iommu->ecap));
1763 if (!desc_page) {
1764 kfree(qi);
1765 iommu->qi = NULL;
1766 return -ENOMEM;
1767 }
1768
1769 qi->desc = page_address(desc_page);
1770
1771 qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1772 if (!qi->desc_status) {
1773 free_page((unsigned long) qi->desc);
1774 kfree(qi);
1775 iommu->qi = NULL;
1776 return -ENOMEM;
1777 }
1778
1779 raw_spin_lock_init(&qi->q_lock);
1780
1781 __dmar_enable_qi(iommu);
1782
1783 return 0;
1784 }
1785
1786 /* iommu interrupt handling. Most stuff are MSI-like. */
1787
1788 enum faulttype {
1789 DMA_REMAP,
1790 INTR_REMAP,
1791 UNKNOWN,
1792 };
1793
1794 static const char *dma_remap_fault_reasons[] =
1795 {
1796 "Software",
1797 "Present bit in root entry is clear",
1798 "Present bit in context entry is clear",
1799 "Invalid context entry",
1800 "Access beyond MGAW",
1801 "PTE Write access is not set",
1802 "PTE Read access is not set",
1803 "Next page table ptr is invalid",
1804 "Root table address invalid",
1805 "Context table ptr is invalid",
1806 "non-zero reserved fields in RTP",
1807 "non-zero reserved fields in CTP",
1808 "non-zero reserved fields in PTE",
1809 "PCE for translation request specifies blocking",
1810 };
1811
1812 static const char * const dma_remap_sm_fault_reasons[] = {
1813 "SM: Invalid Root Table Address",
1814 "SM: TTM 0 for request with PASID",
1815 "SM: TTM 0 for page group request",
1816 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1817 "SM: Error attempting to access Root Entry",
1818 "SM: Present bit in Root Entry is clear",
1819 "SM: Non-zero reserved field set in Root Entry",
1820 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1821 "SM: Error attempting to access Context Entry",
1822 "SM: Present bit in Context Entry is clear",
1823 "SM: Non-zero reserved field set in the Context Entry",
1824 "SM: Invalid Context Entry",
1825 "SM: DTE field in Context Entry is clear",
1826 "SM: PASID Enable field in Context Entry is clear",
1827 "SM: PASID is larger than the max in Context Entry",
1828 "SM: PRE field in Context-Entry is clear",
1829 "SM: RID_PASID field error in Context-Entry",
1830 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1831 "SM: Error attempting to access the PASID Directory Entry",
1832 "SM: Present bit in Directory Entry is clear",
1833 "SM: Non-zero reserved field set in PASID Directory Entry",
1834 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1835 "SM: Error attempting to access PASID Table Entry",
1836 "SM: Present bit in PASID Table Entry is clear",
1837 "SM: Non-zero reserved field set in PASID Table Entry",
1838 "SM: Invalid Scalable-Mode PASID Table Entry",
1839 "SM: ERE field is clear in PASID Table Entry",
1840 "SM: SRE field is clear in PASID Table Entry",
1841 "Unknown", "Unknown",/* 0x5E-0x5F */
1842 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1843 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1844 "SM: Error attempting to access first-level paging entry",
1845 "SM: Present bit in first-level paging entry is clear",
1846 "SM: Non-zero reserved field set in first-level paging entry",
1847 "SM: Error attempting to access FL-PML4 entry",
1848 "SM: First-level entry address beyond MGAW in Nested translation",
1849 "SM: Read permission error in FL-PML4 entry in Nested translation",
1850 "SM: Read permission error in first-level paging entry in Nested translation",
1851 "SM: Write permission error in first-level paging entry in Nested translation",
1852 "SM: Error attempting to access second-level paging entry",
1853 "SM: Read/Write permission error in second-level paging entry",
1854 "SM: Non-zero reserved field set in second-level paging entry",
1855 "SM: Invalid second-level page table pointer",
1856 "SM: A/D bit update needed in second-level entry when set up in no snoop",
1857 "Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1858 "SM: Address in first-level translation is not canonical",
1859 "SM: U/S set 0 for first-level translation with user privilege",
1860 "SM: No execute permission for request with PASID and ER=1",
1861 "SM: Address beyond the DMA hardware max",
1862 "SM: Second-level entry address beyond the max",
1863 "SM: No write permission for Write/AtomicOp request",
1864 "SM: No read permission for Read/AtomicOp request",
1865 "SM: Invalid address-interrupt address",
1866 "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1867 "SM: A/D bit update needed in first-level entry when set up in no snoop",
1868 };
1869
1870 static const char *irq_remap_fault_reasons[] =
1871 {
1872 "Detected reserved fields in the decoded interrupt-remapped request",
1873 "Interrupt index exceeded the interrupt-remapping table size",
1874 "Present field in the IRTE entry is clear",
1875 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1876 "Detected reserved fields in the IRTE entry",
1877 "Blocked a compatibility format interrupt request",
1878 "Blocked an interrupt request due to source-id verification failure",
1879 };
1880
dmar_get_fault_reason(u8 fault_reason,int * fault_type)1881 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1882 {
1883 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1884 ARRAY_SIZE(irq_remap_fault_reasons))) {
1885 *fault_type = INTR_REMAP;
1886 return irq_remap_fault_reasons[fault_reason - 0x20];
1887 } else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1888 ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1889 *fault_type = DMA_REMAP;
1890 return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1891 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1892 *fault_type = DMA_REMAP;
1893 return dma_remap_fault_reasons[fault_reason];
1894 } else {
1895 *fault_type = UNKNOWN;
1896 return "Unknown";
1897 }
1898 }
1899
1900
dmar_msi_reg(struct intel_iommu * iommu,int irq)1901 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1902 {
1903 if (iommu->irq == irq)
1904 return DMAR_FECTL_REG;
1905 else if (iommu->pr_irq == irq)
1906 return DMAR_PECTL_REG;
1907 else if (iommu->perf_irq == irq)
1908 return DMAR_PERFINTRCTL_REG;
1909 else
1910 BUG();
1911 }
1912
dmar_msi_unmask(struct irq_data * data)1913 void dmar_msi_unmask(struct irq_data *data)
1914 {
1915 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1916 int reg = dmar_msi_reg(iommu, data->irq);
1917 unsigned long flag;
1918
1919 /* unmask it */
1920 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1921 writel(0, iommu->reg + reg);
1922 /* Read a reg to force flush the post write */
1923 readl(iommu->reg + reg);
1924 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1925 }
1926
dmar_msi_mask(struct irq_data * data)1927 void dmar_msi_mask(struct irq_data *data)
1928 {
1929 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1930 int reg = dmar_msi_reg(iommu, data->irq);
1931 unsigned long flag;
1932
1933 /* mask it */
1934 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1935 writel(DMA_FECTL_IM, iommu->reg + reg);
1936 /* Read a reg to force flush the post write */
1937 readl(iommu->reg + reg);
1938 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1939 }
1940
dmar_msi_write(int irq,struct msi_msg * msg)1941 void dmar_msi_write(int irq, struct msi_msg *msg)
1942 {
1943 struct intel_iommu *iommu = irq_get_handler_data(irq);
1944 int reg = dmar_msi_reg(iommu, irq);
1945 unsigned long flag;
1946
1947 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1948 writel(msg->data, iommu->reg + reg + 4);
1949 writel(msg->address_lo, iommu->reg + reg + 8);
1950 writel(msg->address_hi, iommu->reg + reg + 12);
1951 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1952 }
1953
dmar_msi_read(int irq,struct msi_msg * msg)1954 void dmar_msi_read(int irq, struct msi_msg *msg)
1955 {
1956 struct intel_iommu *iommu = irq_get_handler_data(irq);
1957 int reg = dmar_msi_reg(iommu, irq);
1958 unsigned long flag;
1959
1960 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1961 msg->data = readl(iommu->reg + reg + 4);
1962 msg->address_lo = readl(iommu->reg + reg + 8);
1963 msg->address_hi = readl(iommu->reg + reg + 12);
1964 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1965 }
1966
dmar_fault_do_one(struct intel_iommu * iommu,int type,u8 fault_reason,u32 pasid,u16 source_id,unsigned long long addr)1967 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1968 u8 fault_reason, u32 pasid, u16 source_id,
1969 unsigned long long addr)
1970 {
1971 const char *reason;
1972 int fault_type;
1973
1974 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1975
1976 if (fault_type == INTR_REMAP) {
1977 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n",
1978 source_id >> 8, PCI_SLOT(source_id & 0xFF),
1979 PCI_FUNC(source_id & 0xFF), addr >> 48,
1980 fault_reason, reason);
1981
1982 return 0;
1983 }
1984
1985 if (pasid == IOMMU_PASID_INVALID)
1986 pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
1987 type ? "DMA Read" : "DMA Write",
1988 source_id >> 8, PCI_SLOT(source_id & 0xFF),
1989 PCI_FUNC(source_id & 0xFF), addr,
1990 fault_reason, reason);
1991 else
1992 pr_err("[%s PASID 0x%x] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
1993 type ? "DMA Read" : "DMA Write", pasid,
1994 source_id >> 8, PCI_SLOT(source_id & 0xFF),
1995 PCI_FUNC(source_id & 0xFF), addr,
1996 fault_reason, reason);
1997
1998 dmar_fault_dump_ptes(iommu, source_id, addr, pasid);
1999
2000 return 0;
2001 }
2002
2003 #define PRIMARY_FAULT_REG_LEN (16)
dmar_fault(int irq,void * dev_id)2004 irqreturn_t dmar_fault(int irq, void *dev_id)
2005 {
2006 struct intel_iommu *iommu = dev_id;
2007 int reg, fault_index;
2008 u32 fault_status;
2009 unsigned long flag;
2010 static DEFINE_RATELIMIT_STATE(rs,
2011 DEFAULT_RATELIMIT_INTERVAL,
2012 DEFAULT_RATELIMIT_BURST);
2013
2014 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2015 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
2016 if (fault_status && __ratelimit(&rs))
2017 pr_err("DRHD: handling fault status reg %x\n", fault_status);
2018
2019 /* TBD: ignore advanced fault log currently */
2020 if (!(fault_status & DMA_FSTS_PPF))
2021 goto unlock_exit;
2022
2023 fault_index = dma_fsts_fault_record_index(fault_status);
2024 reg = cap_fault_reg_offset(iommu->cap);
2025 while (1) {
2026 /* Disable printing, simply clear the fault when ratelimited */
2027 bool ratelimited = !__ratelimit(&rs);
2028 u8 fault_reason;
2029 u16 source_id;
2030 u64 guest_addr;
2031 u32 pasid;
2032 int type;
2033 u32 data;
2034 bool pasid_present;
2035
2036 /* highest 32 bits */
2037 data = readl(iommu->reg + reg +
2038 fault_index * PRIMARY_FAULT_REG_LEN + 12);
2039 if (!(data & DMA_FRCD_F))
2040 break;
2041
2042 if (!ratelimited) {
2043 fault_reason = dma_frcd_fault_reason(data);
2044 type = dma_frcd_type(data);
2045
2046 pasid = dma_frcd_pasid_value(data);
2047 data = readl(iommu->reg + reg +
2048 fault_index * PRIMARY_FAULT_REG_LEN + 8);
2049 source_id = dma_frcd_source_id(data);
2050
2051 pasid_present = dma_frcd_pasid_present(data);
2052 guest_addr = dmar_readq(iommu->reg + reg +
2053 fault_index * PRIMARY_FAULT_REG_LEN);
2054 guest_addr = dma_frcd_page_addr(guest_addr);
2055 }
2056
2057 /* clear the fault */
2058 writel(DMA_FRCD_F, iommu->reg + reg +
2059 fault_index * PRIMARY_FAULT_REG_LEN + 12);
2060
2061 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2062
2063 if (!ratelimited)
2064 /* Using pasid -1 if pasid is not present */
2065 dmar_fault_do_one(iommu, type, fault_reason,
2066 pasid_present ? pasid : IOMMU_PASID_INVALID,
2067 source_id, guest_addr);
2068
2069 fault_index++;
2070 if (fault_index >= cap_num_fault_regs(iommu->cap))
2071 fault_index = 0;
2072 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2073 }
2074
2075 writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
2076 iommu->reg + DMAR_FSTS_REG);
2077
2078 unlock_exit:
2079 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2080 return IRQ_HANDLED;
2081 }
2082
dmar_set_interrupt(struct intel_iommu * iommu)2083 int dmar_set_interrupt(struct intel_iommu *iommu)
2084 {
2085 int irq, ret;
2086
2087 /*
2088 * Check if the fault interrupt is already initialized.
2089 */
2090 if (iommu->irq)
2091 return 0;
2092
2093 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
2094 if (irq > 0) {
2095 iommu->irq = irq;
2096 } else {
2097 pr_err("No free IRQ vectors\n");
2098 return -EINVAL;
2099 }
2100
2101 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
2102 if (ret)
2103 pr_err("Can't request irq\n");
2104 return ret;
2105 }
2106
enable_drhd_fault_handling(void)2107 int __init enable_drhd_fault_handling(void)
2108 {
2109 struct dmar_drhd_unit *drhd;
2110 struct intel_iommu *iommu;
2111
2112 /*
2113 * Enable fault control interrupt.
2114 */
2115 for_each_iommu(iommu, drhd) {
2116 u32 fault_status;
2117 int ret = dmar_set_interrupt(iommu);
2118
2119 if (ret) {
2120 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
2121 (unsigned long long)drhd->reg_base_addr, ret);
2122 return -1;
2123 }
2124
2125 /*
2126 * Clear any previous faults.
2127 */
2128 dmar_fault(iommu->irq, iommu);
2129 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
2130 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
2131 }
2132
2133 return 0;
2134 }
2135
2136 /*
2137 * Re-enable Queued Invalidation interface.
2138 */
dmar_reenable_qi(struct intel_iommu * iommu)2139 int dmar_reenable_qi(struct intel_iommu *iommu)
2140 {
2141 if (!ecap_qis(iommu->ecap))
2142 return -ENOENT;
2143
2144 if (!iommu->qi)
2145 return -ENOENT;
2146
2147 /*
2148 * First disable queued invalidation.
2149 */
2150 dmar_disable_qi(iommu);
2151 /*
2152 * Then enable queued invalidation again. Since there is no pending
2153 * invalidation requests now, it's safe to re-enable queued
2154 * invalidation.
2155 */
2156 __dmar_enable_qi(iommu);
2157
2158 return 0;
2159 }
2160
2161 /*
2162 * Check interrupt remapping support in DMAR table description.
2163 */
dmar_ir_support(void)2164 int __init dmar_ir_support(void)
2165 {
2166 struct acpi_table_dmar *dmar;
2167 dmar = (struct acpi_table_dmar *)dmar_tbl;
2168 if (!dmar)
2169 return 0;
2170 return dmar->flags & 0x1;
2171 }
2172
2173 /* Check whether DMAR units are in use */
dmar_in_use(void)2174 static inline bool dmar_in_use(void)
2175 {
2176 return irq_remapping_enabled || intel_iommu_enabled;
2177 }
2178
dmar_free_unused_resources(void)2179 static int __init dmar_free_unused_resources(void)
2180 {
2181 struct dmar_drhd_unit *dmaru, *dmaru_n;
2182
2183 if (dmar_in_use())
2184 return 0;
2185
2186 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
2187 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
2188
2189 down_write(&dmar_global_lock);
2190 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
2191 list_del(&dmaru->list);
2192 dmar_free_drhd(dmaru);
2193 }
2194 up_write(&dmar_global_lock);
2195
2196 return 0;
2197 }
2198
2199 late_initcall(dmar_free_unused_resources);
2200
2201 /*
2202 * DMAR Hotplug Support
2203 * For more details, please refer to Intel(R) Virtualization Technology
2204 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
2205 * "Remapping Hardware Unit Hot Plug".
2206 */
2207 static guid_t dmar_hp_guid =
2208 GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
2209 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
2210
2211 /*
2212 * Currently there's only one revision and BIOS will not check the revision id,
2213 * so use 0 for safety.
2214 */
2215 #define DMAR_DSM_REV_ID 0
2216 #define DMAR_DSM_FUNC_DRHD 1
2217 #define DMAR_DSM_FUNC_ATSR 2
2218 #define DMAR_DSM_FUNC_RHSA 3
2219 #define DMAR_DSM_FUNC_SATC 4
2220
dmar_detect_dsm(acpi_handle handle,int func)2221 static inline bool dmar_detect_dsm(acpi_handle handle, int func)
2222 {
2223 return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
2224 }
2225
dmar_walk_dsm_resource(acpi_handle handle,int func,dmar_res_handler_t handler,void * arg)2226 static int dmar_walk_dsm_resource(acpi_handle handle, int func,
2227 dmar_res_handler_t handler, void *arg)
2228 {
2229 int ret = -ENODEV;
2230 union acpi_object *obj;
2231 struct acpi_dmar_header *start;
2232 struct dmar_res_callback callback;
2233 static int res_type[] = {
2234 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
2235 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
2236 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
2237 [DMAR_DSM_FUNC_SATC] = ACPI_DMAR_TYPE_SATC,
2238 };
2239
2240 if (!dmar_detect_dsm(handle, func))
2241 return 0;
2242
2243 obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
2244 func, NULL, ACPI_TYPE_BUFFER);
2245 if (!obj)
2246 return -ENODEV;
2247
2248 memset(&callback, 0, sizeof(callback));
2249 callback.cb[res_type[func]] = handler;
2250 callback.arg[res_type[func]] = arg;
2251 start = (struct acpi_dmar_header *)obj->buffer.pointer;
2252 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
2253
2254 ACPI_FREE(obj);
2255
2256 return ret;
2257 }
2258
dmar_hp_add_drhd(struct acpi_dmar_header * header,void * arg)2259 static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
2260 {
2261 int ret;
2262 struct dmar_drhd_unit *dmaru;
2263
2264 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2265 if (!dmaru)
2266 return -ENODEV;
2267
2268 ret = dmar_ir_hotplug(dmaru, true);
2269 if (ret == 0)
2270 ret = dmar_iommu_hotplug(dmaru, true);
2271
2272 return ret;
2273 }
2274
dmar_hp_remove_drhd(struct acpi_dmar_header * header,void * arg)2275 static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
2276 {
2277 int i, ret;
2278 struct device *dev;
2279 struct dmar_drhd_unit *dmaru;
2280
2281 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2282 if (!dmaru)
2283 return 0;
2284
2285 /*
2286 * All PCI devices managed by this unit should have been destroyed.
2287 */
2288 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
2289 for_each_active_dev_scope(dmaru->devices,
2290 dmaru->devices_cnt, i, dev)
2291 return -EBUSY;
2292 }
2293
2294 ret = dmar_ir_hotplug(dmaru, false);
2295 if (ret == 0)
2296 ret = dmar_iommu_hotplug(dmaru, false);
2297
2298 return ret;
2299 }
2300
dmar_hp_release_drhd(struct acpi_dmar_header * header,void * arg)2301 static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2302 {
2303 struct dmar_drhd_unit *dmaru;
2304
2305 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2306 if (dmaru) {
2307 list_del_rcu(&dmaru->list);
2308 synchronize_rcu();
2309 dmar_free_drhd(dmaru);
2310 }
2311
2312 return 0;
2313 }
2314
dmar_hotplug_insert(acpi_handle handle)2315 static int dmar_hotplug_insert(acpi_handle handle)
2316 {
2317 int ret;
2318 int drhd_count = 0;
2319
2320 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2321 &dmar_validate_one_drhd, (void *)1);
2322 if (ret)
2323 goto out;
2324
2325 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2326 &dmar_parse_one_drhd, (void *)&drhd_count);
2327 if (ret == 0 && drhd_count == 0) {
2328 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2329 goto out;
2330 } else if (ret) {
2331 goto release_drhd;
2332 }
2333
2334 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2335 &dmar_parse_one_rhsa, NULL);
2336 if (ret)
2337 goto release_drhd;
2338
2339 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2340 &dmar_parse_one_atsr, NULL);
2341 if (ret)
2342 goto release_atsr;
2343
2344 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2345 &dmar_hp_add_drhd, NULL);
2346 if (!ret)
2347 return 0;
2348
2349 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2350 &dmar_hp_remove_drhd, NULL);
2351 release_atsr:
2352 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2353 &dmar_release_one_atsr, NULL);
2354 release_drhd:
2355 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2356 &dmar_hp_release_drhd, NULL);
2357 out:
2358 return ret;
2359 }
2360
dmar_hotplug_remove(acpi_handle handle)2361 static int dmar_hotplug_remove(acpi_handle handle)
2362 {
2363 int ret;
2364
2365 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2366 &dmar_check_one_atsr, NULL);
2367 if (ret)
2368 return ret;
2369
2370 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2371 &dmar_hp_remove_drhd, NULL);
2372 if (ret == 0) {
2373 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2374 &dmar_release_one_atsr, NULL));
2375 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2376 &dmar_hp_release_drhd, NULL));
2377 } else {
2378 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2379 &dmar_hp_add_drhd, NULL);
2380 }
2381
2382 return ret;
2383 }
2384
dmar_get_dsm_handle(acpi_handle handle,u32 lvl,void * context,void ** retval)2385 static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2386 void *context, void **retval)
2387 {
2388 acpi_handle *phdl = retval;
2389
2390 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2391 *phdl = handle;
2392 return AE_CTRL_TERMINATE;
2393 }
2394
2395 return AE_OK;
2396 }
2397
dmar_device_hotplug(acpi_handle handle,bool insert)2398 static int dmar_device_hotplug(acpi_handle handle, bool insert)
2399 {
2400 int ret;
2401 acpi_handle tmp = NULL;
2402 acpi_status status;
2403
2404 if (!dmar_in_use())
2405 return 0;
2406
2407 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2408 tmp = handle;
2409 } else {
2410 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2411 ACPI_UINT32_MAX,
2412 dmar_get_dsm_handle,
2413 NULL, NULL, &tmp);
2414 if (ACPI_FAILURE(status)) {
2415 pr_warn("Failed to locate _DSM method.\n");
2416 return -ENXIO;
2417 }
2418 }
2419 if (tmp == NULL)
2420 return 0;
2421
2422 down_write(&dmar_global_lock);
2423 if (insert)
2424 ret = dmar_hotplug_insert(tmp);
2425 else
2426 ret = dmar_hotplug_remove(tmp);
2427 up_write(&dmar_global_lock);
2428
2429 return ret;
2430 }
2431
dmar_device_add(acpi_handle handle)2432 int dmar_device_add(acpi_handle handle)
2433 {
2434 return dmar_device_hotplug(handle, true);
2435 }
2436
dmar_device_remove(acpi_handle handle)2437 int dmar_device_remove(acpi_handle handle)
2438 {
2439 return dmar_device_hotplug(handle, false);
2440 }
2441
2442 /*
2443 * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2444 *
2445 * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2446 * the ACPI DMAR table. This means that the platform boot firmware has made
2447 * sure no device can issue DMA outside of RMRR regions.
2448 */
dmar_platform_optin(void)2449 bool dmar_platform_optin(void)
2450 {
2451 struct acpi_table_dmar *dmar;
2452 acpi_status status;
2453 bool ret;
2454
2455 status = acpi_get_table(ACPI_SIG_DMAR, 0,
2456 (struct acpi_table_header **)&dmar);
2457 if (ACPI_FAILURE(status))
2458 return false;
2459
2460 ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2461 acpi_put_table((struct acpi_table_header *)dmar);
2462
2463 return ret;
2464 }
2465 EXPORT_SYMBOL_GPL(dmar_platform_optin);
2466