init.c (b618ae6247bbafe0844355bafd948e59ebd77098) init.c (307959008d80cdfd67fb19a146932582bb68a399)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8#define pr_fmt(fmt) "AMD-Vi: " fmt

--- 538 unchanged lines hidden (view full) ---

547/*
548 * After reading the highest device id from the IOMMU PCI capability header
549 * this function looks if there is a higher device id defined in the ACPI table
550 */
551static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
552{
553 u8 *p = (void *)h, *end = (void *)h;
554 struct ivhd_entry *dev;
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8#define pr_fmt(fmt) "AMD-Vi: " fmt

--- 538 unchanged lines hidden (view full) ---

547/*
548 * After reading the highest device id from the IOMMU PCI capability header
549 * this function looks if there is a higher device id defined in the ACPI table
550 */
551static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
552{
553 u8 *p = (void *)h, *end = (void *)h;
554 struct ivhd_entry *dev;
555 int last_devid = -EINVAL;
555
556 u32 ivhd_size = get_ivhd_header_size(h);
557
558 if (!ivhd_size) {
559 pr_err("Unsupported IVHD type %#x\n", h->type);
560 return -EINVAL;
561 }
562
563 p += ivhd_size;
564 end += h->length;
565
566 while (p < end) {
567 dev = (struct ivhd_entry *)p;
568 switch (dev->type) {
569 case IVHD_DEV_ALL:
570 /* Use maximum BDF value for DEV_ALL */
571 update_last_devid(0xffff);
556
557 u32 ivhd_size = get_ivhd_header_size(h);
558
559 if (!ivhd_size) {
560 pr_err("Unsupported IVHD type %#x\n", h->type);
561 return -EINVAL;
562 }
563
564 p += ivhd_size;
565 end += h->length;
566
567 while (p < end) {
568 dev = (struct ivhd_entry *)p;
569 switch (dev->type) {
570 case IVHD_DEV_ALL:
571 /* Use maximum BDF value for DEV_ALL */
572 update_last_devid(0xffff);
572 break;
573 return 0xffff;
573 case IVHD_DEV_SELECT:
574 case IVHD_DEV_RANGE_END:
575 case IVHD_DEV_ALIAS:
576 case IVHD_DEV_EXT_SELECT:
577 /* all the above subfield types refer to device ids */
578 update_last_devid(dev->devid);
574 case IVHD_DEV_SELECT:
575 case IVHD_DEV_RANGE_END:
576 case IVHD_DEV_ALIAS:
577 case IVHD_DEV_EXT_SELECT:
578 /* all the above subfield types refer to device ids */
579 update_last_devid(dev->devid);
580 if (dev->devid > last_devid)
581 last_devid = dev->devid;
579 break;
580 default:
581 break;
582 }
583 p += ivhd_entry_length(p);
584 }
585
586 WARN_ON(p != end);
587
582 break;
583 default:
584 break;
585 }
586 p += ivhd_entry_length(p);
587 }
588
589 WARN_ON(p != end);
590
588 return 0;
591 return last_devid;
589}
590
591static int __init check_ivrs_checksum(struct acpi_table_header *table)
592{
593 int i;
594 u8 checksum = 0, *p = (u8 *)table;
595
596 for (i = 0; i < table->length; ++i)

--- 7 unchanged lines hidden (view full) ---

604 return 0;
605}
606
607/*
608 * Iterate over all IVHD entries in the ACPI table and find the highest device
609 * id which we need to handle. This is the first of three functions which parse
610 * the ACPI table. So we check the checksum here.
611 */
592}
593
594static int __init check_ivrs_checksum(struct acpi_table_header *table)
595{
596 int i;
597 u8 checksum = 0, *p = (u8 *)table;
598
599 for (i = 0; i < table->length; ++i)

--- 7 unchanged lines hidden (view full) ---

607 return 0;
608}
609
610/*
611 * Iterate over all IVHD entries in the ACPI table and find the highest device
612 * id which we need to handle. This is the first of three functions which parse
613 * the ACPI table. So we check the checksum here.
614 */
612static int __init find_last_devid_acpi(struct acpi_table_header *table)
615static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)
613{
614 u8 *p = (u8 *)table, *end = (u8 *)table;
615 struct ivhd_header *h;
616{
617 u8 *p = (u8 *)table, *end = (u8 *)table;
618 struct ivhd_header *h;
619 int last_devid, last_bdf = 0;
616
617 p += IVRS_HEADER_LENGTH;
618
619 end += table->length;
620 while (p < end) {
621 h = (struct ivhd_header *)p;
620
621 p += IVRS_HEADER_LENGTH;
622
623 end += table->length;
624 while (p < end) {
625 h = (struct ivhd_header *)p;
622 if (h->type == amd_iommu_target_ivhd_type) {
623 int ret = find_last_devid_from_ivhd(h);
626 if (h->pci_seg == pci_seg &&
627 h->type == amd_iommu_target_ivhd_type) {
628 last_devid = find_last_devid_from_ivhd(h);
624
629
625 if (ret)
626 return ret;
630 if (last_devid < 0)
631 return -EINVAL;
632 if (last_devid > last_bdf)
633 last_bdf = last_devid;
627 }
628 p += h->length;
629 }
630 WARN_ON(p != end);
631
634 }
635 p += h->length;
636 }
637 WARN_ON(p != end);
638
632 return 0;
639 return last_bdf;
633}
634
635/****************************************************************************
636 *
637 * The following functions belong to the code path which parses the ACPI table
638 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
639 * data structures, initialize the per PCI segment device/alias/rlookup table
640 * and also basically initialize the hardware.

--- 907 unchanged lines hidden (view full) ---

1548
1549 p += ivhd_entry_length(p);
1550 }
1551
1552 return 0;
1553}
1554
1555/* Allocate PCI segment data structure */
640}
641
642/****************************************************************************
643 *
644 * The following functions belong to the code path which parses the ACPI table
645 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
646 * data structures, initialize the per PCI segment device/alias/rlookup table
647 * and also basically initialize the hardware.

--- 907 unchanged lines hidden (view full) ---

1555
1556 p += ivhd_entry_length(p);
1557 }
1558
1559 return 0;
1560}
1561
1562/* Allocate PCI segment data structure */
1556static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id)
1563static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
1564 struct acpi_table_header *ivrs_base)
1557{
1558 struct amd_iommu_pci_seg *pci_seg;
1565{
1566 struct amd_iommu_pci_seg *pci_seg;
1567 int last_bdf;
1559
1568
1569 /*
1570 * First parse ACPI tables to find the largest Bus/Dev/Func we need to
1571 * handle in this PCI segment. Upon this information the shared data
1572 * structures for the PCI segments in the system will be allocated.
1573 */
1574 last_bdf = find_last_devid_acpi(ivrs_base, id);
1575 if (last_bdf < 0)
1576 return NULL;
1577
1560 pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);
1561 if (pci_seg == NULL)
1562 return NULL;
1563
1578 pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);
1579 if (pci_seg == NULL)
1580 return NULL;
1581
1582 pci_seg->last_bdf = last_bdf;
1583 DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf);
1584
1564 pci_seg->id = id;
1565 init_llist_head(&pci_seg->dev_data_list);
1566 INIT_LIST_HEAD(&pci_seg->unity_map);
1567 list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1568
1569 if (alloc_dev_table(pci_seg))
1570 return NULL;
1571 if (alloc_alias_table(pci_seg))
1572 return NULL;
1573 if (alloc_rlookup_table(pci_seg))
1574 return NULL;
1575
1576 return pci_seg;
1577}
1578
1585 pci_seg->id = id;
1586 init_llist_head(&pci_seg->dev_data_list);
1587 INIT_LIST_HEAD(&pci_seg->unity_map);
1588 list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1589
1590 if (alloc_dev_table(pci_seg))
1591 return NULL;
1592 if (alloc_alias_table(pci_seg))
1593 return NULL;
1594 if (alloc_rlookup_table(pci_seg))
1595 return NULL;
1596
1597 return pci_seg;
1598}
1599
1579static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id)
1600static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id,
1601 struct acpi_table_header *ivrs_base)
1580{
1581 struct amd_iommu_pci_seg *pci_seg;
1582
1583 for_each_pci_segment(pci_seg) {
1584 if (pci_seg->id == id)
1585 return pci_seg;
1586 }
1587
1602{
1603 struct amd_iommu_pci_seg *pci_seg;
1604
1605 for_each_pci_segment(pci_seg) {
1606 if (pci_seg->id == id)
1607 return pci_seg;
1608 }
1609
1588 return alloc_pci_segment(id);
1610 return alloc_pci_segment(id, ivrs_base);
1589}
1590
1591static void __init free_pci_segments(void)
1592{
1593 struct amd_iommu_pci_seg *pci_seg, *next;
1594
1595 for_each_pci_segment_safe(pci_seg, next) {
1596 list_del(&pci_seg->list);

--- 84 unchanged lines hidden (view full) ---

1681 pci_info(iommu->dev, "Applying ATS write check workaround\n");
1682}
1683
1684/*
1685 * This function glues the initialization function for one IOMMU
1686 * together and also allocates the command buffer and programs the
1687 * hardware. It does NOT enable the IOMMU. This is done afterwards.
1688 */
1611}
1612
1613static void __init free_pci_segments(void)
1614{
1615 struct amd_iommu_pci_seg *pci_seg, *next;
1616
1617 for_each_pci_segment_safe(pci_seg, next) {
1618 list_del(&pci_seg->list);

--- 84 unchanged lines hidden (view full) ---

1703 pci_info(iommu->dev, "Applying ATS write check workaround\n");
1704}
1705
1706/*
1707 * This function glues the initialization function for one IOMMU
1708 * together and also allocates the command buffer and programs the
1709 * hardware. It does NOT enable the IOMMU. This is done afterwards.
1710 */
1689static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
1711static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
1712 struct acpi_table_header *ivrs_base)
1690{
1691 struct amd_iommu_pci_seg *pci_seg;
1692 int ret;
1693
1713{
1714 struct amd_iommu_pci_seg *pci_seg;
1715 int ret;
1716
1694 pci_seg = get_pci_segment(h->pci_seg);
1717 pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
1695 if (pci_seg == NULL)
1696 return -ENOMEM;
1697 iommu->pci_seg = pci_seg;
1698
1699 raw_spin_lock_init(&iommu->lock);
1700 iommu->cmd_sem_val = 0;
1701
1702 /* Add IOMMU to internal data structures */

--- 158 unchanged lines hidden (view full) ---

1861 h->pci_seg, h->flags, h->info);
1862 DUMP_printk(" mmio-addr: %016llx\n",
1863 h->mmio_phys);
1864
1865 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1866 if (iommu == NULL)
1867 return -ENOMEM;
1868
1718 if (pci_seg == NULL)
1719 return -ENOMEM;
1720 iommu->pci_seg = pci_seg;
1721
1722 raw_spin_lock_init(&iommu->lock);
1723 iommu->cmd_sem_val = 0;
1724
1725 /* Add IOMMU to internal data structures */

--- 158 unchanged lines hidden (view full) ---

1884 h->pci_seg, h->flags, h->info);
1885 DUMP_printk(" mmio-addr: %016llx\n",
1886 h->mmio_phys);
1887
1888 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1889 if (iommu == NULL)
1890 return -ENOMEM;
1891
1869 ret = init_iommu_one(iommu, h);
1892 ret = init_iommu_one(iommu, h, table);
1870 if (ret)
1871 return ret;
1872 }
1873 p += h->length;
1874
1875 }
1876 WARN_ON(p != end);
1877

--- 529 unchanged lines hidden (view full) ---

2407 list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2408 list_del(&entry->list);
2409 kfree(entry);
2410 }
2411 }
2412}
2413
2414/* called for unity map ACPI definition */
1893 if (ret)
1894 return ret;
1895 }
1896 p += h->length;
1897
1898 }
1899 WARN_ON(p != end);
1900

--- 529 unchanged lines hidden (view full) ---

2430 list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2431 list_del(&entry->list);
2432 kfree(entry);
2433 }
2434 }
2435}
2436
2437/* called for unity map ACPI definition */
2415static int __init init_unity_map_range(struct ivmd_header *m)
2438static int __init init_unity_map_range(struct ivmd_header *m,
2439 struct acpi_table_header *ivrs_base)
2416{
2417 struct unity_map_entry *e = NULL;
2418 struct amd_iommu_pci_seg *pci_seg;
2419 char *s;
2420
2440{
2441 struct unity_map_entry *e = NULL;
2442 struct amd_iommu_pci_seg *pci_seg;
2443 char *s;
2444
2421 pci_seg = get_pci_segment(m->pci_seg);
2445 pci_seg = get_pci_segment(m->pci_seg, ivrs_base);
2422 if (pci_seg == NULL)
2423 return -ENOMEM;
2424
2425 e = kzalloc(sizeof(*e), GFP_KERNEL);
2426 if (e == NULL)
2427 return -ENOMEM;
2428
2429 switch (m->type) {

--- 50 unchanged lines hidden (view full) ---

2480 struct ivmd_header *m;
2481
2482 end += table->length;
2483 p += IVRS_HEADER_LENGTH;
2484
2485 while (p < end) {
2486 m = (struct ivmd_header *)p;
2487 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2446 if (pci_seg == NULL)
2447 return -ENOMEM;
2448
2449 e = kzalloc(sizeof(*e), GFP_KERNEL);
2450 if (e == NULL)
2451 return -ENOMEM;
2452
2453 switch (m->type) {

--- 50 unchanged lines hidden (view full) ---

2504 struct ivmd_header *m;
2505
2506 end += table->length;
2507 p += IVRS_HEADER_LENGTH;
2508
2509 while (p < end) {
2510 m = (struct ivmd_header *)p;
2511 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2488 init_unity_map_range(m);
2512 init_unity_map_range(m, table);
2489
2490 p += m->length;
2491 }
2492
2493 return 0;
2494}
2495
2496/*

--- 407 unchanged lines hidden (view full) ---

2904 if (ret)
2905 goto out;
2906
2907 ivinfo_init(ivrs_base);
2908
2909 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
2910 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
2911
2513
2514 p += m->length;
2515 }
2516
2517 return 0;
2518}
2519
2520/*

--- 407 unchanged lines hidden (view full) ---

2928 if (ret)
2929 goto out;
2930
2931 ivinfo_init(ivrs_base);
2932
2933 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
2934 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
2935
2912 /*
2913 * First parse ACPI tables to find the largest Bus/Dev/Func
2914 * we need to handle. Upon this information the shared data
2915 * structures for the IOMMUs in the system will be allocated
2916 */
2917 ret = find_last_devid_acpi(ivrs_base);
2918 if (ret)
2919 goto out;
2920
2921 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
2922 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
2923 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
2924
2925 /* Device table - directly used by all IOMMUs */
2926 ret = -ENOMEM;
2927 amd_iommu_dev_table = (void *)__get_free_pages(
2928 GFP_KERNEL | __GFP_ZERO | GFP_DMA32,

--- 609 unchanged lines hidden --- 		2936 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
2937 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
2938 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
2939
2940 /* Device table - directly used by all IOMMUs */
2941 ret = -ENOMEM;
2942 amd_iommu_dev_table = (void *)__get_free_pages(
2943 GFP_KERNEL | __GFP_ZERO | GFP_DMA32,

--- 609 unchanged lines hidden ---