1191c2287SAlexey Kardashevskiy // SPDX-License-Identifier: GPL-2.0+ 2191c2287SAlexey Kardashevskiy /* 3191c2287SAlexey Kardashevskiy * TCE helpers for IODA PCI/PCIe on PowerNV platforms 4191c2287SAlexey Kardashevskiy * 5191c2287SAlexey Kardashevskiy * Copyright 2018 IBM Corp. 6191c2287SAlexey Kardashevskiy * 7191c2287SAlexey Kardashevskiy * This program is free software; you can redistribute it and/or 8191c2287SAlexey Kardashevskiy * modify it under the terms of the GNU General Public License 9191c2287SAlexey Kardashevskiy * as published by the Free Software Foundation; either version 10191c2287SAlexey Kardashevskiy * 2 of the License, or (at your option) any later version. 11191c2287SAlexey Kardashevskiy */ 12191c2287SAlexey Kardashevskiy 13191c2287SAlexey Kardashevskiy #include <linux/kernel.h> 14191c2287SAlexey Kardashevskiy #include <linux/iommu.h> 15191c2287SAlexey Kardashevskiy 16191c2287SAlexey Kardashevskiy #include <asm/iommu.h> 17191c2287SAlexey Kardashevskiy #include <asm/tce.h> 18191c2287SAlexey Kardashevskiy #include "pci.h" 19191c2287SAlexey Kardashevskiy 20191c2287SAlexey Kardashevskiy void pnv_pci_setup_iommu_table(struct iommu_table *tbl, 21191c2287SAlexey Kardashevskiy void *tce_mem, u64 tce_size, 22191c2287SAlexey Kardashevskiy u64 dma_offset, unsigned int page_shift) 23191c2287SAlexey Kardashevskiy { 24191c2287SAlexey Kardashevskiy tbl->it_blocksize = 16; 25191c2287SAlexey Kardashevskiy tbl->it_base = (unsigned long)tce_mem; 26191c2287SAlexey Kardashevskiy tbl->it_page_shift = page_shift; 27191c2287SAlexey Kardashevskiy tbl->it_offset = dma_offset >> tbl->it_page_shift; 28191c2287SAlexey Kardashevskiy tbl->it_index = 0; 29191c2287SAlexey Kardashevskiy tbl->it_size = tce_size >> 3; 30191c2287SAlexey Kardashevskiy tbl->it_busno = 0; 31191c2287SAlexey Kardashevskiy tbl->it_type = TCE_PCI; 32191c2287SAlexey Kardashevskiy } 33191c2287SAlexey Kardashevskiy 34090bad39SAlexey Kardashevskiy static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx) 35191c2287SAlexey Kardashevskiy { 36090bad39SAlexey Kardashevskiy __be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base; 37191c2287SAlexey Kardashevskiy int level = tbl->it_indirect_levels; 38191c2287SAlexey Kardashevskiy const long shift = ilog2(tbl->it_level_size); 39191c2287SAlexey Kardashevskiy unsigned long mask = (tbl->it_level_size - 1) << (level * shift); 40191c2287SAlexey Kardashevskiy 41191c2287SAlexey Kardashevskiy while (level) { 42191c2287SAlexey Kardashevskiy int n = (idx & mask) >> (level * shift); 43191c2287SAlexey Kardashevskiy unsigned long tce = be64_to_cpu(tmp[n]); 44191c2287SAlexey Kardashevskiy 45191c2287SAlexey Kardashevskiy tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE)); 46191c2287SAlexey Kardashevskiy idx &= ~mask; 47191c2287SAlexey Kardashevskiy mask >>= shift; 48191c2287SAlexey Kardashevskiy --level; 49191c2287SAlexey Kardashevskiy } 50191c2287SAlexey Kardashevskiy 51191c2287SAlexey Kardashevskiy return tmp + idx; 52191c2287SAlexey Kardashevskiy } 53191c2287SAlexey Kardashevskiy 54191c2287SAlexey Kardashevskiy int pnv_tce_build(struct iommu_table *tbl, long index, long npages, 55191c2287SAlexey Kardashevskiy unsigned long uaddr, enum dma_data_direction direction, 56191c2287SAlexey Kardashevskiy unsigned long attrs) 57191c2287SAlexey Kardashevskiy { 58191c2287SAlexey Kardashevskiy u64 proto_tce = iommu_direction_to_tce_perm(direction); 59191c2287SAlexey Kardashevskiy u64 rpn = __pa(uaddr) >> tbl->it_page_shift; 60191c2287SAlexey Kardashevskiy long i; 61191c2287SAlexey Kardashevskiy 62191c2287SAlexey Kardashevskiy if (proto_tce & TCE_PCI_WRITE) 63191c2287SAlexey Kardashevskiy proto_tce |= TCE_PCI_READ; 64191c2287SAlexey Kardashevskiy 65191c2287SAlexey Kardashevskiy for (i = 0; i < npages; i++) { 66191c2287SAlexey Kardashevskiy unsigned long newtce = proto_tce | 67191c2287SAlexey Kardashevskiy ((rpn + i) << tbl->it_page_shift); 68191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset + i; 69191c2287SAlexey Kardashevskiy 70090bad39SAlexey Kardashevskiy *(pnv_tce(tbl, false, idx)) = cpu_to_be64(newtce); 71191c2287SAlexey Kardashevskiy } 72191c2287SAlexey Kardashevskiy 73191c2287SAlexey Kardashevskiy return 0; 74191c2287SAlexey Kardashevskiy } 75191c2287SAlexey Kardashevskiy 76191c2287SAlexey Kardashevskiy #ifdef CONFIG_IOMMU_API 77191c2287SAlexey Kardashevskiy int pnv_tce_xchg(struct iommu_table *tbl, long index, 78191c2287SAlexey Kardashevskiy unsigned long *hpa, enum dma_data_direction *direction) 79191c2287SAlexey Kardashevskiy { 80191c2287SAlexey Kardashevskiy u64 proto_tce = iommu_direction_to_tce_perm(*direction); 81191c2287SAlexey Kardashevskiy unsigned long newtce = *hpa | proto_tce, oldtce; 82191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset; 83191c2287SAlexey Kardashevskiy 84191c2287SAlexey Kardashevskiy BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)); 85191c2287SAlexey Kardashevskiy 86191c2287SAlexey Kardashevskiy if (newtce & TCE_PCI_WRITE) 87191c2287SAlexey Kardashevskiy newtce |= TCE_PCI_READ; 88191c2287SAlexey Kardashevskiy 89090bad39SAlexey Kardashevskiy oldtce = be64_to_cpu(xchg(pnv_tce(tbl, false, idx), 90090bad39SAlexey Kardashevskiy cpu_to_be64(newtce))); 91191c2287SAlexey Kardashevskiy *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); 92191c2287SAlexey Kardashevskiy *direction = iommu_tce_direction(oldtce); 93191c2287SAlexey Kardashevskiy 94191c2287SAlexey Kardashevskiy return 0; 95191c2287SAlexey Kardashevskiy } 96090bad39SAlexey Kardashevskiy 97090bad39SAlexey Kardashevskiy __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index) 98090bad39SAlexey Kardashevskiy { 99090bad39SAlexey Kardashevskiy if (WARN_ON_ONCE(!tbl->it_userspace)) 100090bad39SAlexey Kardashevskiy return NULL; 101090bad39SAlexey Kardashevskiy 102090bad39SAlexey Kardashevskiy return pnv_tce(tbl, true, index - tbl->it_offset); 103090bad39SAlexey Kardashevskiy } 104191c2287SAlexey Kardashevskiy #endif 105191c2287SAlexey Kardashevskiy 106191c2287SAlexey Kardashevskiy void pnv_tce_free(struct iommu_table *tbl, long index, long npages) 107191c2287SAlexey Kardashevskiy { 108191c2287SAlexey Kardashevskiy long i; 109191c2287SAlexey Kardashevskiy 110191c2287SAlexey Kardashevskiy for (i = 0; i < npages; i++) { 111191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset + i; 112191c2287SAlexey Kardashevskiy 113090bad39SAlexey Kardashevskiy *(pnv_tce(tbl, false, idx)) = cpu_to_be64(0); 114191c2287SAlexey Kardashevskiy } 115191c2287SAlexey Kardashevskiy } 116191c2287SAlexey Kardashevskiy 117191c2287SAlexey Kardashevskiy unsigned long pnv_tce_get(struct iommu_table *tbl, long index) 118191c2287SAlexey Kardashevskiy { 119090bad39SAlexey Kardashevskiy __be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset); 120090bad39SAlexey Kardashevskiy 121090bad39SAlexey Kardashevskiy return be64_to_cpu(*ptce); 122191c2287SAlexey Kardashevskiy } 123191c2287SAlexey Kardashevskiy 124191c2287SAlexey Kardashevskiy static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, 125191c2287SAlexey Kardashevskiy unsigned long size, unsigned int levels) 126191c2287SAlexey Kardashevskiy { 127191c2287SAlexey Kardashevskiy const unsigned long addr_ul = (unsigned long) addr & 128191c2287SAlexey Kardashevskiy ~(TCE_PCI_READ | TCE_PCI_WRITE); 129191c2287SAlexey Kardashevskiy 130191c2287SAlexey Kardashevskiy if (levels) { 131191c2287SAlexey Kardashevskiy long i; 132191c2287SAlexey Kardashevskiy u64 *tmp = (u64 *) addr_ul; 133191c2287SAlexey Kardashevskiy 134191c2287SAlexey Kardashevskiy for (i = 0; i < size; ++i) { 135191c2287SAlexey Kardashevskiy unsigned long hpa = be64_to_cpu(tmp[i]); 136191c2287SAlexey Kardashevskiy 137191c2287SAlexey Kardashevskiy if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE))) 138191c2287SAlexey Kardashevskiy continue; 139191c2287SAlexey Kardashevskiy 140191c2287SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(__va(hpa), size, 141191c2287SAlexey Kardashevskiy levels - 1); 142191c2287SAlexey Kardashevskiy } 143191c2287SAlexey Kardashevskiy } 144191c2287SAlexey Kardashevskiy 145191c2287SAlexey Kardashevskiy free_pages(addr_ul, get_order(size << 3)); 146191c2287SAlexey Kardashevskiy } 147191c2287SAlexey Kardashevskiy 148191c2287SAlexey Kardashevskiy void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl) 149191c2287SAlexey Kardashevskiy { 150191c2287SAlexey Kardashevskiy const unsigned long size = tbl->it_indirect_levels ? 151191c2287SAlexey Kardashevskiy tbl->it_level_size : tbl->it_size; 152191c2287SAlexey Kardashevskiy 153191c2287SAlexey Kardashevskiy if (!tbl->it_size) 154191c2287SAlexey Kardashevskiy return; 155191c2287SAlexey Kardashevskiy 156191c2287SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size, 157191c2287SAlexey Kardashevskiy tbl->it_indirect_levels); 158090bad39SAlexey Kardashevskiy if (tbl->it_userspace) { 159090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size, 160090bad39SAlexey Kardashevskiy tbl->it_indirect_levels); 161090bad39SAlexey Kardashevskiy } 162191c2287SAlexey Kardashevskiy } 163191c2287SAlexey Kardashevskiy 164191c2287SAlexey Kardashevskiy static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift, 165191c2287SAlexey Kardashevskiy unsigned int levels, unsigned long limit, 166191c2287SAlexey Kardashevskiy unsigned long *current_offset, unsigned long *total_allocated) 167191c2287SAlexey Kardashevskiy { 168191c2287SAlexey Kardashevskiy struct page *tce_mem = NULL; 169191c2287SAlexey Kardashevskiy __be64 *addr, *tmp; 170191c2287SAlexey Kardashevskiy unsigned int order = max_t(unsigned int, shift, PAGE_SHIFT) - 171191c2287SAlexey Kardashevskiy PAGE_SHIFT; 172191c2287SAlexey Kardashevskiy unsigned long allocated = 1UL << (order + PAGE_SHIFT); 173191c2287SAlexey Kardashevskiy unsigned int entries = 1UL << (shift - 3); 174191c2287SAlexey Kardashevskiy long i; 175191c2287SAlexey Kardashevskiy 176191c2287SAlexey Kardashevskiy tce_mem = alloc_pages_node(nid, GFP_KERNEL, order); 177191c2287SAlexey Kardashevskiy if (!tce_mem) { 178191c2287SAlexey Kardashevskiy pr_err("Failed to allocate a TCE memory, order=%d\n", order); 179191c2287SAlexey Kardashevskiy return NULL; 180191c2287SAlexey Kardashevskiy } 181191c2287SAlexey Kardashevskiy addr = page_address(tce_mem); 182191c2287SAlexey Kardashevskiy memset(addr, 0, allocated); 183191c2287SAlexey Kardashevskiy *total_allocated += allocated; 184191c2287SAlexey Kardashevskiy 185191c2287SAlexey Kardashevskiy --levels; 186191c2287SAlexey Kardashevskiy if (!levels) { 187191c2287SAlexey Kardashevskiy *current_offset += allocated; 188191c2287SAlexey Kardashevskiy return addr; 189191c2287SAlexey Kardashevskiy } 190191c2287SAlexey Kardashevskiy 191191c2287SAlexey Kardashevskiy for (i = 0; i < entries; ++i) { 192191c2287SAlexey Kardashevskiy tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift, 193191c2287SAlexey Kardashevskiy levels, limit, current_offset, total_allocated); 194191c2287SAlexey Kardashevskiy if (!tmp) 195191c2287SAlexey Kardashevskiy break; 196191c2287SAlexey Kardashevskiy 197191c2287SAlexey Kardashevskiy addr[i] = cpu_to_be64(__pa(tmp) | 198191c2287SAlexey Kardashevskiy TCE_PCI_READ | TCE_PCI_WRITE); 199191c2287SAlexey Kardashevskiy 200191c2287SAlexey Kardashevskiy if (*current_offset >= limit) 201191c2287SAlexey Kardashevskiy break; 202191c2287SAlexey Kardashevskiy } 203191c2287SAlexey Kardashevskiy 204191c2287SAlexey Kardashevskiy return addr; 205191c2287SAlexey Kardashevskiy } 206191c2287SAlexey Kardashevskiy 207191c2287SAlexey Kardashevskiy long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, 208191c2287SAlexey Kardashevskiy __u32 page_shift, __u64 window_size, __u32 levels, 209090bad39SAlexey Kardashevskiy bool alloc_userspace_copy, struct iommu_table *tbl) 210191c2287SAlexey Kardashevskiy { 211090bad39SAlexey Kardashevskiy void *addr, *uas = NULL; 212191c2287SAlexey Kardashevskiy unsigned long offset = 0, level_shift, total_allocated = 0; 213090bad39SAlexey Kardashevskiy unsigned long total_allocated_uas = 0; 214191c2287SAlexey Kardashevskiy const unsigned int window_shift = ilog2(window_size); 215191c2287SAlexey Kardashevskiy unsigned int entries_shift = window_shift - page_shift; 216191c2287SAlexey Kardashevskiy unsigned int table_shift = max_t(unsigned int, entries_shift + 3, 217191c2287SAlexey Kardashevskiy PAGE_SHIFT); 218191c2287SAlexey Kardashevskiy const unsigned long tce_table_size = 1UL << table_shift; 219191c2287SAlexey Kardashevskiy 220191c2287SAlexey Kardashevskiy if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS)) 221191c2287SAlexey Kardashevskiy return -EINVAL; 222191c2287SAlexey Kardashevskiy 223191c2287SAlexey Kardashevskiy if (!is_power_of_2(window_size)) 224191c2287SAlexey Kardashevskiy return -EINVAL; 225191c2287SAlexey Kardashevskiy 226191c2287SAlexey Kardashevskiy /* Adjust direct table size from window_size and levels */ 227191c2287SAlexey Kardashevskiy entries_shift = (entries_shift + levels - 1) / levels; 228191c2287SAlexey Kardashevskiy level_shift = entries_shift + 3; 229191c2287SAlexey Kardashevskiy level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT); 230191c2287SAlexey Kardashevskiy 231191c2287SAlexey Kardashevskiy if ((level_shift - 3) * levels + page_shift >= 60) 232191c2287SAlexey Kardashevskiy return -EINVAL; 233191c2287SAlexey Kardashevskiy 234191c2287SAlexey Kardashevskiy /* Allocate TCE table */ 235191c2287SAlexey Kardashevskiy addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 236191c2287SAlexey Kardashevskiy levels, tce_table_size, &offset, &total_allocated); 237191c2287SAlexey Kardashevskiy 238191c2287SAlexey Kardashevskiy /* addr==NULL means that the first level allocation failed */ 239191c2287SAlexey Kardashevskiy if (!addr) 240191c2287SAlexey Kardashevskiy return -ENOMEM; 241191c2287SAlexey Kardashevskiy 242191c2287SAlexey Kardashevskiy /* 243191c2287SAlexey Kardashevskiy * First level was allocated but some lower level failed as 244191c2287SAlexey Kardashevskiy * we did not allocate as much as we wanted, 245191c2287SAlexey Kardashevskiy * release partially allocated table. 246191c2287SAlexey Kardashevskiy */ 247090bad39SAlexey Kardashevskiy if (offset < tce_table_size) 248090bad39SAlexey Kardashevskiy goto free_tces_exit; 249090bad39SAlexey Kardashevskiy 250090bad39SAlexey Kardashevskiy /* Allocate userspace view of the TCE table */ 251090bad39SAlexey Kardashevskiy if (alloc_userspace_copy) { 252090bad39SAlexey Kardashevskiy offset = 0; 253090bad39SAlexey Kardashevskiy uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 254090bad39SAlexey Kardashevskiy levels, tce_table_size, &offset, 255090bad39SAlexey Kardashevskiy &total_allocated_uas); 256090bad39SAlexey Kardashevskiy if (!uas) 257090bad39SAlexey Kardashevskiy goto free_tces_exit; 258090bad39SAlexey Kardashevskiy if (offset < tce_table_size || 259090bad39SAlexey Kardashevskiy total_allocated_uas != total_allocated) 260090bad39SAlexey Kardashevskiy goto free_uas_exit; 261191c2287SAlexey Kardashevskiy } 262191c2287SAlexey Kardashevskiy 263191c2287SAlexey Kardashevskiy /* Setup linux iommu table */ 264191c2287SAlexey Kardashevskiy pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset, 265191c2287SAlexey Kardashevskiy page_shift); 266191c2287SAlexey Kardashevskiy tbl->it_level_size = 1ULL << (level_shift - 3); 267191c2287SAlexey Kardashevskiy tbl->it_indirect_levels = levels - 1; 268191c2287SAlexey Kardashevskiy tbl->it_allocated_size = total_allocated; 269090bad39SAlexey Kardashevskiy tbl->it_userspace = uas; 270191c2287SAlexey Kardashevskiy 271090bad39SAlexey Kardashevskiy pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d\n", 272090bad39SAlexey Kardashevskiy window_size, tce_table_size, bus_offset, tbl->it_base, 273090bad39SAlexey Kardashevskiy tbl->it_userspace, levels); 274191c2287SAlexey Kardashevskiy 275191c2287SAlexey Kardashevskiy return 0; 276090bad39SAlexey Kardashevskiy 277090bad39SAlexey Kardashevskiy free_uas_exit: 278090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(uas, 279090bad39SAlexey Kardashevskiy 1ULL << (level_shift - 3), levels - 1); 280090bad39SAlexey Kardashevskiy free_tces_exit: 281090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(addr, 282090bad39SAlexey Kardashevskiy 1ULL << (level_shift - 3), levels - 1); 283090bad39SAlexey Kardashevskiy 284090bad39SAlexey Kardashevskiy return -ENOMEM; 285191c2287SAlexey Kardashevskiy } 286191c2287SAlexey Kardashevskiy 287191c2287SAlexey Kardashevskiy static void pnv_iommu_table_group_link_free(struct rcu_head *head) 288191c2287SAlexey Kardashevskiy { 289191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl = container_of(head, 290191c2287SAlexey Kardashevskiy struct iommu_table_group_link, rcu); 291191c2287SAlexey Kardashevskiy 292191c2287SAlexey Kardashevskiy kfree(tgl); 293191c2287SAlexey Kardashevskiy } 294191c2287SAlexey Kardashevskiy 295191c2287SAlexey Kardashevskiy void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, 296191c2287SAlexey Kardashevskiy struct iommu_table_group *table_group) 297191c2287SAlexey Kardashevskiy { 298191c2287SAlexey Kardashevskiy long i; 299191c2287SAlexey Kardashevskiy bool found; 300191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl; 301191c2287SAlexey Kardashevskiy 302191c2287SAlexey Kardashevskiy if (!tbl || !table_group) 303191c2287SAlexey Kardashevskiy return; 304191c2287SAlexey Kardashevskiy 305191c2287SAlexey Kardashevskiy /* Remove link to a group from table's list of attached groups */ 306191c2287SAlexey Kardashevskiy found = false; 307191c2287SAlexey Kardashevskiy list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { 308191c2287SAlexey Kardashevskiy if (tgl->table_group == table_group) { 309191c2287SAlexey Kardashevskiy list_del_rcu(&tgl->next); 310191c2287SAlexey Kardashevskiy call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free); 311191c2287SAlexey Kardashevskiy found = true; 312191c2287SAlexey Kardashevskiy break; 313191c2287SAlexey Kardashevskiy } 314191c2287SAlexey Kardashevskiy } 315191c2287SAlexey Kardashevskiy if (WARN_ON(!found)) 316191c2287SAlexey Kardashevskiy return; 317191c2287SAlexey Kardashevskiy 318191c2287SAlexey Kardashevskiy /* Clean a pointer to iommu_table in iommu_table_group::tables[] */ 319191c2287SAlexey Kardashevskiy found = false; 320191c2287SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 321191c2287SAlexey Kardashevskiy if (table_group->tables[i] == tbl) { 322191c2287SAlexey Kardashevskiy table_group->tables[i] = NULL; 323191c2287SAlexey Kardashevskiy found = true; 324191c2287SAlexey Kardashevskiy break; 325191c2287SAlexey Kardashevskiy } 326191c2287SAlexey Kardashevskiy } 327191c2287SAlexey Kardashevskiy WARN_ON(!found); 328191c2287SAlexey Kardashevskiy } 329191c2287SAlexey Kardashevskiy 330191c2287SAlexey Kardashevskiy long pnv_pci_link_table_and_group(int node, int num, 331191c2287SAlexey Kardashevskiy struct iommu_table *tbl, 332191c2287SAlexey Kardashevskiy struct iommu_table_group *table_group) 333191c2287SAlexey Kardashevskiy { 334191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl = NULL; 335191c2287SAlexey Kardashevskiy 336191c2287SAlexey Kardashevskiy if (WARN_ON(!tbl || !table_group)) 337191c2287SAlexey Kardashevskiy return -EINVAL; 338191c2287SAlexey Kardashevskiy 339191c2287SAlexey Kardashevskiy tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, 340191c2287SAlexey Kardashevskiy node); 341191c2287SAlexey Kardashevskiy if (!tgl) 342191c2287SAlexey Kardashevskiy return -ENOMEM; 343191c2287SAlexey Kardashevskiy 344191c2287SAlexey Kardashevskiy tgl->table_group = table_group; 345191c2287SAlexey Kardashevskiy list_add_rcu(&tgl->next, &tbl->it_group_list); 346191c2287SAlexey Kardashevskiy 347191c2287SAlexey Kardashevskiy table_group->tables[num] = tbl; 348191c2287SAlexey Kardashevskiy 349191c2287SAlexey Kardashevskiy return 0; 350191c2287SAlexey Kardashevskiy } 351