1191c2287SAlexey Kardashevskiy // SPDX-License-Identifier: GPL-2.0+ 2191c2287SAlexey Kardashevskiy /* 3191c2287SAlexey Kardashevskiy * TCE helpers for IODA PCI/PCIe on PowerNV platforms 4191c2287SAlexey Kardashevskiy * 5191c2287SAlexey Kardashevskiy * Copyright 2018 IBM Corp. 6191c2287SAlexey Kardashevskiy * 7191c2287SAlexey Kardashevskiy * This program is free software; you can redistribute it and/or 8191c2287SAlexey Kardashevskiy * modify it under the terms of the GNU General Public License 9191c2287SAlexey Kardashevskiy * as published by the Free Software Foundation; either version 10191c2287SAlexey Kardashevskiy * 2 of the License, or (at your option) any later version. 11191c2287SAlexey Kardashevskiy */ 12191c2287SAlexey Kardashevskiy 13191c2287SAlexey Kardashevskiy #include <linux/kernel.h> 14191c2287SAlexey Kardashevskiy #include <linux/iommu.h> 15191c2287SAlexey Kardashevskiy 16191c2287SAlexey Kardashevskiy #include <asm/iommu.h> 17191c2287SAlexey Kardashevskiy #include <asm/tce.h> 18191c2287SAlexey Kardashevskiy #include "pci.h" 19191c2287SAlexey Kardashevskiy 20191c2287SAlexey Kardashevskiy void pnv_pci_setup_iommu_table(struct iommu_table *tbl, 21191c2287SAlexey Kardashevskiy void *tce_mem, u64 tce_size, 22191c2287SAlexey Kardashevskiy u64 dma_offset, unsigned int page_shift) 23191c2287SAlexey Kardashevskiy { 24191c2287SAlexey Kardashevskiy tbl->it_blocksize = 16; 25191c2287SAlexey Kardashevskiy tbl->it_base = (unsigned long)tce_mem; 26191c2287SAlexey Kardashevskiy tbl->it_page_shift = page_shift; 27191c2287SAlexey Kardashevskiy tbl->it_offset = dma_offset >> tbl->it_page_shift; 28191c2287SAlexey Kardashevskiy tbl->it_index = 0; 29191c2287SAlexey Kardashevskiy tbl->it_size = tce_size >> 3; 30191c2287SAlexey Kardashevskiy tbl->it_busno = 0; 31191c2287SAlexey Kardashevskiy tbl->it_type = TCE_PCI; 32191c2287SAlexey Kardashevskiy } 33191c2287SAlexey Kardashevskiy 349bc98c8aSAlexey Kardashevskiy static __be64 *pnv_alloc_tce_level(int nid, unsigned int shift) 359bc98c8aSAlexey Kardashevskiy { 369bc98c8aSAlexey Kardashevskiy struct page *tce_mem = NULL; 379bc98c8aSAlexey Kardashevskiy __be64 *addr; 389bc98c8aSAlexey Kardashevskiy 399bc98c8aSAlexey Kardashevskiy tce_mem = alloc_pages_node(nid, GFP_KERNEL, shift - PAGE_SHIFT); 409bc98c8aSAlexey Kardashevskiy if (!tce_mem) { 419bc98c8aSAlexey Kardashevskiy pr_err("Failed to allocate a TCE memory, level shift=%d\n", 429bc98c8aSAlexey Kardashevskiy shift); 439bc98c8aSAlexey Kardashevskiy return NULL; 449bc98c8aSAlexey Kardashevskiy } 459bc98c8aSAlexey Kardashevskiy addr = page_address(tce_mem); 469bc98c8aSAlexey Kardashevskiy memset(addr, 0, 1UL << shift); 479bc98c8aSAlexey Kardashevskiy 489bc98c8aSAlexey Kardashevskiy return addr; 499bc98c8aSAlexey Kardashevskiy } 509bc98c8aSAlexey Kardashevskiy 51a68bd126SAlexey Kardashevskiy static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc) 52191c2287SAlexey Kardashevskiy { 53090bad39SAlexey Kardashevskiy __be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base; 54191c2287SAlexey Kardashevskiy int level = tbl->it_indirect_levels; 55191c2287SAlexey Kardashevskiy const long shift = ilog2(tbl->it_level_size); 56191c2287SAlexey Kardashevskiy unsigned long mask = (tbl->it_level_size - 1) << (level * shift); 57191c2287SAlexey Kardashevskiy 58191c2287SAlexey Kardashevskiy while (level) { 59191c2287SAlexey Kardashevskiy int n = (idx & mask) >> (level * shift); 60a68bd126SAlexey Kardashevskiy unsigned long tce; 61a68bd126SAlexey Kardashevskiy 62a68bd126SAlexey Kardashevskiy if (tmp[n] == 0) { 63a68bd126SAlexey Kardashevskiy __be64 *tmp2; 64a68bd126SAlexey Kardashevskiy 65a68bd126SAlexey Kardashevskiy if (!alloc) 66a68bd126SAlexey Kardashevskiy return NULL; 67a68bd126SAlexey Kardashevskiy 68a68bd126SAlexey Kardashevskiy tmp2 = pnv_alloc_tce_level(tbl->it_nid, 69a68bd126SAlexey Kardashevskiy ilog2(tbl->it_level_size) + 3); 70a68bd126SAlexey Kardashevskiy if (!tmp2) 71a68bd126SAlexey Kardashevskiy return NULL; 72a68bd126SAlexey Kardashevskiy 73a68bd126SAlexey Kardashevskiy tmp[n] = cpu_to_be64(__pa(tmp2) | 74a68bd126SAlexey Kardashevskiy TCE_PCI_READ | TCE_PCI_WRITE); 75a68bd126SAlexey Kardashevskiy } 76a68bd126SAlexey Kardashevskiy tce = be64_to_cpu(tmp[n]); 77191c2287SAlexey Kardashevskiy 78191c2287SAlexey Kardashevskiy tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE)); 79191c2287SAlexey Kardashevskiy idx &= ~mask; 80191c2287SAlexey Kardashevskiy mask >>= shift; 81191c2287SAlexey Kardashevskiy --level; 82191c2287SAlexey Kardashevskiy } 83191c2287SAlexey Kardashevskiy 84191c2287SAlexey Kardashevskiy return tmp + idx; 85191c2287SAlexey Kardashevskiy } 86191c2287SAlexey Kardashevskiy 87191c2287SAlexey Kardashevskiy int pnv_tce_build(struct iommu_table *tbl, long index, long npages, 88191c2287SAlexey Kardashevskiy unsigned long uaddr, enum dma_data_direction direction, 89191c2287SAlexey Kardashevskiy unsigned long attrs) 90191c2287SAlexey Kardashevskiy { 91191c2287SAlexey Kardashevskiy u64 proto_tce = iommu_direction_to_tce_perm(direction); 92191c2287SAlexey Kardashevskiy u64 rpn = __pa(uaddr) >> tbl->it_page_shift; 93191c2287SAlexey Kardashevskiy long i; 94191c2287SAlexey Kardashevskiy 95191c2287SAlexey Kardashevskiy if (proto_tce & TCE_PCI_WRITE) 96191c2287SAlexey Kardashevskiy proto_tce |= TCE_PCI_READ; 97191c2287SAlexey Kardashevskiy 98191c2287SAlexey Kardashevskiy for (i = 0; i < npages; i++) { 99191c2287SAlexey Kardashevskiy unsigned long newtce = proto_tce | 100191c2287SAlexey Kardashevskiy ((rpn + i) << tbl->it_page_shift); 101191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset + i; 102191c2287SAlexey Kardashevskiy 103a68bd126SAlexey Kardashevskiy *(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce); 104191c2287SAlexey Kardashevskiy } 105191c2287SAlexey Kardashevskiy 106191c2287SAlexey Kardashevskiy return 0; 107191c2287SAlexey Kardashevskiy } 108191c2287SAlexey Kardashevskiy 109191c2287SAlexey Kardashevskiy #ifdef CONFIG_IOMMU_API 110191c2287SAlexey Kardashevskiy int pnv_tce_xchg(struct iommu_table *tbl, long index, 111a68bd126SAlexey Kardashevskiy unsigned long *hpa, enum dma_data_direction *direction, 112a68bd126SAlexey Kardashevskiy bool alloc) 113191c2287SAlexey Kardashevskiy { 114191c2287SAlexey Kardashevskiy u64 proto_tce = iommu_direction_to_tce_perm(*direction); 115191c2287SAlexey Kardashevskiy unsigned long newtce = *hpa | proto_tce, oldtce; 116191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset; 117a68bd126SAlexey Kardashevskiy __be64 *ptce = NULL; 118191c2287SAlexey Kardashevskiy 119191c2287SAlexey Kardashevskiy BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)); 120191c2287SAlexey Kardashevskiy 121a68bd126SAlexey Kardashevskiy if (*direction == DMA_NONE) { 122a68bd126SAlexey Kardashevskiy ptce = pnv_tce(tbl, false, idx, false); 123a68bd126SAlexey Kardashevskiy if (!ptce) { 124a68bd126SAlexey Kardashevskiy *hpa = 0; 125a68bd126SAlexey Kardashevskiy return 0; 126a68bd126SAlexey Kardashevskiy } 127a68bd126SAlexey Kardashevskiy } 128a68bd126SAlexey Kardashevskiy 129a68bd126SAlexey Kardashevskiy if (!ptce) { 130a68bd126SAlexey Kardashevskiy ptce = pnv_tce(tbl, false, idx, alloc); 131a68bd126SAlexey Kardashevskiy if (!ptce) 132a68bd126SAlexey Kardashevskiy return alloc ? H_HARDWARE : H_TOO_HARD; 133a68bd126SAlexey Kardashevskiy } 134a68bd126SAlexey Kardashevskiy 135191c2287SAlexey Kardashevskiy if (newtce & TCE_PCI_WRITE) 136191c2287SAlexey Kardashevskiy newtce |= TCE_PCI_READ; 137191c2287SAlexey Kardashevskiy 138a68bd126SAlexey Kardashevskiy oldtce = be64_to_cpu(xchg(ptce, cpu_to_be64(newtce))); 139191c2287SAlexey Kardashevskiy *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); 140191c2287SAlexey Kardashevskiy *direction = iommu_tce_direction(oldtce); 141191c2287SAlexey Kardashevskiy 142191c2287SAlexey Kardashevskiy return 0; 143191c2287SAlexey Kardashevskiy } 144090bad39SAlexey Kardashevskiy 145a68bd126SAlexey Kardashevskiy __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc) 146090bad39SAlexey Kardashevskiy { 147090bad39SAlexey Kardashevskiy if (WARN_ON_ONCE(!tbl->it_userspace)) 148090bad39SAlexey Kardashevskiy return NULL; 149090bad39SAlexey Kardashevskiy 150a68bd126SAlexey Kardashevskiy return pnv_tce(tbl, true, index - tbl->it_offset, alloc); 151090bad39SAlexey Kardashevskiy } 152191c2287SAlexey Kardashevskiy #endif 153191c2287SAlexey Kardashevskiy 154191c2287SAlexey Kardashevskiy void pnv_tce_free(struct iommu_table *tbl, long index, long npages) 155191c2287SAlexey Kardashevskiy { 156191c2287SAlexey Kardashevskiy long i; 157191c2287SAlexey Kardashevskiy 158191c2287SAlexey Kardashevskiy for (i = 0; i < npages; i++) { 159191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset + i; 160a68bd126SAlexey Kardashevskiy __be64 *ptce = pnv_tce(tbl, false, idx, false); 161191c2287SAlexey Kardashevskiy 162a68bd126SAlexey Kardashevskiy if (ptce) 163a68bd126SAlexey Kardashevskiy *ptce = cpu_to_be64(0); 164191c2287SAlexey Kardashevskiy } 165191c2287SAlexey Kardashevskiy } 166191c2287SAlexey Kardashevskiy 167191c2287SAlexey Kardashevskiy unsigned long pnv_tce_get(struct iommu_table *tbl, long index) 168191c2287SAlexey Kardashevskiy { 169a68bd126SAlexey Kardashevskiy __be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false); 170a68bd126SAlexey Kardashevskiy 171a68bd126SAlexey Kardashevskiy if (!ptce) 172a68bd126SAlexey Kardashevskiy return 0; 173090bad39SAlexey Kardashevskiy 174090bad39SAlexey Kardashevskiy return be64_to_cpu(*ptce); 175191c2287SAlexey Kardashevskiy } 176191c2287SAlexey Kardashevskiy 177191c2287SAlexey Kardashevskiy static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, 178191c2287SAlexey Kardashevskiy unsigned long size, unsigned int levels) 179191c2287SAlexey Kardashevskiy { 180191c2287SAlexey Kardashevskiy const unsigned long addr_ul = (unsigned long) addr & 181191c2287SAlexey Kardashevskiy ~(TCE_PCI_READ | TCE_PCI_WRITE); 182191c2287SAlexey Kardashevskiy 183191c2287SAlexey Kardashevskiy if (levels) { 184191c2287SAlexey Kardashevskiy long i; 185191c2287SAlexey Kardashevskiy u64 *tmp = (u64 *) addr_ul; 186191c2287SAlexey Kardashevskiy 187191c2287SAlexey Kardashevskiy for (i = 0; i < size; ++i) { 188191c2287SAlexey Kardashevskiy unsigned long hpa = be64_to_cpu(tmp[i]); 189191c2287SAlexey Kardashevskiy 190191c2287SAlexey Kardashevskiy if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE))) 191191c2287SAlexey Kardashevskiy continue; 192191c2287SAlexey Kardashevskiy 193191c2287SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(__va(hpa), size, 194191c2287SAlexey Kardashevskiy levels - 1); 195191c2287SAlexey Kardashevskiy } 196191c2287SAlexey Kardashevskiy } 197191c2287SAlexey Kardashevskiy 198191c2287SAlexey Kardashevskiy free_pages(addr_ul, get_order(size << 3)); 199191c2287SAlexey Kardashevskiy } 200191c2287SAlexey Kardashevskiy 201191c2287SAlexey Kardashevskiy void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl) 202191c2287SAlexey Kardashevskiy { 203191c2287SAlexey Kardashevskiy const unsigned long size = tbl->it_indirect_levels ? 204191c2287SAlexey Kardashevskiy tbl->it_level_size : tbl->it_size; 205191c2287SAlexey Kardashevskiy 206191c2287SAlexey Kardashevskiy if (!tbl->it_size) 207191c2287SAlexey Kardashevskiy return; 208191c2287SAlexey Kardashevskiy 209191c2287SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size, 210191c2287SAlexey Kardashevskiy tbl->it_indirect_levels); 211090bad39SAlexey Kardashevskiy if (tbl->it_userspace) { 212090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size, 213090bad39SAlexey Kardashevskiy tbl->it_indirect_levels); 214090bad39SAlexey Kardashevskiy } 215191c2287SAlexey Kardashevskiy } 216191c2287SAlexey Kardashevskiy 217191c2287SAlexey Kardashevskiy static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift, 218191c2287SAlexey Kardashevskiy unsigned int levels, unsigned long limit, 219191c2287SAlexey Kardashevskiy unsigned long *current_offset, unsigned long *total_allocated) 220191c2287SAlexey Kardashevskiy { 221191c2287SAlexey Kardashevskiy __be64 *addr, *tmp; 2229bc98c8aSAlexey Kardashevskiy unsigned long allocated = 1UL << shift; 223191c2287SAlexey Kardashevskiy unsigned int entries = 1UL << (shift - 3); 224191c2287SAlexey Kardashevskiy long i; 225191c2287SAlexey Kardashevskiy 2269bc98c8aSAlexey Kardashevskiy addr = pnv_alloc_tce_level(nid, shift); 227191c2287SAlexey Kardashevskiy *total_allocated += allocated; 228191c2287SAlexey Kardashevskiy 229191c2287SAlexey Kardashevskiy --levels; 230191c2287SAlexey Kardashevskiy if (!levels) { 231191c2287SAlexey Kardashevskiy *current_offset += allocated; 232191c2287SAlexey Kardashevskiy return addr; 233191c2287SAlexey Kardashevskiy } 234191c2287SAlexey Kardashevskiy 235191c2287SAlexey Kardashevskiy for (i = 0; i < entries; ++i) { 236191c2287SAlexey Kardashevskiy tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift, 237191c2287SAlexey Kardashevskiy levels, limit, current_offset, total_allocated); 238191c2287SAlexey Kardashevskiy if (!tmp) 239191c2287SAlexey Kardashevskiy break; 240191c2287SAlexey Kardashevskiy 241191c2287SAlexey Kardashevskiy addr[i] = cpu_to_be64(__pa(tmp) | 242191c2287SAlexey Kardashevskiy TCE_PCI_READ | TCE_PCI_WRITE); 243191c2287SAlexey Kardashevskiy 244191c2287SAlexey Kardashevskiy if (*current_offset >= limit) 245191c2287SAlexey Kardashevskiy break; 246191c2287SAlexey Kardashevskiy } 247191c2287SAlexey Kardashevskiy 248191c2287SAlexey Kardashevskiy return addr; 249191c2287SAlexey Kardashevskiy } 250191c2287SAlexey Kardashevskiy 251191c2287SAlexey Kardashevskiy long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, 252191c2287SAlexey Kardashevskiy __u32 page_shift, __u64 window_size, __u32 levels, 253090bad39SAlexey Kardashevskiy bool alloc_userspace_copy, struct iommu_table *tbl) 254191c2287SAlexey Kardashevskiy { 255090bad39SAlexey Kardashevskiy void *addr, *uas = NULL; 256191c2287SAlexey Kardashevskiy unsigned long offset = 0, level_shift, total_allocated = 0; 257090bad39SAlexey Kardashevskiy unsigned long total_allocated_uas = 0; 258191c2287SAlexey Kardashevskiy const unsigned int window_shift = ilog2(window_size); 259191c2287SAlexey Kardashevskiy unsigned int entries_shift = window_shift - page_shift; 260191c2287SAlexey Kardashevskiy unsigned int table_shift = max_t(unsigned int, entries_shift + 3, 261191c2287SAlexey Kardashevskiy PAGE_SHIFT); 262191c2287SAlexey Kardashevskiy const unsigned long tce_table_size = 1UL << table_shift; 263a68bd126SAlexey Kardashevskiy unsigned int tmplevels = levels; 264191c2287SAlexey Kardashevskiy 265191c2287SAlexey Kardashevskiy if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS)) 266191c2287SAlexey Kardashevskiy return -EINVAL; 267191c2287SAlexey Kardashevskiy 268191c2287SAlexey Kardashevskiy if (!is_power_of_2(window_size)) 269191c2287SAlexey Kardashevskiy return -EINVAL; 270191c2287SAlexey Kardashevskiy 271a68bd126SAlexey Kardashevskiy if (alloc_userspace_copy && (window_size > (1ULL << 32))) 272a68bd126SAlexey Kardashevskiy tmplevels = 1; 273a68bd126SAlexey Kardashevskiy 274191c2287SAlexey Kardashevskiy /* Adjust direct table size from window_size and levels */ 275191c2287SAlexey Kardashevskiy entries_shift = (entries_shift + levels - 1) / levels; 276191c2287SAlexey Kardashevskiy level_shift = entries_shift + 3; 277191c2287SAlexey Kardashevskiy level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT); 278191c2287SAlexey Kardashevskiy 279191c2287SAlexey Kardashevskiy if ((level_shift - 3) * levels + page_shift >= 60) 280191c2287SAlexey Kardashevskiy return -EINVAL; 281191c2287SAlexey Kardashevskiy 282191c2287SAlexey Kardashevskiy /* Allocate TCE table */ 283191c2287SAlexey Kardashevskiy addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 284a68bd126SAlexey Kardashevskiy tmplevels, tce_table_size, &offset, &total_allocated); 285191c2287SAlexey Kardashevskiy 286191c2287SAlexey Kardashevskiy /* addr==NULL means that the first level allocation failed */ 287191c2287SAlexey Kardashevskiy if (!addr) 288191c2287SAlexey Kardashevskiy return -ENOMEM; 289191c2287SAlexey Kardashevskiy 290191c2287SAlexey Kardashevskiy /* 291191c2287SAlexey Kardashevskiy * First level was allocated but some lower level failed as 292191c2287SAlexey Kardashevskiy * we did not allocate as much as we wanted, 293191c2287SAlexey Kardashevskiy * release partially allocated table. 294191c2287SAlexey Kardashevskiy */ 295a68bd126SAlexey Kardashevskiy if (tmplevels == levels && offset < tce_table_size) 296090bad39SAlexey Kardashevskiy goto free_tces_exit; 297090bad39SAlexey Kardashevskiy 298090bad39SAlexey Kardashevskiy /* Allocate userspace view of the TCE table */ 299090bad39SAlexey Kardashevskiy if (alloc_userspace_copy) { 300090bad39SAlexey Kardashevskiy offset = 0; 301090bad39SAlexey Kardashevskiy uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 302090bad39SAlexey Kardashevskiy levels, tce_table_size, &offset, 303090bad39SAlexey Kardashevskiy &total_allocated_uas); 304090bad39SAlexey Kardashevskiy if (!uas) 305090bad39SAlexey Kardashevskiy goto free_tces_exit; 306a68bd126SAlexey Kardashevskiy if (tmplevels == levels && (offset < tce_table_size || 307a68bd126SAlexey Kardashevskiy total_allocated_uas != total_allocated)) 308090bad39SAlexey Kardashevskiy goto free_uas_exit; 309191c2287SAlexey Kardashevskiy } 310191c2287SAlexey Kardashevskiy 311191c2287SAlexey Kardashevskiy /* Setup linux iommu table */ 312191c2287SAlexey Kardashevskiy pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset, 313191c2287SAlexey Kardashevskiy page_shift); 314191c2287SAlexey Kardashevskiy tbl->it_level_size = 1ULL << (level_shift - 3); 315191c2287SAlexey Kardashevskiy tbl->it_indirect_levels = levels - 1; 316191c2287SAlexey Kardashevskiy tbl->it_allocated_size = total_allocated; 317090bad39SAlexey Kardashevskiy tbl->it_userspace = uas; 318a68bd126SAlexey Kardashevskiy tbl->it_nid = nid; 319191c2287SAlexey Kardashevskiy 320a68bd126SAlexey Kardashevskiy pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d/%d\n", 321090bad39SAlexey Kardashevskiy window_size, tce_table_size, bus_offset, tbl->it_base, 322a68bd126SAlexey Kardashevskiy tbl->it_userspace, tmplevels, levels); 323191c2287SAlexey Kardashevskiy 324191c2287SAlexey Kardashevskiy return 0; 325090bad39SAlexey Kardashevskiy 326090bad39SAlexey Kardashevskiy free_uas_exit: 327090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(uas, 328090bad39SAlexey Kardashevskiy 1ULL << (level_shift - 3), levels - 1); 329090bad39SAlexey Kardashevskiy free_tces_exit: 330090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(addr, 331090bad39SAlexey Kardashevskiy 1ULL << (level_shift - 3), levels - 1); 332090bad39SAlexey Kardashevskiy 333090bad39SAlexey Kardashevskiy return -ENOMEM; 334191c2287SAlexey Kardashevskiy } 335191c2287SAlexey Kardashevskiy 336191c2287SAlexey Kardashevskiy static void pnv_iommu_table_group_link_free(struct rcu_head *head) 337191c2287SAlexey Kardashevskiy { 338191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl = container_of(head, 339191c2287SAlexey Kardashevskiy struct iommu_table_group_link, rcu); 340191c2287SAlexey Kardashevskiy 341191c2287SAlexey Kardashevskiy kfree(tgl); 342191c2287SAlexey Kardashevskiy } 343191c2287SAlexey Kardashevskiy 344191c2287SAlexey Kardashevskiy void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, 345191c2287SAlexey Kardashevskiy struct iommu_table_group *table_group) 346191c2287SAlexey Kardashevskiy { 347191c2287SAlexey Kardashevskiy long i; 348191c2287SAlexey Kardashevskiy bool found; 349191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl; 350191c2287SAlexey Kardashevskiy 351191c2287SAlexey Kardashevskiy if (!tbl || !table_group) 352191c2287SAlexey Kardashevskiy return; 353191c2287SAlexey Kardashevskiy 354191c2287SAlexey Kardashevskiy /* Remove link to a group from table's list of attached groups */ 355191c2287SAlexey Kardashevskiy found = false; 356191c2287SAlexey Kardashevskiy list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { 357191c2287SAlexey Kardashevskiy if (tgl->table_group == table_group) { 358191c2287SAlexey Kardashevskiy list_del_rcu(&tgl->next); 359191c2287SAlexey Kardashevskiy call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free); 360191c2287SAlexey Kardashevskiy found = true; 361191c2287SAlexey Kardashevskiy break; 362191c2287SAlexey Kardashevskiy } 363191c2287SAlexey Kardashevskiy } 364191c2287SAlexey Kardashevskiy if (WARN_ON(!found)) 365191c2287SAlexey Kardashevskiy return; 366191c2287SAlexey Kardashevskiy 367191c2287SAlexey Kardashevskiy /* Clean a pointer to iommu_table in iommu_table_group::tables[] */ 368191c2287SAlexey Kardashevskiy found = false; 369191c2287SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 370191c2287SAlexey Kardashevskiy if (table_group->tables[i] == tbl) { 371191c2287SAlexey Kardashevskiy table_group->tables[i] = NULL; 372191c2287SAlexey Kardashevskiy found = true; 373191c2287SAlexey Kardashevskiy break; 374191c2287SAlexey Kardashevskiy } 375191c2287SAlexey Kardashevskiy } 376191c2287SAlexey Kardashevskiy WARN_ON(!found); 377191c2287SAlexey Kardashevskiy } 378191c2287SAlexey Kardashevskiy 379191c2287SAlexey Kardashevskiy long pnv_pci_link_table_and_group(int node, int num, 380191c2287SAlexey Kardashevskiy struct iommu_table *tbl, 381191c2287SAlexey Kardashevskiy struct iommu_table_group *table_group) 382191c2287SAlexey Kardashevskiy { 383191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl = NULL; 384191c2287SAlexey Kardashevskiy 385191c2287SAlexey Kardashevskiy if (WARN_ON(!tbl || !table_group)) 386191c2287SAlexey Kardashevskiy return -EINVAL; 387191c2287SAlexey Kardashevskiy 388191c2287SAlexey Kardashevskiy tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, 389191c2287SAlexey Kardashevskiy node); 390191c2287SAlexey Kardashevskiy if (!tgl) 391191c2287SAlexey Kardashevskiy return -ENOMEM; 392191c2287SAlexey Kardashevskiy 393191c2287SAlexey Kardashevskiy tgl->table_group = table_group; 394191c2287SAlexey Kardashevskiy list_add_rcu(&tgl->next, &tbl->it_group_list); 395191c2287SAlexey Kardashevskiy 396191c2287SAlexey Kardashevskiy table_group->tables[num] = tbl; 397191c2287SAlexey Kardashevskiy 398191c2287SAlexey Kardashevskiy return 0; 399191c2287SAlexey Kardashevskiy } 400