1191c2287SAlexey Kardashevskiy // SPDX-License-Identifier: GPL-2.0+ 2191c2287SAlexey Kardashevskiy /* 3191c2287SAlexey Kardashevskiy * TCE helpers for IODA PCI/PCIe on PowerNV platforms 4191c2287SAlexey Kardashevskiy * 5191c2287SAlexey Kardashevskiy * Copyright 2018 IBM Corp. 6191c2287SAlexey Kardashevskiy * 7191c2287SAlexey Kardashevskiy * This program is free software; you can redistribute it and/or 8191c2287SAlexey Kardashevskiy * modify it under the terms of the GNU General Public License 9191c2287SAlexey Kardashevskiy * as published by the Free Software Foundation; either version 10191c2287SAlexey Kardashevskiy * 2 of the License, or (at your option) any later version. 11191c2287SAlexey Kardashevskiy */ 12191c2287SAlexey Kardashevskiy 13191c2287SAlexey Kardashevskiy #include <linux/kernel.h> 14191c2287SAlexey Kardashevskiy #include <linux/iommu.h> 15191c2287SAlexey Kardashevskiy 16191c2287SAlexey Kardashevskiy #include <asm/iommu.h> 17191c2287SAlexey Kardashevskiy #include <asm/tce.h> 18191c2287SAlexey Kardashevskiy #include "pci.h" 19191c2287SAlexey Kardashevskiy 2096e2006aSOliver O'Halloran unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb) 2196e2006aSOliver O'Halloran { 2296e2006aSOliver O'Halloran struct pci_controller *hose = phb->hose; 2396e2006aSOliver O'Halloran struct device_node *dn = hose->dn; 2496e2006aSOliver O'Halloran unsigned long mask = 0; 2596e2006aSOliver O'Halloran int i, rc, count; 2696e2006aSOliver O'Halloran u32 val; 2796e2006aSOliver O'Halloran 2896e2006aSOliver O'Halloran count = of_property_count_u32_elems(dn, "ibm,supported-tce-sizes"); 2996e2006aSOliver O'Halloran if (count <= 0) { 3096e2006aSOliver O'Halloran mask = SZ_4K | SZ_64K; 3196e2006aSOliver O'Halloran /* Add 16M for POWER8 by default */ 3296e2006aSOliver O'Halloran if (cpu_has_feature(CPU_FTR_ARCH_207S) && 3396e2006aSOliver O'Halloran !cpu_has_feature(CPU_FTR_ARCH_300)) 3496e2006aSOliver O'Halloran mask |= SZ_16M | SZ_256M; 3596e2006aSOliver O'Halloran return mask; 3696e2006aSOliver O'Halloran } 3796e2006aSOliver O'Halloran 3896e2006aSOliver O'Halloran for (i = 0; i < count; i++) { 3996e2006aSOliver O'Halloran rc = of_property_read_u32_index(dn, "ibm,supported-tce-sizes", 4096e2006aSOliver O'Halloran i, &val); 4196e2006aSOliver O'Halloran if (rc == 0) 4296e2006aSOliver O'Halloran mask |= 1ULL << val; 4396e2006aSOliver O'Halloran } 4496e2006aSOliver O'Halloran 4596e2006aSOliver O'Halloran return mask; 4696e2006aSOliver O'Halloran } 4796e2006aSOliver O'Halloran 48191c2287SAlexey Kardashevskiy void pnv_pci_setup_iommu_table(struct iommu_table *tbl, 49191c2287SAlexey Kardashevskiy void *tce_mem, u64 tce_size, 50191c2287SAlexey Kardashevskiy u64 dma_offset, unsigned int page_shift) 51191c2287SAlexey Kardashevskiy { 52191c2287SAlexey Kardashevskiy tbl->it_blocksize = 16; 53191c2287SAlexey Kardashevskiy tbl->it_base = (unsigned long)tce_mem; 54191c2287SAlexey Kardashevskiy tbl->it_page_shift = page_shift; 55191c2287SAlexey Kardashevskiy tbl->it_offset = dma_offset >> tbl->it_page_shift; 56191c2287SAlexey Kardashevskiy tbl->it_index = 0; 57191c2287SAlexey Kardashevskiy tbl->it_size = tce_size >> 3; 58191c2287SAlexey Kardashevskiy tbl->it_busno = 0; 59191c2287SAlexey Kardashevskiy tbl->it_type = TCE_PCI; 60191c2287SAlexey Kardashevskiy } 61191c2287SAlexey Kardashevskiy 629bc98c8aSAlexey Kardashevskiy static __be64 *pnv_alloc_tce_level(int nid, unsigned int shift) 639bc98c8aSAlexey Kardashevskiy { 649bc98c8aSAlexey Kardashevskiy struct page *tce_mem = NULL; 659bc98c8aSAlexey Kardashevskiy __be64 *addr; 669bc98c8aSAlexey Kardashevskiy 67c37c792dSAlexey Kardashevskiy tce_mem = alloc_pages_node(nid, GFP_ATOMIC | __GFP_NOWARN, 68c37c792dSAlexey Kardashevskiy shift - PAGE_SHIFT); 699bc98c8aSAlexey Kardashevskiy if (!tce_mem) { 709bc98c8aSAlexey Kardashevskiy pr_err("Failed to allocate a TCE memory, level shift=%d\n", 719bc98c8aSAlexey Kardashevskiy shift); 729bc98c8aSAlexey Kardashevskiy return NULL; 739bc98c8aSAlexey Kardashevskiy } 749bc98c8aSAlexey Kardashevskiy addr = page_address(tce_mem); 759bc98c8aSAlexey Kardashevskiy memset(addr, 0, 1UL << shift); 769bc98c8aSAlexey Kardashevskiy 779bc98c8aSAlexey Kardashevskiy return addr; 789bc98c8aSAlexey Kardashevskiy } 799bc98c8aSAlexey Kardashevskiy 8056090a39SAlexey Kardashevskiy static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, 8156090a39SAlexey Kardashevskiy unsigned long size, unsigned int levels); 8256090a39SAlexey Kardashevskiy 83a68bd126SAlexey Kardashevskiy static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc) 84191c2287SAlexey Kardashevskiy { 85090bad39SAlexey Kardashevskiy __be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base; 86191c2287SAlexey Kardashevskiy int level = tbl->it_indirect_levels; 87191c2287SAlexey Kardashevskiy const long shift = ilog2(tbl->it_level_size); 88191c2287SAlexey Kardashevskiy unsigned long mask = (tbl->it_level_size - 1) << (level * shift); 89191c2287SAlexey Kardashevskiy 90191c2287SAlexey Kardashevskiy while (level) { 91191c2287SAlexey Kardashevskiy int n = (idx & mask) >> (level * shift); 9256090a39SAlexey Kardashevskiy unsigned long oldtce, tce = be64_to_cpu(READ_ONCE(tmp[n])); 93a68bd126SAlexey Kardashevskiy 9456090a39SAlexey Kardashevskiy if (!tce) { 95a68bd126SAlexey Kardashevskiy __be64 *tmp2; 96a68bd126SAlexey Kardashevskiy 97a68bd126SAlexey Kardashevskiy if (!alloc) 98a68bd126SAlexey Kardashevskiy return NULL; 99a68bd126SAlexey Kardashevskiy 100a68bd126SAlexey Kardashevskiy tmp2 = pnv_alloc_tce_level(tbl->it_nid, 101a68bd126SAlexey Kardashevskiy ilog2(tbl->it_level_size) + 3); 102a68bd126SAlexey Kardashevskiy if (!tmp2) 103a68bd126SAlexey Kardashevskiy return NULL; 104a68bd126SAlexey Kardashevskiy 10556090a39SAlexey Kardashevskiy tce = __pa(tmp2) | TCE_PCI_READ | TCE_PCI_WRITE; 10656090a39SAlexey Kardashevskiy oldtce = be64_to_cpu(cmpxchg(&tmp[n], 0, 10756090a39SAlexey Kardashevskiy cpu_to_be64(tce))); 10856090a39SAlexey Kardashevskiy if (oldtce) { 10956090a39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(tmp2, 11056090a39SAlexey Kardashevskiy ilog2(tbl->it_level_size) + 3, 1); 11156090a39SAlexey Kardashevskiy tce = oldtce; 112a68bd126SAlexey Kardashevskiy } 11356090a39SAlexey Kardashevskiy } 114191c2287SAlexey Kardashevskiy 115191c2287SAlexey Kardashevskiy tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE)); 116191c2287SAlexey Kardashevskiy idx &= ~mask; 117191c2287SAlexey Kardashevskiy mask >>= shift; 118191c2287SAlexey Kardashevskiy --level; 119191c2287SAlexey Kardashevskiy } 120191c2287SAlexey Kardashevskiy 121191c2287SAlexey Kardashevskiy return tmp + idx; 122191c2287SAlexey Kardashevskiy } 123191c2287SAlexey Kardashevskiy 124191c2287SAlexey Kardashevskiy int pnv_tce_build(struct iommu_table *tbl, long index, long npages, 125191c2287SAlexey Kardashevskiy unsigned long uaddr, enum dma_data_direction direction, 126191c2287SAlexey Kardashevskiy unsigned long attrs) 127191c2287SAlexey Kardashevskiy { 128191c2287SAlexey Kardashevskiy u64 proto_tce = iommu_direction_to_tce_perm(direction); 129191c2287SAlexey Kardashevskiy u64 rpn = __pa(uaddr) >> tbl->it_page_shift; 130191c2287SAlexey Kardashevskiy long i; 131191c2287SAlexey Kardashevskiy 132191c2287SAlexey Kardashevskiy if (proto_tce & TCE_PCI_WRITE) 133191c2287SAlexey Kardashevskiy proto_tce |= TCE_PCI_READ; 134191c2287SAlexey Kardashevskiy 135191c2287SAlexey Kardashevskiy for (i = 0; i < npages; i++) { 136191c2287SAlexey Kardashevskiy unsigned long newtce = proto_tce | 137191c2287SAlexey Kardashevskiy ((rpn + i) << tbl->it_page_shift); 138191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset + i; 139191c2287SAlexey Kardashevskiy 140a68bd126SAlexey Kardashevskiy *(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce); 141191c2287SAlexey Kardashevskiy } 142191c2287SAlexey Kardashevskiy 143191c2287SAlexey Kardashevskiy return 0; 144191c2287SAlexey Kardashevskiy } 145191c2287SAlexey Kardashevskiy 146191c2287SAlexey Kardashevskiy #ifdef CONFIG_IOMMU_API 147191c2287SAlexey Kardashevskiy int pnv_tce_xchg(struct iommu_table *tbl, long index, 148a68bd126SAlexey Kardashevskiy unsigned long *hpa, enum dma_data_direction *direction, 149a68bd126SAlexey Kardashevskiy bool alloc) 150191c2287SAlexey Kardashevskiy { 151191c2287SAlexey Kardashevskiy u64 proto_tce = iommu_direction_to_tce_perm(*direction); 152191c2287SAlexey Kardashevskiy unsigned long newtce = *hpa | proto_tce, oldtce; 153191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset; 154a68bd126SAlexey Kardashevskiy __be64 *ptce = NULL; 155191c2287SAlexey Kardashevskiy 156191c2287SAlexey Kardashevskiy BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)); 157191c2287SAlexey Kardashevskiy 158a68bd126SAlexey Kardashevskiy if (*direction == DMA_NONE) { 159a68bd126SAlexey Kardashevskiy ptce = pnv_tce(tbl, false, idx, false); 160a68bd126SAlexey Kardashevskiy if (!ptce) { 161a68bd126SAlexey Kardashevskiy *hpa = 0; 162a68bd126SAlexey Kardashevskiy return 0; 163a68bd126SAlexey Kardashevskiy } 164a68bd126SAlexey Kardashevskiy } 165a68bd126SAlexey Kardashevskiy 166a68bd126SAlexey Kardashevskiy if (!ptce) { 167a68bd126SAlexey Kardashevskiy ptce = pnv_tce(tbl, false, idx, alloc); 168a68bd126SAlexey Kardashevskiy if (!ptce) 1695f202c1aSAlexey Kardashevskiy return -ENOMEM; 170a68bd126SAlexey Kardashevskiy } 171a68bd126SAlexey Kardashevskiy 172191c2287SAlexey Kardashevskiy if (newtce & TCE_PCI_WRITE) 173191c2287SAlexey Kardashevskiy newtce |= TCE_PCI_READ; 174191c2287SAlexey Kardashevskiy 175a68bd126SAlexey Kardashevskiy oldtce = be64_to_cpu(xchg(ptce, cpu_to_be64(newtce))); 176191c2287SAlexey Kardashevskiy *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); 177191c2287SAlexey Kardashevskiy *direction = iommu_tce_direction(oldtce); 178191c2287SAlexey Kardashevskiy 179191c2287SAlexey Kardashevskiy return 0; 180191c2287SAlexey Kardashevskiy } 181090bad39SAlexey Kardashevskiy 182a68bd126SAlexey Kardashevskiy __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc) 183090bad39SAlexey Kardashevskiy { 184090bad39SAlexey Kardashevskiy if (WARN_ON_ONCE(!tbl->it_userspace)) 185090bad39SAlexey Kardashevskiy return NULL; 186090bad39SAlexey Kardashevskiy 187a68bd126SAlexey Kardashevskiy return pnv_tce(tbl, true, index - tbl->it_offset, alloc); 188090bad39SAlexey Kardashevskiy } 189191c2287SAlexey Kardashevskiy #endif 190191c2287SAlexey Kardashevskiy 191191c2287SAlexey Kardashevskiy void pnv_tce_free(struct iommu_table *tbl, long index, long npages) 192191c2287SAlexey Kardashevskiy { 193191c2287SAlexey Kardashevskiy long i; 194191c2287SAlexey Kardashevskiy 195191c2287SAlexey Kardashevskiy for (i = 0; i < npages; i++) { 196191c2287SAlexey Kardashevskiy unsigned long idx = index - tbl->it_offset + i; 197a68bd126SAlexey Kardashevskiy __be64 *ptce = pnv_tce(tbl, false, idx, false); 198191c2287SAlexey Kardashevskiy 199a68bd126SAlexey Kardashevskiy if (ptce) 200a68bd126SAlexey Kardashevskiy *ptce = cpu_to_be64(0); 201c37c792dSAlexey Kardashevskiy else 202c37c792dSAlexey Kardashevskiy /* Skip the rest of the level */ 203c37c792dSAlexey Kardashevskiy i |= tbl->it_level_size - 1; 204191c2287SAlexey Kardashevskiy } 205191c2287SAlexey Kardashevskiy } 206191c2287SAlexey Kardashevskiy 207191c2287SAlexey Kardashevskiy unsigned long pnv_tce_get(struct iommu_table *tbl, long index) 208191c2287SAlexey Kardashevskiy { 209a68bd126SAlexey Kardashevskiy __be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false); 210a68bd126SAlexey Kardashevskiy 211a68bd126SAlexey Kardashevskiy if (!ptce) 212a68bd126SAlexey Kardashevskiy return 0; 213090bad39SAlexey Kardashevskiy 214090bad39SAlexey Kardashevskiy return be64_to_cpu(*ptce); 215191c2287SAlexey Kardashevskiy } 216191c2287SAlexey Kardashevskiy 217191c2287SAlexey Kardashevskiy static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, 218191c2287SAlexey Kardashevskiy unsigned long size, unsigned int levels) 219191c2287SAlexey Kardashevskiy { 220191c2287SAlexey Kardashevskiy const unsigned long addr_ul = (unsigned long) addr & 221191c2287SAlexey Kardashevskiy ~(TCE_PCI_READ | TCE_PCI_WRITE); 222191c2287SAlexey Kardashevskiy 223191c2287SAlexey Kardashevskiy if (levels) { 224191c2287SAlexey Kardashevskiy long i; 225191c2287SAlexey Kardashevskiy u64 *tmp = (u64 *) addr_ul; 226191c2287SAlexey Kardashevskiy 227191c2287SAlexey Kardashevskiy for (i = 0; i < size; ++i) { 228191c2287SAlexey Kardashevskiy unsigned long hpa = be64_to_cpu(tmp[i]); 229191c2287SAlexey Kardashevskiy 230191c2287SAlexey Kardashevskiy if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE))) 231191c2287SAlexey Kardashevskiy continue; 232191c2287SAlexey Kardashevskiy 233191c2287SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(__va(hpa), size, 234191c2287SAlexey Kardashevskiy levels - 1); 235191c2287SAlexey Kardashevskiy } 236191c2287SAlexey Kardashevskiy } 237191c2287SAlexey Kardashevskiy 238191c2287SAlexey Kardashevskiy free_pages(addr_ul, get_order(size << 3)); 239191c2287SAlexey Kardashevskiy } 240191c2287SAlexey Kardashevskiy 241191c2287SAlexey Kardashevskiy void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl) 242191c2287SAlexey Kardashevskiy { 243191c2287SAlexey Kardashevskiy const unsigned long size = tbl->it_indirect_levels ? 244191c2287SAlexey Kardashevskiy tbl->it_level_size : tbl->it_size; 245191c2287SAlexey Kardashevskiy 246191c2287SAlexey Kardashevskiy if (!tbl->it_size) 247191c2287SAlexey Kardashevskiy return; 248191c2287SAlexey Kardashevskiy 249191c2287SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size, 250191c2287SAlexey Kardashevskiy tbl->it_indirect_levels); 251090bad39SAlexey Kardashevskiy if (tbl->it_userspace) { 252090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size, 253090bad39SAlexey Kardashevskiy tbl->it_indirect_levels); 254090bad39SAlexey Kardashevskiy } 255191c2287SAlexey Kardashevskiy } 256191c2287SAlexey Kardashevskiy 257191c2287SAlexey Kardashevskiy static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift, 258191c2287SAlexey Kardashevskiy unsigned int levels, unsigned long limit, 259191c2287SAlexey Kardashevskiy unsigned long *current_offset, unsigned long *total_allocated) 260191c2287SAlexey Kardashevskiy { 261191c2287SAlexey Kardashevskiy __be64 *addr, *tmp; 2629bc98c8aSAlexey Kardashevskiy unsigned long allocated = 1UL << shift; 263191c2287SAlexey Kardashevskiy unsigned int entries = 1UL << (shift - 3); 264191c2287SAlexey Kardashevskiy long i; 265191c2287SAlexey Kardashevskiy 2669bc98c8aSAlexey Kardashevskiy addr = pnv_alloc_tce_level(nid, shift); 267191c2287SAlexey Kardashevskiy *total_allocated += allocated; 268191c2287SAlexey Kardashevskiy 269191c2287SAlexey Kardashevskiy --levels; 270191c2287SAlexey Kardashevskiy if (!levels) { 271191c2287SAlexey Kardashevskiy *current_offset += allocated; 272191c2287SAlexey Kardashevskiy return addr; 273191c2287SAlexey Kardashevskiy } 274191c2287SAlexey Kardashevskiy 275191c2287SAlexey Kardashevskiy for (i = 0; i < entries; ++i) { 276191c2287SAlexey Kardashevskiy tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift, 277191c2287SAlexey Kardashevskiy levels, limit, current_offset, total_allocated); 278191c2287SAlexey Kardashevskiy if (!tmp) 279191c2287SAlexey Kardashevskiy break; 280191c2287SAlexey Kardashevskiy 281191c2287SAlexey Kardashevskiy addr[i] = cpu_to_be64(__pa(tmp) | 282191c2287SAlexey Kardashevskiy TCE_PCI_READ | TCE_PCI_WRITE); 283191c2287SAlexey Kardashevskiy 284191c2287SAlexey Kardashevskiy if (*current_offset >= limit) 285191c2287SAlexey Kardashevskiy break; 286191c2287SAlexey Kardashevskiy } 287191c2287SAlexey Kardashevskiy 288191c2287SAlexey Kardashevskiy return addr; 289191c2287SAlexey Kardashevskiy } 290191c2287SAlexey Kardashevskiy 291191c2287SAlexey Kardashevskiy long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, 292191c2287SAlexey Kardashevskiy __u32 page_shift, __u64 window_size, __u32 levels, 293090bad39SAlexey Kardashevskiy bool alloc_userspace_copy, struct iommu_table *tbl) 294191c2287SAlexey Kardashevskiy { 295090bad39SAlexey Kardashevskiy void *addr, *uas = NULL; 296191c2287SAlexey Kardashevskiy unsigned long offset = 0, level_shift, total_allocated = 0; 297090bad39SAlexey Kardashevskiy unsigned long total_allocated_uas = 0; 298191c2287SAlexey Kardashevskiy const unsigned int window_shift = ilog2(window_size); 299191c2287SAlexey Kardashevskiy unsigned int entries_shift = window_shift - page_shift; 300191c2287SAlexey Kardashevskiy unsigned int table_shift = max_t(unsigned int, entries_shift + 3, 301191c2287SAlexey Kardashevskiy PAGE_SHIFT); 302191c2287SAlexey Kardashevskiy const unsigned long tce_table_size = 1UL << table_shift; 303191c2287SAlexey Kardashevskiy 304191c2287SAlexey Kardashevskiy if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS)) 305191c2287SAlexey Kardashevskiy return -EINVAL; 306191c2287SAlexey Kardashevskiy 307191c2287SAlexey Kardashevskiy if (!is_power_of_2(window_size)) 308191c2287SAlexey Kardashevskiy return -EINVAL; 309191c2287SAlexey Kardashevskiy 310191c2287SAlexey Kardashevskiy /* Adjust direct table size from window_size and levels */ 311191c2287SAlexey Kardashevskiy entries_shift = (entries_shift + levels - 1) / levels; 312191c2287SAlexey Kardashevskiy level_shift = entries_shift + 3; 313191c2287SAlexey Kardashevskiy level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT); 314191c2287SAlexey Kardashevskiy 3157233b8caSAlexey Kardashevskiy if ((level_shift - 3) * levels + page_shift >= 55) 316191c2287SAlexey Kardashevskiy return -EINVAL; 317191c2287SAlexey Kardashevskiy 318191c2287SAlexey Kardashevskiy /* Allocate TCE table */ 319191c2287SAlexey Kardashevskiy addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 320c37c792dSAlexey Kardashevskiy 1, tce_table_size, &offset, &total_allocated); 321191c2287SAlexey Kardashevskiy 322191c2287SAlexey Kardashevskiy /* addr==NULL means that the first level allocation failed */ 323191c2287SAlexey Kardashevskiy if (!addr) 324191c2287SAlexey Kardashevskiy return -ENOMEM; 325191c2287SAlexey Kardashevskiy 326191c2287SAlexey Kardashevskiy /* 327191c2287SAlexey Kardashevskiy * First level was allocated but some lower level failed as 328191c2287SAlexey Kardashevskiy * we did not allocate as much as we wanted, 329191c2287SAlexey Kardashevskiy * release partially allocated table. 330191c2287SAlexey Kardashevskiy */ 331c37c792dSAlexey Kardashevskiy if (levels == 1 && offset < tce_table_size) 332090bad39SAlexey Kardashevskiy goto free_tces_exit; 333090bad39SAlexey Kardashevskiy 334090bad39SAlexey Kardashevskiy /* Allocate userspace view of the TCE table */ 335090bad39SAlexey Kardashevskiy if (alloc_userspace_copy) { 336090bad39SAlexey Kardashevskiy offset = 0; 337090bad39SAlexey Kardashevskiy uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 338c37c792dSAlexey Kardashevskiy 1, tce_table_size, &offset, 339090bad39SAlexey Kardashevskiy &total_allocated_uas); 340090bad39SAlexey Kardashevskiy if (!uas) 341090bad39SAlexey Kardashevskiy goto free_tces_exit; 342c37c792dSAlexey Kardashevskiy if (levels == 1 && (offset < tce_table_size || 343a68bd126SAlexey Kardashevskiy total_allocated_uas != total_allocated)) 344090bad39SAlexey Kardashevskiy goto free_uas_exit; 345191c2287SAlexey Kardashevskiy } 346191c2287SAlexey Kardashevskiy 347191c2287SAlexey Kardashevskiy /* Setup linux iommu table */ 348191c2287SAlexey Kardashevskiy pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset, 349191c2287SAlexey Kardashevskiy page_shift); 350191c2287SAlexey Kardashevskiy tbl->it_level_size = 1ULL << (level_shift - 3); 351191c2287SAlexey Kardashevskiy tbl->it_indirect_levels = levels - 1; 352090bad39SAlexey Kardashevskiy tbl->it_userspace = uas; 353a68bd126SAlexey Kardashevskiy tbl->it_nid = nid; 354191c2287SAlexey Kardashevskiy 355a68bd126SAlexey Kardashevskiy pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d/%d\n", 356090bad39SAlexey Kardashevskiy window_size, tce_table_size, bus_offset, tbl->it_base, 357c37c792dSAlexey Kardashevskiy tbl->it_userspace, 1, levels); 358191c2287SAlexey Kardashevskiy 359191c2287SAlexey Kardashevskiy return 0; 360090bad39SAlexey Kardashevskiy 361090bad39SAlexey Kardashevskiy free_uas_exit: 362090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(uas, 363090bad39SAlexey Kardashevskiy 1ULL << (level_shift - 3), levels - 1); 364090bad39SAlexey Kardashevskiy free_tces_exit: 365090bad39SAlexey Kardashevskiy pnv_pci_ioda2_table_do_free_pages(addr, 366090bad39SAlexey Kardashevskiy 1ULL << (level_shift - 3), levels - 1); 367090bad39SAlexey Kardashevskiy 368090bad39SAlexey Kardashevskiy return -ENOMEM; 369191c2287SAlexey Kardashevskiy } 370191c2287SAlexey Kardashevskiy 371191c2287SAlexey Kardashevskiy void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, 372191c2287SAlexey Kardashevskiy struct iommu_table_group *table_group) 373191c2287SAlexey Kardashevskiy { 374191c2287SAlexey Kardashevskiy long i; 375191c2287SAlexey Kardashevskiy bool found; 376191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl; 377191c2287SAlexey Kardashevskiy 378191c2287SAlexey Kardashevskiy if (!tbl || !table_group) 379191c2287SAlexey Kardashevskiy return; 380191c2287SAlexey Kardashevskiy 381191c2287SAlexey Kardashevskiy /* Remove link to a group from table's list of attached groups */ 382191c2287SAlexey Kardashevskiy found = false; 383*c9790fb5SQian Cai 384*c9790fb5SQian Cai rcu_read_lock(); 385191c2287SAlexey Kardashevskiy list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { 386191c2287SAlexey Kardashevskiy if (tgl->table_group == table_group) { 387191c2287SAlexey Kardashevskiy list_del_rcu(&tgl->next); 388c312d14eSYueHaibing kfree_rcu(tgl, rcu); 389191c2287SAlexey Kardashevskiy found = true; 390191c2287SAlexey Kardashevskiy break; 391191c2287SAlexey Kardashevskiy } 392191c2287SAlexey Kardashevskiy } 393*c9790fb5SQian Cai rcu_read_unlock(); 394*c9790fb5SQian Cai 395191c2287SAlexey Kardashevskiy if (WARN_ON(!found)) 396191c2287SAlexey Kardashevskiy return; 397191c2287SAlexey Kardashevskiy 398191c2287SAlexey Kardashevskiy /* Clean a pointer to iommu_table in iommu_table_group::tables[] */ 399191c2287SAlexey Kardashevskiy found = false; 400191c2287SAlexey Kardashevskiy for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 401191c2287SAlexey Kardashevskiy if (table_group->tables[i] == tbl) { 402847e6563SAlexey Kardashevskiy iommu_tce_table_put(tbl); 403191c2287SAlexey Kardashevskiy table_group->tables[i] = NULL; 404191c2287SAlexey Kardashevskiy found = true; 405191c2287SAlexey Kardashevskiy break; 406191c2287SAlexey Kardashevskiy } 407191c2287SAlexey Kardashevskiy } 408191c2287SAlexey Kardashevskiy WARN_ON(!found); 409191c2287SAlexey Kardashevskiy } 410191c2287SAlexey Kardashevskiy 411191c2287SAlexey Kardashevskiy long pnv_pci_link_table_and_group(int node, int num, 412191c2287SAlexey Kardashevskiy struct iommu_table *tbl, 413191c2287SAlexey Kardashevskiy struct iommu_table_group *table_group) 414191c2287SAlexey Kardashevskiy { 415191c2287SAlexey Kardashevskiy struct iommu_table_group_link *tgl = NULL; 416191c2287SAlexey Kardashevskiy 417191c2287SAlexey Kardashevskiy if (WARN_ON(!tbl || !table_group)) 418191c2287SAlexey Kardashevskiy return -EINVAL; 419191c2287SAlexey Kardashevskiy 420191c2287SAlexey Kardashevskiy tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, 421191c2287SAlexey Kardashevskiy node); 422191c2287SAlexey Kardashevskiy if (!tgl) 423191c2287SAlexey Kardashevskiy return -ENOMEM; 424191c2287SAlexey Kardashevskiy 425191c2287SAlexey Kardashevskiy tgl->table_group = table_group; 426191c2287SAlexey Kardashevskiy list_add_rcu(&tgl->next, &tbl->it_group_list); 427191c2287SAlexey Kardashevskiy 428847e6563SAlexey Kardashevskiy table_group->tables[num] = iommu_tce_table_get(tbl); 429191c2287SAlexey Kardashevskiy 430191c2287SAlexey Kardashevskiy return 0; 431191c2287SAlexey Kardashevskiy } 432