1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * TCE helpers for IODA PCI/PCIe on PowerNV platforms 4 * 5 * Copyright 2018 IBM Corp. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 13 #include <linux/kernel.h> 14 #include <linux/iommu.h> 15 16 #include <asm/iommu.h> 17 #include <asm/tce.h> 18 #include "pci.h" 19 20 void pnv_pci_setup_iommu_table(struct iommu_table *tbl, 21 void *tce_mem, u64 tce_size, 22 u64 dma_offset, unsigned int page_shift) 23 { 24 tbl->it_blocksize = 16; 25 tbl->it_base = (unsigned long)tce_mem; 26 tbl->it_page_shift = page_shift; 27 tbl->it_offset = dma_offset >> tbl->it_page_shift; 28 tbl->it_index = 0; 29 tbl->it_size = tce_size >> 3; 30 tbl->it_busno = 0; 31 tbl->it_type = TCE_PCI; 32 } 33 34 static __be64 *pnv_tce(struct iommu_table *tbl, long idx) 35 { 36 __be64 *tmp = ((__be64 *)tbl->it_base); 37 int level = tbl->it_indirect_levels; 38 const long shift = ilog2(tbl->it_level_size); 39 unsigned long mask = (tbl->it_level_size - 1) << (level * shift); 40 41 while (level) { 42 int n = (idx & mask) >> (level * shift); 43 unsigned long tce = be64_to_cpu(tmp[n]); 44 45 tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE)); 46 idx &= ~mask; 47 mask >>= shift; 48 --level; 49 } 50 51 return tmp + idx; 52 } 53 54 int pnv_tce_build(struct iommu_table *tbl, long index, long npages, 55 unsigned long uaddr, enum dma_data_direction direction, 56 unsigned long attrs) 57 { 58 u64 proto_tce = iommu_direction_to_tce_perm(direction); 59 u64 rpn = __pa(uaddr) >> tbl->it_page_shift; 60 long i; 61 62 if (proto_tce & TCE_PCI_WRITE) 63 proto_tce |= TCE_PCI_READ; 64 65 for (i = 0; i < npages; i++) { 66 unsigned long newtce = proto_tce | 67 ((rpn + i) << tbl->it_page_shift); 68 unsigned long idx = index - tbl->it_offset + i; 69 70 *(pnv_tce(tbl, idx)) = cpu_to_be64(newtce); 71 } 72 73 return 0; 74 } 75 76 #ifdef CONFIG_IOMMU_API 77 int pnv_tce_xchg(struct iommu_table *tbl, long index, 78 unsigned long *hpa, enum dma_data_direction *direction) 79 { 80 u64 proto_tce = iommu_direction_to_tce_perm(*direction); 81 unsigned long newtce = *hpa | proto_tce, oldtce; 82 unsigned long idx = index - tbl->it_offset; 83 84 BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)); 85 86 if (newtce & TCE_PCI_WRITE) 87 newtce |= TCE_PCI_READ; 88 89 oldtce = be64_to_cpu(xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce))); 90 *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); 91 *direction = iommu_tce_direction(oldtce); 92 93 return 0; 94 } 95 #endif 96 97 void pnv_tce_free(struct iommu_table *tbl, long index, long npages) 98 { 99 long i; 100 101 for (i = 0; i < npages; i++) { 102 unsigned long idx = index - tbl->it_offset + i; 103 104 *(pnv_tce(tbl, idx)) = cpu_to_be64(0); 105 } 106 } 107 108 unsigned long pnv_tce_get(struct iommu_table *tbl, long index) 109 { 110 return be64_to_cpu(*(pnv_tce(tbl, index - tbl->it_offset))); 111 } 112 113 static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, 114 unsigned long size, unsigned int levels) 115 { 116 const unsigned long addr_ul = (unsigned long) addr & 117 ~(TCE_PCI_READ | TCE_PCI_WRITE); 118 119 if (levels) { 120 long i; 121 u64 *tmp = (u64 *) addr_ul; 122 123 for (i = 0; i < size; ++i) { 124 unsigned long hpa = be64_to_cpu(tmp[i]); 125 126 if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE))) 127 continue; 128 129 pnv_pci_ioda2_table_do_free_pages(__va(hpa), size, 130 levels - 1); 131 } 132 } 133 134 free_pages(addr_ul, get_order(size << 3)); 135 } 136 137 void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl) 138 { 139 const unsigned long size = tbl->it_indirect_levels ? 140 tbl->it_level_size : tbl->it_size; 141 142 if (!tbl->it_size) 143 return; 144 145 pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size, 146 tbl->it_indirect_levels); 147 } 148 149 static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift, 150 unsigned int levels, unsigned long limit, 151 unsigned long *current_offset, unsigned long *total_allocated) 152 { 153 struct page *tce_mem = NULL; 154 __be64 *addr, *tmp; 155 unsigned int order = max_t(unsigned int, shift, PAGE_SHIFT) - 156 PAGE_SHIFT; 157 unsigned long allocated = 1UL << (order + PAGE_SHIFT); 158 unsigned int entries = 1UL << (shift - 3); 159 long i; 160 161 tce_mem = alloc_pages_node(nid, GFP_KERNEL, order); 162 if (!tce_mem) { 163 pr_err("Failed to allocate a TCE memory, order=%d\n", order); 164 return NULL; 165 } 166 addr = page_address(tce_mem); 167 memset(addr, 0, allocated); 168 *total_allocated += allocated; 169 170 --levels; 171 if (!levels) { 172 *current_offset += allocated; 173 return addr; 174 } 175 176 for (i = 0; i < entries; ++i) { 177 tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift, 178 levels, limit, current_offset, total_allocated); 179 if (!tmp) 180 break; 181 182 addr[i] = cpu_to_be64(__pa(tmp) | 183 TCE_PCI_READ | TCE_PCI_WRITE); 184 185 if (*current_offset >= limit) 186 break; 187 } 188 189 return addr; 190 } 191 192 long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, 193 __u32 page_shift, __u64 window_size, __u32 levels, 194 struct iommu_table *tbl) 195 { 196 void *addr; 197 unsigned long offset = 0, level_shift, total_allocated = 0; 198 const unsigned int window_shift = ilog2(window_size); 199 unsigned int entries_shift = window_shift - page_shift; 200 unsigned int table_shift = max_t(unsigned int, entries_shift + 3, 201 PAGE_SHIFT); 202 const unsigned long tce_table_size = 1UL << table_shift; 203 204 if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS)) 205 return -EINVAL; 206 207 if (!is_power_of_2(window_size)) 208 return -EINVAL; 209 210 /* Adjust direct table size from window_size and levels */ 211 entries_shift = (entries_shift + levels - 1) / levels; 212 level_shift = entries_shift + 3; 213 level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT); 214 215 if ((level_shift - 3) * levels + page_shift >= 60) 216 return -EINVAL; 217 218 /* Allocate TCE table */ 219 addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, 220 levels, tce_table_size, &offset, &total_allocated); 221 222 /* addr==NULL means that the first level allocation failed */ 223 if (!addr) 224 return -ENOMEM; 225 226 /* 227 * First level was allocated but some lower level failed as 228 * we did not allocate as much as we wanted, 229 * release partially allocated table. 230 */ 231 if (offset < tce_table_size) { 232 pnv_pci_ioda2_table_do_free_pages(addr, 233 1ULL << (level_shift - 3), levels - 1); 234 return -ENOMEM; 235 } 236 237 /* Setup linux iommu table */ 238 pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset, 239 page_shift); 240 tbl->it_level_size = 1ULL << (level_shift - 3); 241 tbl->it_indirect_levels = levels - 1; 242 tbl->it_allocated_size = total_allocated; 243 244 pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n", 245 window_size, tce_table_size, bus_offset); 246 247 return 0; 248 } 249 250 static void pnv_iommu_table_group_link_free(struct rcu_head *head) 251 { 252 struct iommu_table_group_link *tgl = container_of(head, 253 struct iommu_table_group_link, rcu); 254 255 kfree(tgl); 256 } 257 258 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, 259 struct iommu_table_group *table_group) 260 { 261 long i; 262 bool found; 263 struct iommu_table_group_link *tgl; 264 265 if (!tbl || !table_group) 266 return; 267 268 /* Remove link to a group from table's list of attached groups */ 269 found = false; 270 list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { 271 if (tgl->table_group == table_group) { 272 list_del_rcu(&tgl->next); 273 call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free); 274 found = true; 275 break; 276 } 277 } 278 if (WARN_ON(!found)) 279 return; 280 281 /* Clean a pointer to iommu_table in iommu_table_group::tables[] */ 282 found = false; 283 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { 284 if (table_group->tables[i] == tbl) { 285 table_group->tables[i] = NULL; 286 found = true; 287 break; 288 } 289 } 290 WARN_ON(!found); 291 } 292 293 long pnv_pci_link_table_and_group(int node, int num, 294 struct iommu_table *tbl, 295 struct iommu_table_group *table_group) 296 { 297 struct iommu_table_group_link *tgl = NULL; 298 299 if (WARN_ON(!tbl || !table_group)) 300 return -EINVAL; 301 302 tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, 303 node); 304 if (!tgl) 305 return -ENOMEM; 306 307 tgl->table_group = table_group; 308 list_add_rcu(&tgl->next, &tbl->it_group_list); 309 310 table_group->tables[num] = tbl; 311 312 return 0; 313 } 314