1695093e3SVarun Sethi /* 2695093e3SVarun Sethi * This program is free software; you can redistribute it and/or modify 3695093e3SVarun Sethi * it under the terms of the GNU General Public License, version 2, as 4695093e3SVarun Sethi * published by the Free Software Foundation. 5695093e3SVarun Sethi * 6695093e3SVarun Sethi * This program is distributed in the hope that it will be useful, 7695093e3SVarun Sethi * but WITHOUT ANY WARRANTY; without even the implied warranty of 8695093e3SVarun Sethi * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9695093e3SVarun Sethi * GNU General Public License for more details. 10695093e3SVarun Sethi * 11695093e3SVarun Sethi * You should have received a copy of the GNU General Public License 12695093e3SVarun Sethi * along with this program; if not, write to the Free Software 13695093e3SVarun Sethi * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 14695093e3SVarun Sethi * 15695093e3SVarun Sethi * Copyright (C) 2013 Freescale Semiconductor, Inc. 16695093e3SVarun Sethi * Author: Varun Sethi <varun.sethi@freescale.com> 17695093e3SVarun Sethi * 18695093e3SVarun Sethi */ 19695093e3SVarun Sethi 20695093e3SVarun Sethi #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__ 21695093e3SVarun Sethi 22695093e3SVarun Sethi #include <linux/init.h> 23695093e3SVarun Sethi #include <linux/iommu.h> 24695093e3SVarun Sethi #include <linux/notifier.h> 25695093e3SVarun Sethi #include <linux/slab.h> 26695093e3SVarun Sethi #include <linux/module.h> 27695093e3SVarun Sethi #include <linux/types.h> 28695093e3SVarun Sethi #include <linux/mm.h> 29695093e3SVarun Sethi #include <linux/interrupt.h> 30695093e3SVarun Sethi #include <linux/device.h> 31695093e3SVarun Sethi #include <linux/of_platform.h> 32695093e3SVarun Sethi #include <linux/bootmem.h> 33695093e3SVarun Sethi #include <linux/err.h> 34695093e3SVarun Sethi #include <asm/io.h> 35695093e3SVarun Sethi #include <asm/bitops.h> 36695093e3SVarun Sethi 37695093e3SVarun Sethi #include <asm/pci-bridge.h> 38695093e3SVarun Sethi #include <sysdev/fsl_pci.h> 39695093e3SVarun Sethi 40695093e3SVarun Sethi #include "fsl_pamu_domain.h" 41695093e3SVarun Sethi #include "pci.h" 42695093e3SVarun Sethi 43695093e3SVarun Sethi /* 44695093e3SVarun Sethi * Global spinlock that needs to be held while 45695093e3SVarun Sethi * configuring PAMU. 46695093e3SVarun Sethi */ 47695093e3SVarun Sethi static DEFINE_SPINLOCK(iommu_lock); 48695093e3SVarun Sethi 49695093e3SVarun Sethi static struct kmem_cache *fsl_pamu_domain_cache; 50695093e3SVarun Sethi static struct kmem_cache *iommu_devinfo_cache; 51695093e3SVarun Sethi static DEFINE_SPINLOCK(device_domain_lock); 52695093e3SVarun Sethi 53695093e3SVarun Sethi static int __init iommu_init_mempool(void) 54695093e3SVarun Sethi { 55695093e3SVarun Sethi 56695093e3SVarun Sethi fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain", 57695093e3SVarun Sethi sizeof(struct fsl_dma_domain), 58695093e3SVarun Sethi 0, 59695093e3SVarun Sethi SLAB_HWCACHE_ALIGN, 60695093e3SVarun Sethi 61695093e3SVarun Sethi NULL); 62695093e3SVarun Sethi if (!fsl_pamu_domain_cache) { 63695093e3SVarun Sethi pr_debug("Couldn't create fsl iommu_domain cache\n"); 64695093e3SVarun Sethi return -ENOMEM; 65695093e3SVarun Sethi } 66695093e3SVarun Sethi 67695093e3SVarun Sethi iommu_devinfo_cache = kmem_cache_create("iommu_devinfo", 68695093e3SVarun Sethi sizeof(struct device_domain_info), 69695093e3SVarun Sethi 0, 70695093e3SVarun Sethi SLAB_HWCACHE_ALIGN, 71695093e3SVarun Sethi NULL); 72695093e3SVarun Sethi if (!iommu_devinfo_cache) { 73695093e3SVarun Sethi pr_debug("Couldn't create devinfo cache\n"); 74695093e3SVarun Sethi kmem_cache_destroy(fsl_pamu_domain_cache); 75695093e3SVarun Sethi return -ENOMEM; 76695093e3SVarun Sethi } 77695093e3SVarun Sethi 78695093e3SVarun Sethi return 0; 79695093e3SVarun Sethi } 80695093e3SVarun Sethi 81695093e3SVarun Sethi static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) 82695093e3SVarun Sethi { 83695093e3SVarun Sethi u32 win_cnt = dma_domain->win_cnt; 84695093e3SVarun Sethi struct dma_window *win_ptr = 85695093e3SVarun Sethi &dma_domain->win_arr[0]; 86695093e3SVarun Sethi struct iommu_domain_geometry *geom; 87695093e3SVarun Sethi 88695093e3SVarun Sethi geom = &dma_domain->iommu_domain->geometry; 89695093e3SVarun Sethi 90695093e3SVarun Sethi if (!win_cnt || !dma_domain->geom_size) { 91695093e3SVarun Sethi pr_debug("Number of windows/geometry not configured for the domain\n"); 92695093e3SVarun Sethi return 0; 93695093e3SVarun Sethi } 94695093e3SVarun Sethi 95695093e3SVarun Sethi if (win_cnt > 1) { 96695093e3SVarun Sethi u64 subwin_size; 97695093e3SVarun Sethi dma_addr_t subwin_iova; 98695093e3SVarun Sethi u32 wnd; 99695093e3SVarun Sethi 100695093e3SVarun Sethi subwin_size = dma_domain->geom_size >> ilog2(win_cnt); 101695093e3SVarun Sethi subwin_iova = iova & ~(subwin_size - 1); 102695093e3SVarun Sethi wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size); 103695093e3SVarun Sethi win_ptr = &dma_domain->win_arr[wnd]; 104695093e3SVarun Sethi } 105695093e3SVarun Sethi 106695093e3SVarun Sethi if (win_ptr->valid) 107695093e3SVarun Sethi return (win_ptr->paddr + (iova & (win_ptr->size - 1))); 108695093e3SVarun Sethi 109695093e3SVarun Sethi return 0; 110695093e3SVarun Sethi } 111695093e3SVarun Sethi 112695093e3SVarun Sethi static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) 113695093e3SVarun Sethi { 114695093e3SVarun Sethi struct dma_window *sub_win_ptr = 115695093e3SVarun Sethi &dma_domain->win_arr[0]; 116695093e3SVarun Sethi int i, ret; 117695093e3SVarun Sethi unsigned long rpn, flags; 118695093e3SVarun Sethi 119695093e3SVarun Sethi for (i = 0; i < dma_domain->win_cnt; i++) { 120695093e3SVarun Sethi if (sub_win_ptr[i].valid) { 121695093e3SVarun Sethi rpn = sub_win_ptr[i].paddr >> 122695093e3SVarun Sethi PAMU_PAGE_SHIFT; 123695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 124695093e3SVarun Sethi ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i, 125695093e3SVarun Sethi sub_win_ptr[i].size, 126695093e3SVarun Sethi ~(u32)0, 127695093e3SVarun Sethi rpn, 128695093e3SVarun Sethi dma_domain->snoop_id, 129695093e3SVarun Sethi dma_domain->stash_id, 130695093e3SVarun Sethi (i > 0) ? 1 : 0, 131695093e3SVarun Sethi sub_win_ptr[i].prot); 132695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 133695093e3SVarun Sethi if (ret) { 134695093e3SVarun Sethi pr_debug("PAMU SPAACE configuration failed for liodn %d\n", 135695093e3SVarun Sethi liodn); 136695093e3SVarun Sethi return ret; 137695093e3SVarun Sethi } 138695093e3SVarun Sethi } 139695093e3SVarun Sethi } 140695093e3SVarun Sethi 141695093e3SVarun Sethi return ret; 142695093e3SVarun Sethi } 143695093e3SVarun Sethi 144695093e3SVarun Sethi static int map_win(int liodn, struct fsl_dma_domain *dma_domain) 145695093e3SVarun Sethi { 146695093e3SVarun Sethi int ret; 147695093e3SVarun Sethi struct dma_window *wnd = &dma_domain->win_arr[0]; 148695093e3SVarun Sethi phys_addr_t wnd_addr = dma_domain->iommu_domain->geometry.aperture_start; 149695093e3SVarun Sethi unsigned long flags; 150695093e3SVarun Sethi 151695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 152695093e3SVarun Sethi ret = pamu_config_ppaace(liodn, wnd_addr, 153695093e3SVarun Sethi wnd->size, 154695093e3SVarun Sethi ~(u32)0, 155695093e3SVarun Sethi wnd->paddr >> PAMU_PAGE_SHIFT, 156695093e3SVarun Sethi dma_domain->snoop_id, dma_domain->stash_id, 157695093e3SVarun Sethi 0, wnd->prot); 158695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 159695093e3SVarun Sethi if (ret) 160695093e3SVarun Sethi pr_debug("PAMU PAACE configuration failed for liodn %d\n", 161695093e3SVarun Sethi liodn); 162695093e3SVarun Sethi 163695093e3SVarun Sethi return ret; 164695093e3SVarun Sethi } 165695093e3SVarun Sethi 166695093e3SVarun Sethi /* Map the DMA window corresponding to the LIODN */ 167695093e3SVarun Sethi static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) 168695093e3SVarun Sethi { 169695093e3SVarun Sethi if (dma_domain->win_cnt > 1) 170695093e3SVarun Sethi return map_subwins(liodn, dma_domain); 171695093e3SVarun Sethi else 172695093e3SVarun Sethi return map_win(liodn, dma_domain); 173695093e3SVarun Sethi 174695093e3SVarun Sethi } 175695093e3SVarun Sethi 176695093e3SVarun Sethi /* Update window/subwindow mapping for the LIODN */ 177695093e3SVarun Sethi static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr) 178695093e3SVarun Sethi { 179695093e3SVarun Sethi int ret; 180695093e3SVarun Sethi struct dma_window *wnd = &dma_domain->win_arr[wnd_nr]; 181695093e3SVarun Sethi unsigned long flags; 182695093e3SVarun Sethi 183695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 184695093e3SVarun Sethi if (dma_domain->win_cnt > 1) { 185695093e3SVarun Sethi ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr, 186695093e3SVarun Sethi wnd->size, 187695093e3SVarun Sethi ~(u32)0, 188695093e3SVarun Sethi wnd->paddr >> PAMU_PAGE_SHIFT, 189695093e3SVarun Sethi dma_domain->snoop_id, 190695093e3SVarun Sethi dma_domain->stash_id, 191695093e3SVarun Sethi (wnd_nr > 0) ? 1 : 0, 192695093e3SVarun Sethi wnd->prot); 193695093e3SVarun Sethi if (ret) 194695093e3SVarun Sethi pr_debug("Subwindow reconfiguration failed for liodn %d\n", liodn); 195695093e3SVarun Sethi } else { 196695093e3SVarun Sethi phys_addr_t wnd_addr; 197695093e3SVarun Sethi 198695093e3SVarun Sethi wnd_addr = dma_domain->iommu_domain->geometry.aperture_start; 199695093e3SVarun Sethi 200695093e3SVarun Sethi ret = pamu_config_ppaace(liodn, wnd_addr, 201695093e3SVarun Sethi wnd->size, 202695093e3SVarun Sethi ~(u32)0, 203695093e3SVarun Sethi wnd->paddr >> PAMU_PAGE_SHIFT, 204695093e3SVarun Sethi dma_domain->snoop_id, dma_domain->stash_id, 205695093e3SVarun Sethi 0, wnd->prot); 206695093e3SVarun Sethi if (ret) 207695093e3SVarun Sethi pr_debug("Window reconfiguration failed for liodn %d\n", liodn); 208695093e3SVarun Sethi } 209695093e3SVarun Sethi 210695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 211695093e3SVarun Sethi 212695093e3SVarun Sethi return ret; 213695093e3SVarun Sethi } 214695093e3SVarun Sethi 215695093e3SVarun Sethi static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, 216695093e3SVarun Sethi u32 val) 217695093e3SVarun Sethi { 218695093e3SVarun Sethi int ret = 0, i; 219695093e3SVarun Sethi unsigned long flags; 220695093e3SVarun Sethi 221695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 222695093e3SVarun Sethi if (!dma_domain->win_arr) { 223695093e3SVarun Sethi pr_debug("Windows not configured, stash destination update failed for liodn %d\n", liodn); 224695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 225695093e3SVarun Sethi return -EINVAL; 226695093e3SVarun Sethi } 227695093e3SVarun Sethi 228695093e3SVarun Sethi for (i = 0; i < dma_domain->win_cnt; i++) { 229695093e3SVarun Sethi ret = pamu_update_paace_stash(liodn, i, val); 230695093e3SVarun Sethi if (ret) { 231695093e3SVarun Sethi pr_debug("Failed to update SPAACE %d field for liodn %d\n ", i, liodn); 232695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 233695093e3SVarun Sethi return ret; 234695093e3SVarun Sethi } 235695093e3SVarun Sethi } 236695093e3SVarun Sethi 237695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 238695093e3SVarun Sethi 239695093e3SVarun Sethi return ret; 240695093e3SVarun Sethi } 241695093e3SVarun Sethi 242695093e3SVarun Sethi /* Set the geometry parameters for a LIODN */ 243695093e3SVarun Sethi static int pamu_set_liodn(int liodn, struct device *dev, 244695093e3SVarun Sethi struct fsl_dma_domain *dma_domain, 245695093e3SVarun Sethi struct iommu_domain_geometry *geom_attr, 246695093e3SVarun Sethi u32 win_cnt) 247695093e3SVarun Sethi { 248695093e3SVarun Sethi phys_addr_t window_addr, window_size; 249695093e3SVarun Sethi phys_addr_t subwin_size; 250695093e3SVarun Sethi int ret = 0, i; 251695093e3SVarun Sethi u32 omi_index = ~(u32)0; 252695093e3SVarun Sethi unsigned long flags; 253695093e3SVarun Sethi 254695093e3SVarun Sethi /* 255695093e3SVarun Sethi * Configure the omi_index at the geometry setup time. 256695093e3SVarun Sethi * This is a static value which depends on the type of 257695093e3SVarun Sethi * device and would not change thereafter. 258695093e3SVarun Sethi */ 259695093e3SVarun Sethi get_ome_index(&omi_index, dev); 260695093e3SVarun Sethi 261695093e3SVarun Sethi window_addr = geom_attr->aperture_start; 262695093e3SVarun Sethi window_size = dma_domain->geom_size; 263695093e3SVarun Sethi 264695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 265695093e3SVarun Sethi ret = pamu_disable_liodn(liodn); 266695093e3SVarun Sethi if (!ret) 267695093e3SVarun Sethi ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index, 268695093e3SVarun Sethi 0, dma_domain->snoop_id, 269695093e3SVarun Sethi dma_domain->stash_id, win_cnt, 0); 270695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 271695093e3SVarun Sethi if (ret) { 272695093e3SVarun Sethi pr_debug("PAMU PAACE configuration failed for liodn %d, win_cnt =%d\n", liodn, win_cnt); 273695093e3SVarun Sethi return ret; 274695093e3SVarun Sethi } 275695093e3SVarun Sethi 276695093e3SVarun Sethi if (win_cnt > 1) { 277695093e3SVarun Sethi subwin_size = window_size >> ilog2(win_cnt); 278695093e3SVarun Sethi for (i = 0; i < win_cnt; i++) { 279695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 280695093e3SVarun Sethi ret = pamu_disable_spaace(liodn, i); 281695093e3SVarun Sethi if (!ret) 282695093e3SVarun Sethi ret = pamu_config_spaace(liodn, win_cnt, i, 283695093e3SVarun Sethi subwin_size, omi_index, 284695093e3SVarun Sethi 0, dma_domain->snoop_id, 285695093e3SVarun Sethi dma_domain->stash_id, 286695093e3SVarun Sethi 0, 0); 287695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 288695093e3SVarun Sethi if (ret) { 289695093e3SVarun Sethi pr_debug("PAMU SPAACE configuration failed for liodn %d\n", liodn); 290695093e3SVarun Sethi return ret; 291695093e3SVarun Sethi } 292695093e3SVarun Sethi } 293695093e3SVarun Sethi } 294695093e3SVarun Sethi 295695093e3SVarun Sethi return ret; 296695093e3SVarun Sethi } 297695093e3SVarun Sethi 298695093e3SVarun Sethi static int check_size(u64 size, dma_addr_t iova) 299695093e3SVarun Sethi { 300695093e3SVarun Sethi /* 301695093e3SVarun Sethi * Size must be a power of two and at least be equal 302695093e3SVarun Sethi * to PAMU page size. 303695093e3SVarun Sethi */ 304d033f48fSVarun Sethi if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) { 305695093e3SVarun Sethi pr_debug("%s: size too small or not a power of two\n", __func__); 306695093e3SVarun Sethi return -EINVAL; 307695093e3SVarun Sethi } 308695093e3SVarun Sethi 309695093e3SVarun Sethi /* iova must be page size aligned*/ 310695093e3SVarun Sethi if (iova & (size - 1)) { 311695093e3SVarun Sethi pr_debug("%s: address is not aligned with window size\n", __func__); 312695093e3SVarun Sethi return -EINVAL; 313695093e3SVarun Sethi } 314695093e3SVarun Sethi 315695093e3SVarun Sethi return 0; 316695093e3SVarun Sethi } 317695093e3SVarun Sethi 318695093e3SVarun Sethi static struct fsl_dma_domain *iommu_alloc_dma_domain(void) 319695093e3SVarun Sethi { 320695093e3SVarun Sethi struct fsl_dma_domain *domain; 321695093e3SVarun Sethi 322695093e3SVarun Sethi domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); 323695093e3SVarun Sethi if (!domain) 324695093e3SVarun Sethi return NULL; 325695093e3SVarun Sethi 326695093e3SVarun Sethi domain->stash_id = ~(u32)0; 327695093e3SVarun Sethi domain->snoop_id = ~(u32)0; 328695093e3SVarun Sethi domain->win_cnt = pamu_get_max_subwin_cnt(); 329695093e3SVarun Sethi domain->geom_size = 0; 330695093e3SVarun Sethi 331695093e3SVarun Sethi INIT_LIST_HEAD(&domain->devices); 332695093e3SVarun Sethi 333695093e3SVarun Sethi spin_lock_init(&domain->domain_lock); 334695093e3SVarun Sethi 335695093e3SVarun Sethi return domain; 336695093e3SVarun Sethi } 337695093e3SVarun Sethi 338695093e3SVarun Sethi static inline struct device_domain_info *find_domain(struct device *dev) 339695093e3SVarun Sethi { 340695093e3SVarun Sethi return dev->archdata.iommu_domain; 341695093e3SVarun Sethi } 342695093e3SVarun Sethi 343695093e3SVarun Sethi static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) 344695093e3SVarun Sethi { 345695093e3SVarun Sethi unsigned long flags; 346695093e3SVarun Sethi 347695093e3SVarun Sethi list_del(&info->link); 348695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 349695093e3SVarun Sethi if (win_cnt > 1) 350695093e3SVarun Sethi pamu_free_subwins(info->liodn); 351695093e3SVarun Sethi pamu_disable_liodn(info->liodn); 352695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 353695093e3SVarun Sethi spin_lock_irqsave(&device_domain_lock, flags); 354695093e3SVarun Sethi info->dev->archdata.iommu_domain = NULL; 355695093e3SVarun Sethi kmem_cache_free(iommu_devinfo_cache, info); 356695093e3SVarun Sethi spin_unlock_irqrestore(&device_domain_lock, flags); 357695093e3SVarun Sethi } 358695093e3SVarun Sethi 359695093e3SVarun Sethi static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain) 360695093e3SVarun Sethi { 361695093e3SVarun Sethi struct device_domain_info *info, *tmp; 362695093e3SVarun Sethi unsigned long flags; 363695093e3SVarun Sethi 364695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 365695093e3SVarun Sethi /* Remove the device from the domain device list */ 366695093e3SVarun Sethi list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) { 367695093e3SVarun Sethi if (!dev || (info->dev == dev)) 368695093e3SVarun Sethi remove_device_ref(info, dma_domain->win_cnt); 369695093e3SVarun Sethi } 370695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 371695093e3SVarun Sethi } 372695093e3SVarun Sethi 373695093e3SVarun Sethi static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev) 374695093e3SVarun Sethi { 375695093e3SVarun Sethi struct device_domain_info *info, *old_domain_info; 376695093e3SVarun Sethi unsigned long flags; 377695093e3SVarun Sethi 378695093e3SVarun Sethi spin_lock_irqsave(&device_domain_lock, flags); 379695093e3SVarun Sethi /* 380695093e3SVarun Sethi * Check here if the device is already attached to domain or not. 381695093e3SVarun Sethi * If the device is already attached to a domain detach it. 382695093e3SVarun Sethi */ 383695093e3SVarun Sethi old_domain_info = find_domain(dev); 384695093e3SVarun Sethi if (old_domain_info && old_domain_info->domain != dma_domain) { 385695093e3SVarun Sethi spin_unlock_irqrestore(&device_domain_lock, flags); 386695093e3SVarun Sethi detach_device(dev, old_domain_info->domain); 387695093e3SVarun Sethi spin_lock_irqsave(&device_domain_lock, flags); 388695093e3SVarun Sethi } 389695093e3SVarun Sethi 390695093e3SVarun Sethi info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC); 391695093e3SVarun Sethi 392695093e3SVarun Sethi info->dev = dev; 393695093e3SVarun Sethi info->liodn = liodn; 394695093e3SVarun Sethi info->domain = dma_domain; 395695093e3SVarun Sethi 396695093e3SVarun Sethi list_add(&info->link, &dma_domain->devices); 397695093e3SVarun Sethi /* 398695093e3SVarun Sethi * In case of devices with multiple LIODNs just store 399695093e3SVarun Sethi * the info for the first LIODN as all 400695093e3SVarun Sethi * LIODNs share the same domain 401695093e3SVarun Sethi */ 402695093e3SVarun Sethi if (!old_domain_info) 403695093e3SVarun Sethi dev->archdata.iommu_domain = info; 404695093e3SVarun Sethi spin_unlock_irqrestore(&device_domain_lock, flags); 405695093e3SVarun Sethi 406695093e3SVarun Sethi } 407695093e3SVarun Sethi 408695093e3SVarun Sethi static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, 409695093e3SVarun Sethi dma_addr_t iova) 410695093e3SVarun Sethi { 411695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 412695093e3SVarun Sethi 413695093e3SVarun Sethi if ((iova < domain->geometry.aperture_start) || 414695093e3SVarun Sethi iova > (domain->geometry.aperture_end)) 415695093e3SVarun Sethi return 0; 416695093e3SVarun Sethi 417695093e3SVarun Sethi return get_phys_addr(dma_domain, iova); 418695093e3SVarun Sethi } 419695093e3SVarun Sethi 420695093e3SVarun Sethi static int fsl_pamu_domain_has_cap(struct iommu_domain *domain, 421695093e3SVarun Sethi unsigned long cap) 422695093e3SVarun Sethi { 423695093e3SVarun Sethi return cap == IOMMU_CAP_CACHE_COHERENCY; 424695093e3SVarun Sethi } 425695093e3SVarun Sethi 426695093e3SVarun Sethi static void fsl_pamu_domain_destroy(struct iommu_domain *domain) 427695093e3SVarun Sethi { 428695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 429695093e3SVarun Sethi 430695093e3SVarun Sethi domain->priv = NULL; 431695093e3SVarun Sethi 432695093e3SVarun Sethi /* remove all the devices from the device list */ 433695093e3SVarun Sethi detach_device(NULL, dma_domain); 434695093e3SVarun Sethi 435695093e3SVarun Sethi dma_domain->enabled = 0; 436695093e3SVarun Sethi dma_domain->mapped = 0; 437695093e3SVarun Sethi 438695093e3SVarun Sethi kmem_cache_free(fsl_pamu_domain_cache, dma_domain); 439695093e3SVarun Sethi } 440695093e3SVarun Sethi 441695093e3SVarun Sethi static int fsl_pamu_domain_init(struct iommu_domain *domain) 442695093e3SVarun Sethi { 443695093e3SVarun Sethi struct fsl_dma_domain *dma_domain; 444695093e3SVarun Sethi 445695093e3SVarun Sethi dma_domain = iommu_alloc_dma_domain(); 446695093e3SVarun Sethi if (!dma_domain) { 447695093e3SVarun Sethi pr_debug("dma_domain allocation failed\n"); 448695093e3SVarun Sethi return -ENOMEM; 449695093e3SVarun Sethi } 450695093e3SVarun Sethi domain->priv = dma_domain; 451695093e3SVarun Sethi dma_domain->iommu_domain = domain; 452695093e3SVarun Sethi /* defaul geometry 64 GB i.e. maximum system address */ 453695093e3SVarun Sethi domain->geometry.aperture_start = 0; 454695093e3SVarun Sethi domain->geometry.aperture_end = (1ULL << 36) - 1; 455695093e3SVarun Sethi domain->geometry.force_aperture = true; 456695093e3SVarun Sethi 457695093e3SVarun Sethi return 0; 458695093e3SVarun Sethi } 459695093e3SVarun Sethi 460695093e3SVarun Sethi /* Configure geometry settings for all LIODNs associated with domain */ 461695093e3SVarun Sethi static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain, 462695093e3SVarun Sethi struct iommu_domain_geometry *geom_attr, 463695093e3SVarun Sethi u32 win_cnt) 464695093e3SVarun Sethi { 465695093e3SVarun Sethi struct device_domain_info *info; 466695093e3SVarun Sethi int ret = 0; 467695093e3SVarun Sethi 468695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 469695093e3SVarun Sethi ret = pamu_set_liodn(info->liodn, info->dev, dma_domain, 470695093e3SVarun Sethi geom_attr, win_cnt); 471695093e3SVarun Sethi if (ret) 472695093e3SVarun Sethi break; 473695093e3SVarun Sethi } 474695093e3SVarun Sethi 475695093e3SVarun Sethi return ret; 476695093e3SVarun Sethi } 477695093e3SVarun Sethi 478695093e3SVarun Sethi /* Update stash destination for all LIODNs associated with the domain */ 479695093e3SVarun Sethi static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) 480695093e3SVarun Sethi { 481695093e3SVarun Sethi struct device_domain_info *info; 482695093e3SVarun Sethi int ret = 0; 483695093e3SVarun Sethi 484695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 485695093e3SVarun Sethi ret = update_liodn_stash(info->liodn, dma_domain, val); 486695093e3SVarun Sethi if (ret) 487695093e3SVarun Sethi break; 488695093e3SVarun Sethi } 489695093e3SVarun Sethi 490695093e3SVarun Sethi return ret; 491695093e3SVarun Sethi } 492695093e3SVarun Sethi 493695093e3SVarun Sethi /* Update domain mappings for all LIODNs associated with the domain */ 494695093e3SVarun Sethi static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 495695093e3SVarun Sethi { 496695093e3SVarun Sethi struct device_domain_info *info; 497695093e3SVarun Sethi int ret = 0; 498695093e3SVarun Sethi 499695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 500695093e3SVarun Sethi ret = update_liodn(info->liodn, dma_domain, wnd_nr); 501695093e3SVarun Sethi if (ret) 502695093e3SVarun Sethi break; 503695093e3SVarun Sethi } 504695093e3SVarun Sethi return ret; 505695093e3SVarun Sethi } 506695093e3SVarun Sethi 507695093e3SVarun Sethi static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 508695093e3SVarun Sethi { 509695093e3SVarun Sethi struct device_domain_info *info; 510695093e3SVarun Sethi int ret = 0; 511695093e3SVarun Sethi 512695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 513695093e3SVarun Sethi if (dma_domain->win_cnt == 1 && dma_domain->enabled) { 514695093e3SVarun Sethi ret = pamu_disable_liodn(info->liodn); 515695093e3SVarun Sethi if (!ret) 516695093e3SVarun Sethi dma_domain->enabled = 0; 517695093e3SVarun Sethi } else { 518695093e3SVarun Sethi ret = pamu_disable_spaace(info->liodn, wnd_nr); 519695093e3SVarun Sethi } 520695093e3SVarun Sethi } 521695093e3SVarun Sethi 522695093e3SVarun Sethi return ret; 523695093e3SVarun Sethi } 524695093e3SVarun Sethi 525695093e3SVarun Sethi static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr) 526695093e3SVarun Sethi { 527695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 528695093e3SVarun Sethi unsigned long flags; 529695093e3SVarun Sethi int ret; 530695093e3SVarun Sethi 531695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 532695093e3SVarun Sethi if (!dma_domain->win_arr) { 533695093e3SVarun Sethi pr_debug("Number of windows not configured\n"); 534695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 535695093e3SVarun Sethi return; 536695093e3SVarun Sethi } 537695093e3SVarun Sethi 538695093e3SVarun Sethi if (wnd_nr >= dma_domain->win_cnt) { 539695093e3SVarun Sethi pr_debug("Invalid window index\n"); 540695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 541695093e3SVarun Sethi return; 542695093e3SVarun Sethi } 543695093e3SVarun Sethi 544695093e3SVarun Sethi if (dma_domain->win_arr[wnd_nr].valid) { 545695093e3SVarun Sethi ret = disable_domain_win(dma_domain, wnd_nr); 546695093e3SVarun Sethi if (!ret) { 547695093e3SVarun Sethi dma_domain->win_arr[wnd_nr].valid = 0; 548695093e3SVarun Sethi dma_domain->mapped--; 549695093e3SVarun Sethi } 550695093e3SVarun Sethi } 551695093e3SVarun Sethi 552695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 553695093e3SVarun Sethi 554695093e3SVarun Sethi } 555695093e3SVarun Sethi 556695093e3SVarun Sethi static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, 557695093e3SVarun Sethi phys_addr_t paddr, u64 size, int prot) 558695093e3SVarun Sethi { 559695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 560695093e3SVarun Sethi struct dma_window *wnd; 561695093e3SVarun Sethi int pamu_prot = 0; 562695093e3SVarun Sethi int ret; 563695093e3SVarun Sethi unsigned long flags; 564695093e3SVarun Sethi u64 win_size; 565695093e3SVarun Sethi 566695093e3SVarun Sethi if (prot & IOMMU_READ) 567695093e3SVarun Sethi pamu_prot |= PAACE_AP_PERMS_QUERY; 568695093e3SVarun Sethi if (prot & IOMMU_WRITE) 569695093e3SVarun Sethi pamu_prot |= PAACE_AP_PERMS_UPDATE; 570695093e3SVarun Sethi 571695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 572695093e3SVarun Sethi if (!dma_domain->win_arr) { 573695093e3SVarun Sethi pr_debug("Number of windows not configured\n"); 574695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 575695093e3SVarun Sethi return -ENODEV; 576695093e3SVarun Sethi } 577695093e3SVarun Sethi 578695093e3SVarun Sethi if (wnd_nr >= dma_domain->win_cnt) { 579695093e3SVarun Sethi pr_debug("Invalid window index\n"); 580695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 581695093e3SVarun Sethi return -EINVAL; 582695093e3SVarun Sethi } 583695093e3SVarun Sethi 584695093e3SVarun Sethi win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt); 585695093e3SVarun Sethi if (size > win_size) { 586695093e3SVarun Sethi pr_debug("Invalid window size \n"); 587695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 588695093e3SVarun Sethi return -EINVAL; 589695093e3SVarun Sethi } 590695093e3SVarun Sethi 591695093e3SVarun Sethi if (dma_domain->win_cnt == 1) { 592695093e3SVarun Sethi if (dma_domain->enabled) { 593695093e3SVarun Sethi pr_debug("Disable the window before updating the mapping\n"); 594695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 595695093e3SVarun Sethi return -EBUSY; 596695093e3SVarun Sethi } 597695093e3SVarun Sethi 598695093e3SVarun Sethi ret = check_size(size, domain->geometry.aperture_start); 599695093e3SVarun Sethi if (ret) { 600695093e3SVarun Sethi pr_debug("Aperture start not aligned to the size\n"); 601695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 602695093e3SVarun Sethi return -EINVAL; 603695093e3SVarun Sethi } 604695093e3SVarun Sethi } 605695093e3SVarun Sethi 606695093e3SVarun Sethi wnd = &dma_domain->win_arr[wnd_nr]; 607695093e3SVarun Sethi if (!wnd->valid) { 608695093e3SVarun Sethi wnd->paddr = paddr; 609695093e3SVarun Sethi wnd->size = size; 610695093e3SVarun Sethi wnd->prot = pamu_prot; 611695093e3SVarun Sethi 612695093e3SVarun Sethi ret = update_domain_mapping(dma_domain, wnd_nr); 613695093e3SVarun Sethi if (!ret) { 614695093e3SVarun Sethi wnd->valid = 1; 615695093e3SVarun Sethi dma_domain->mapped++; 616695093e3SVarun Sethi } 617695093e3SVarun Sethi } else { 618695093e3SVarun Sethi pr_debug("Disable the window before updating the mapping\n"); 619695093e3SVarun Sethi ret = -EBUSY; 620695093e3SVarun Sethi } 621695093e3SVarun Sethi 622695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 623695093e3SVarun Sethi 624695093e3SVarun Sethi return ret; 625695093e3SVarun Sethi } 626695093e3SVarun Sethi 627695093e3SVarun Sethi /* 628695093e3SVarun Sethi * Attach the LIODN to the DMA domain and configure the geometry 629695093e3SVarun Sethi * and window mappings. 630695093e3SVarun Sethi */ 631695093e3SVarun Sethi static int handle_attach_device(struct fsl_dma_domain *dma_domain, 632695093e3SVarun Sethi struct device *dev, const u32 *liodn, 633695093e3SVarun Sethi int num) 634695093e3SVarun Sethi { 635695093e3SVarun Sethi unsigned long flags; 636695093e3SVarun Sethi struct iommu_domain *domain = dma_domain->iommu_domain; 637695093e3SVarun Sethi int ret = 0; 638695093e3SVarun Sethi int i; 639695093e3SVarun Sethi 640695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 641695093e3SVarun Sethi for (i = 0; i < num; i++) { 642695093e3SVarun Sethi 643695093e3SVarun Sethi /* Ensure that LIODN value is valid */ 644695093e3SVarun Sethi if (liodn[i] >= PAACE_NUMBER_ENTRIES) { 645695093e3SVarun Sethi pr_debug("Invalid liodn %d, attach device failed for %s\n", 646695093e3SVarun Sethi liodn[i], dev->of_node->full_name); 647695093e3SVarun Sethi ret = -EINVAL; 648695093e3SVarun Sethi break; 649695093e3SVarun Sethi } 650695093e3SVarun Sethi 651695093e3SVarun Sethi attach_device(dma_domain, liodn[i], dev); 652695093e3SVarun Sethi /* 653695093e3SVarun Sethi * Check if geometry has already been configured 654695093e3SVarun Sethi * for the domain. If yes, set the geometry for 655695093e3SVarun Sethi * the LIODN. 656695093e3SVarun Sethi */ 657695093e3SVarun Sethi if (dma_domain->win_arr) { 658695093e3SVarun Sethi u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0; 659695093e3SVarun Sethi ret = pamu_set_liodn(liodn[i], dev, dma_domain, 660695093e3SVarun Sethi &domain->geometry, 661695093e3SVarun Sethi win_cnt); 662695093e3SVarun Sethi if (ret) 663695093e3SVarun Sethi break; 664695093e3SVarun Sethi if (dma_domain->mapped) { 665695093e3SVarun Sethi /* 666695093e3SVarun Sethi * Create window/subwindow mapping for 667695093e3SVarun Sethi * the LIODN. 668695093e3SVarun Sethi */ 669695093e3SVarun Sethi ret = map_liodn(liodn[i], dma_domain); 670695093e3SVarun Sethi if (ret) 671695093e3SVarun Sethi break; 672695093e3SVarun Sethi } 673695093e3SVarun Sethi } 674695093e3SVarun Sethi } 675695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 676695093e3SVarun Sethi 677695093e3SVarun Sethi return ret; 678695093e3SVarun Sethi } 679695093e3SVarun Sethi 680695093e3SVarun Sethi static int fsl_pamu_attach_device(struct iommu_domain *domain, 681695093e3SVarun Sethi struct device *dev) 682695093e3SVarun Sethi { 683695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 684695093e3SVarun Sethi const u32 *liodn; 685695093e3SVarun Sethi u32 liodn_cnt; 686695093e3SVarun Sethi int len, ret = 0; 687695093e3SVarun Sethi struct pci_dev *pdev = NULL; 688695093e3SVarun Sethi struct pci_controller *pci_ctl; 689695093e3SVarun Sethi 690695093e3SVarun Sethi /* 691695093e3SVarun Sethi * Use LIODN of the PCI controller while attaching a 692695093e3SVarun Sethi * PCI device. 693695093e3SVarun Sethi */ 694b3eb76d1SYijing Wang if (dev_is_pci(dev)) { 695695093e3SVarun Sethi pdev = to_pci_dev(dev); 696695093e3SVarun Sethi pci_ctl = pci_bus_to_host(pdev->bus); 697695093e3SVarun Sethi /* 698695093e3SVarun Sethi * make dev point to pci controller device 699695093e3SVarun Sethi * so we can get the LIODN programmed by 700695093e3SVarun Sethi * u-boot. 701695093e3SVarun Sethi */ 702695093e3SVarun Sethi dev = pci_ctl->parent; 703695093e3SVarun Sethi } 704695093e3SVarun Sethi 705695093e3SVarun Sethi liodn = of_get_property(dev->of_node, "fsl,liodn", &len); 706695093e3SVarun Sethi if (liodn) { 707695093e3SVarun Sethi liodn_cnt = len / sizeof(u32); 708695093e3SVarun Sethi ret = handle_attach_device(dma_domain, dev, 709695093e3SVarun Sethi liodn, liodn_cnt); 710695093e3SVarun Sethi } else { 711695093e3SVarun Sethi pr_debug("missing fsl,liodn property at %s\n", 712695093e3SVarun Sethi dev->of_node->full_name); 713695093e3SVarun Sethi ret = -EINVAL; 714695093e3SVarun Sethi } 715695093e3SVarun Sethi 716695093e3SVarun Sethi return ret; 717695093e3SVarun Sethi } 718695093e3SVarun Sethi 719695093e3SVarun Sethi static void fsl_pamu_detach_device(struct iommu_domain *domain, 720695093e3SVarun Sethi struct device *dev) 721695093e3SVarun Sethi { 722695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 723695093e3SVarun Sethi const u32 *prop; 724695093e3SVarun Sethi int len; 725695093e3SVarun Sethi struct pci_dev *pdev = NULL; 726695093e3SVarun Sethi struct pci_controller *pci_ctl; 727695093e3SVarun Sethi 728695093e3SVarun Sethi /* 729695093e3SVarun Sethi * Use LIODN of the PCI controller while detaching a 730695093e3SVarun Sethi * PCI device. 731695093e3SVarun Sethi */ 732b3eb76d1SYijing Wang if (dev_is_pci(dev)) { 733695093e3SVarun Sethi pdev = to_pci_dev(dev); 734695093e3SVarun Sethi pci_ctl = pci_bus_to_host(pdev->bus); 735695093e3SVarun Sethi /* 736695093e3SVarun Sethi * make dev point to pci controller device 737695093e3SVarun Sethi * so we can get the LIODN programmed by 738695093e3SVarun Sethi * u-boot. 739695093e3SVarun Sethi */ 740695093e3SVarun Sethi dev = pci_ctl->parent; 741695093e3SVarun Sethi } 742695093e3SVarun Sethi 743695093e3SVarun Sethi prop = of_get_property(dev->of_node, "fsl,liodn", &len); 744695093e3SVarun Sethi if (prop) 745695093e3SVarun Sethi detach_device(dev, dma_domain); 746695093e3SVarun Sethi else 747695093e3SVarun Sethi pr_debug("missing fsl,liodn property at %s\n", 748695093e3SVarun Sethi dev->of_node->full_name); 749695093e3SVarun Sethi } 750695093e3SVarun Sethi 751695093e3SVarun Sethi static int configure_domain_geometry(struct iommu_domain *domain, void *data) 752695093e3SVarun Sethi { 753695093e3SVarun Sethi struct iommu_domain_geometry *geom_attr = data; 754695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 755695093e3SVarun Sethi dma_addr_t geom_size; 756695093e3SVarun Sethi unsigned long flags; 757695093e3SVarun Sethi 758695093e3SVarun Sethi geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1; 759695093e3SVarun Sethi /* 760695093e3SVarun Sethi * Sanity check the geometry size. Also, we do not support 761695093e3SVarun Sethi * DMA outside of the geometry. 762695093e3SVarun Sethi */ 763695093e3SVarun Sethi if (check_size(geom_size, geom_attr->aperture_start) || 764695093e3SVarun Sethi !geom_attr->force_aperture) { 765695093e3SVarun Sethi pr_debug("Invalid PAMU geometry attributes\n"); 766695093e3SVarun Sethi return -EINVAL; 767695093e3SVarun Sethi } 768695093e3SVarun Sethi 769695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 770695093e3SVarun Sethi if (dma_domain->enabled) { 771695093e3SVarun Sethi pr_debug("Can't set geometry attributes as domain is active\n"); 772695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 773695093e3SVarun Sethi return -EBUSY; 774695093e3SVarun Sethi } 775695093e3SVarun Sethi 776695093e3SVarun Sethi /* Copy the domain geometry information */ 777695093e3SVarun Sethi memcpy(&domain->geometry, geom_attr, 778695093e3SVarun Sethi sizeof(struct iommu_domain_geometry)); 779695093e3SVarun Sethi dma_domain->geom_size = geom_size; 780695093e3SVarun Sethi 781695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 782695093e3SVarun Sethi 783695093e3SVarun Sethi return 0; 784695093e3SVarun Sethi } 785695093e3SVarun Sethi 786695093e3SVarun Sethi /* Set the domain stash attribute */ 787695093e3SVarun Sethi static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) 788695093e3SVarun Sethi { 789695093e3SVarun Sethi struct pamu_stash_attribute *stash_attr = data; 790695093e3SVarun Sethi unsigned long flags; 791695093e3SVarun Sethi int ret; 792695093e3SVarun Sethi 793695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 794695093e3SVarun Sethi 795695093e3SVarun Sethi memcpy(&dma_domain->dma_stash, stash_attr, 796695093e3SVarun Sethi sizeof(struct pamu_stash_attribute)); 797695093e3SVarun Sethi 798695093e3SVarun Sethi dma_domain->stash_id = get_stash_id(stash_attr->cache, 799695093e3SVarun Sethi stash_attr->cpu); 800695093e3SVarun Sethi if (dma_domain->stash_id == ~(u32)0) { 801695093e3SVarun Sethi pr_debug("Invalid stash attributes\n"); 802695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 803695093e3SVarun Sethi return -EINVAL; 804695093e3SVarun Sethi } 805695093e3SVarun Sethi 806695093e3SVarun Sethi ret = update_domain_stash(dma_domain, dma_domain->stash_id); 807695093e3SVarun Sethi 808695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 809695093e3SVarun Sethi 810695093e3SVarun Sethi return ret; 811695093e3SVarun Sethi } 812695093e3SVarun Sethi 813695093e3SVarun Sethi /* Configure domain dma state i.e. enable/disable DMA*/ 814695093e3SVarun Sethi static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable) 815695093e3SVarun Sethi { 816695093e3SVarun Sethi struct device_domain_info *info; 817695093e3SVarun Sethi unsigned long flags; 818695093e3SVarun Sethi int ret; 819695093e3SVarun Sethi 820695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 821695093e3SVarun Sethi 822695093e3SVarun Sethi if (enable && !dma_domain->mapped) { 823695093e3SVarun Sethi pr_debug("Can't enable DMA domain without valid mapping\n"); 824695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 825695093e3SVarun Sethi return -ENODEV; 826695093e3SVarun Sethi } 827695093e3SVarun Sethi 828695093e3SVarun Sethi dma_domain->enabled = enable; 829695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, 830695093e3SVarun Sethi link) { 831695093e3SVarun Sethi ret = (enable) ? pamu_enable_liodn(info->liodn) : 832695093e3SVarun Sethi pamu_disable_liodn(info->liodn); 833695093e3SVarun Sethi if (ret) 834695093e3SVarun Sethi pr_debug("Unable to set dma state for liodn %d", 835695093e3SVarun Sethi info->liodn); 836695093e3SVarun Sethi } 837695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 838695093e3SVarun Sethi 839695093e3SVarun Sethi return 0; 840695093e3SVarun Sethi } 841695093e3SVarun Sethi 842695093e3SVarun Sethi static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, 843695093e3SVarun Sethi enum iommu_attr attr_type, void *data) 844695093e3SVarun Sethi { 845695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 846695093e3SVarun Sethi int ret = 0; 847695093e3SVarun Sethi 848695093e3SVarun Sethi 849695093e3SVarun Sethi switch (attr_type) { 850695093e3SVarun Sethi case DOMAIN_ATTR_GEOMETRY: 851695093e3SVarun Sethi ret = configure_domain_geometry(domain, data); 852695093e3SVarun Sethi break; 853695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_STASH: 854695093e3SVarun Sethi ret = configure_domain_stash(dma_domain, data); 855695093e3SVarun Sethi break; 856695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_ENABLE: 857695093e3SVarun Sethi ret = configure_domain_dma_state(dma_domain, *(int *)data); 858695093e3SVarun Sethi break; 859695093e3SVarun Sethi default: 860695093e3SVarun Sethi pr_debug("Unsupported attribute type\n"); 861695093e3SVarun Sethi ret = -EINVAL; 862695093e3SVarun Sethi break; 863695093e3SVarun Sethi }; 864695093e3SVarun Sethi 865695093e3SVarun Sethi return ret; 866695093e3SVarun Sethi } 867695093e3SVarun Sethi 868695093e3SVarun Sethi static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, 869695093e3SVarun Sethi enum iommu_attr attr_type, void *data) 870695093e3SVarun Sethi { 871695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 872695093e3SVarun Sethi int ret = 0; 873695093e3SVarun Sethi 874695093e3SVarun Sethi 875695093e3SVarun Sethi switch (attr_type) { 876695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_STASH: 877695093e3SVarun Sethi memcpy((struct pamu_stash_attribute *) data, &dma_domain->dma_stash, 878695093e3SVarun Sethi sizeof(struct pamu_stash_attribute)); 879695093e3SVarun Sethi break; 880695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_ENABLE: 881695093e3SVarun Sethi *(int *)data = dma_domain->enabled; 882695093e3SVarun Sethi break; 883695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMUV1: 884695093e3SVarun Sethi *(int *)data = DOMAIN_ATTR_FSL_PAMUV1; 885695093e3SVarun Sethi break; 886695093e3SVarun Sethi default: 887695093e3SVarun Sethi pr_debug("Unsupported attribute type\n"); 888695093e3SVarun Sethi ret = -EINVAL; 889695093e3SVarun Sethi break; 890695093e3SVarun Sethi }; 891695093e3SVarun Sethi 892695093e3SVarun Sethi return ret; 893695093e3SVarun Sethi } 894695093e3SVarun Sethi 895695093e3SVarun Sethi #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) 896695093e3SVarun Sethi 897695093e3SVarun Sethi static struct iommu_group *get_device_iommu_group(struct device *dev) 898695093e3SVarun Sethi { 899695093e3SVarun Sethi struct iommu_group *group; 900695093e3SVarun Sethi 901695093e3SVarun Sethi group = iommu_group_get(dev); 902695093e3SVarun Sethi if (!group) 903695093e3SVarun Sethi group = iommu_group_alloc(); 904695093e3SVarun Sethi 905695093e3SVarun Sethi return group; 906695093e3SVarun Sethi } 907695093e3SVarun Sethi 908695093e3SVarun Sethi static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl) 909695093e3SVarun Sethi { 910695093e3SVarun Sethi u32 version; 911695093e3SVarun Sethi 912695093e3SVarun Sethi /* Check the PCI controller version number by readding BRR1 register */ 913695093e3SVarun Sethi version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2)); 914695093e3SVarun Sethi version &= PCI_FSL_BRR1_VER; 915695093e3SVarun Sethi /* If PCI controller version is >= 0x204 we can partition endpoints*/ 916695093e3SVarun Sethi if (version >= 0x204) 917695093e3SVarun Sethi return 1; 918695093e3SVarun Sethi 919695093e3SVarun Sethi return 0; 920695093e3SVarun Sethi } 921695093e3SVarun Sethi 922695093e3SVarun Sethi /* Get iommu group information from peer devices or devices on the parent bus */ 923695093e3SVarun Sethi static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev) 924695093e3SVarun Sethi { 925695093e3SVarun Sethi struct pci_dev *tmp; 926695093e3SVarun Sethi struct iommu_group *group; 927695093e3SVarun Sethi struct pci_bus *bus = pdev->bus; 928695093e3SVarun Sethi 929695093e3SVarun Sethi /* 930695093e3SVarun Sethi * Traverese the pci bus device list to get 931695093e3SVarun Sethi * the shared iommu group. 932695093e3SVarun Sethi */ 933695093e3SVarun Sethi while (bus) { 934695093e3SVarun Sethi list_for_each_entry(tmp, &bus->devices, bus_list) { 935695093e3SVarun Sethi if (tmp == pdev) 936695093e3SVarun Sethi continue; 937695093e3SVarun Sethi group = iommu_group_get(&tmp->dev); 938695093e3SVarun Sethi if (group) 939695093e3SVarun Sethi return group; 940695093e3SVarun Sethi } 941695093e3SVarun Sethi 942695093e3SVarun Sethi bus = bus->parent; 943695093e3SVarun Sethi } 944695093e3SVarun Sethi 945695093e3SVarun Sethi return NULL; 946695093e3SVarun Sethi } 947695093e3SVarun Sethi 948695093e3SVarun Sethi static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) 949695093e3SVarun Sethi { 950695093e3SVarun Sethi struct pci_controller *pci_ctl; 951695093e3SVarun Sethi bool pci_endpt_partioning; 952695093e3SVarun Sethi struct iommu_group *group = NULL; 953695093e3SVarun Sethi struct pci_dev *bridge, *dma_pdev = NULL; 954695093e3SVarun Sethi 955695093e3SVarun Sethi pci_ctl = pci_bus_to_host(pdev->bus); 956695093e3SVarun Sethi pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl); 957695093e3SVarun Sethi /* We can partition PCIe devices so assign device group to the device */ 958695093e3SVarun Sethi if (pci_endpt_partioning) { 959695093e3SVarun Sethi bridge = pci_find_upstream_pcie_bridge(pdev); 960695093e3SVarun Sethi if (bridge) { 961695093e3SVarun Sethi if (pci_is_pcie(bridge)) 962695093e3SVarun Sethi dma_pdev = pci_get_domain_bus_and_slot( 963695093e3SVarun Sethi pci_domain_nr(pdev->bus), 964695093e3SVarun Sethi bridge->subordinate->number, 0); 965695093e3SVarun Sethi if (!dma_pdev) 966695093e3SVarun Sethi dma_pdev = pci_dev_get(bridge); 967695093e3SVarun Sethi } else 968695093e3SVarun Sethi dma_pdev = pci_dev_get(pdev); 969695093e3SVarun Sethi 970695093e3SVarun Sethi /* Account for quirked devices */ 971695093e3SVarun Sethi swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev)); 972695093e3SVarun Sethi 973695093e3SVarun Sethi /* 974695093e3SVarun Sethi * If it's a multifunction device that does not support our 975695093e3SVarun Sethi * required ACS flags, add to the same group as lowest numbered 976695093e3SVarun Sethi * function that also does not suport the required ACS flags. 977695093e3SVarun Sethi */ 978695093e3SVarun Sethi if (dma_pdev->multifunction && 979695093e3SVarun Sethi !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) { 980695093e3SVarun Sethi u8 i, slot = PCI_SLOT(dma_pdev->devfn); 981695093e3SVarun Sethi 982695093e3SVarun Sethi for (i = 0; i < 8; i++) { 983695093e3SVarun Sethi struct pci_dev *tmp; 984695093e3SVarun Sethi 985695093e3SVarun Sethi tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i)); 986695093e3SVarun Sethi if (!tmp) 987695093e3SVarun Sethi continue; 988695093e3SVarun Sethi 989695093e3SVarun Sethi if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) { 990695093e3SVarun Sethi swap_pci_ref(&dma_pdev, tmp); 991695093e3SVarun Sethi break; 992695093e3SVarun Sethi } 993695093e3SVarun Sethi pci_dev_put(tmp); 994695093e3SVarun Sethi } 995695093e3SVarun Sethi } 996695093e3SVarun Sethi 997695093e3SVarun Sethi /* 998695093e3SVarun Sethi * Devices on the root bus go through the iommu. If that's not us, 999695093e3SVarun Sethi * find the next upstream device and test ACS up to the root bus. 1000695093e3SVarun Sethi * Finding the next device may require skipping virtual buses. 1001695093e3SVarun Sethi */ 1002695093e3SVarun Sethi while (!pci_is_root_bus(dma_pdev->bus)) { 1003695093e3SVarun Sethi struct pci_bus *bus = dma_pdev->bus; 1004695093e3SVarun Sethi 1005695093e3SVarun Sethi while (!bus->self) { 1006695093e3SVarun Sethi if (!pci_is_root_bus(bus)) 1007695093e3SVarun Sethi bus = bus->parent; 1008695093e3SVarun Sethi else 1009695093e3SVarun Sethi goto root_bus; 1010695093e3SVarun Sethi } 1011695093e3SVarun Sethi 1012695093e3SVarun Sethi if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS)) 1013695093e3SVarun Sethi break; 1014695093e3SVarun Sethi 1015695093e3SVarun Sethi swap_pci_ref(&dma_pdev, pci_dev_get(bus->self)); 1016695093e3SVarun Sethi } 1017695093e3SVarun Sethi 1018695093e3SVarun Sethi root_bus: 1019695093e3SVarun Sethi group = get_device_iommu_group(&dma_pdev->dev); 1020695093e3SVarun Sethi pci_dev_put(dma_pdev); 1021695093e3SVarun Sethi /* 1022695093e3SVarun Sethi * PCIe controller is not a paritionable entity 1023695093e3SVarun Sethi * free the controller device iommu_group. 1024695093e3SVarun Sethi */ 1025695093e3SVarun Sethi if (pci_ctl->parent->iommu_group) 1026695093e3SVarun Sethi iommu_group_remove_device(pci_ctl->parent); 1027695093e3SVarun Sethi } else { 1028695093e3SVarun Sethi /* 1029695093e3SVarun Sethi * All devices connected to the controller will share the 1030695093e3SVarun Sethi * PCI controllers device group. If this is the first 1031695093e3SVarun Sethi * device to be probed for the pci controller, copy the 1032695093e3SVarun Sethi * device group information from the PCI controller device 1033695093e3SVarun Sethi * node and remove the PCI controller iommu group. 1034695093e3SVarun Sethi * For subsequent devices, the iommu group information can 1035695093e3SVarun Sethi * be obtained from sibling devices (i.e. from the bus_devices 1036695093e3SVarun Sethi * link list). 1037695093e3SVarun Sethi */ 1038695093e3SVarun Sethi if (pci_ctl->parent->iommu_group) { 1039695093e3SVarun Sethi group = get_device_iommu_group(pci_ctl->parent); 1040695093e3SVarun Sethi iommu_group_remove_device(pci_ctl->parent); 1041695093e3SVarun Sethi } else 1042695093e3SVarun Sethi group = get_shared_pci_device_group(pdev); 1043695093e3SVarun Sethi } 1044695093e3SVarun Sethi 1045695093e3SVarun Sethi return group; 1046695093e3SVarun Sethi } 1047695093e3SVarun Sethi 1048695093e3SVarun Sethi static int fsl_pamu_add_device(struct device *dev) 1049695093e3SVarun Sethi { 1050695093e3SVarun Sethi struct iommu_group *group = NULL; 1051695093e3SVarun Sethi struct pci_dev *pdev; 1052695093e3SVarun Sethi const u32 *prop; 1053695093e3SVarun Sethi int ret, len; 1054695093e3SVarun Sethi 1055695093e3SVarun Sethi /* 1056695093e3SVarun Sethi * For platform devices we allocate a separate group for 1057695093e3SVarun Sethi * each of the devices. 1058695093e3SVarun Sethi */ 1059b3eb76d1SYijing Wang if (dev_is_pci(dev)) { 1060695093e3SVarun Sethi pdev = to_pci_dev(dev); 1061695093e3SVarun Sethi /* Don't create device groups for virtual PCI bridges */ 1062695093e3SVarun Sethi if (pdev->subordinate) 1063695093e3SVarun Sethi return 0; 1064695093e3SVarun Sethi 1065695093e3SVarun Sethi group = get_pci_device_group(pdev); 1066695093e3SVarun Sethi 1067695093e3SVarun Sethi } else { 1068695093e3SVarun Sethi prop = of_get_property(dev->of_node, "fsl,liodn", &len); 1069695093e3SVarun Sethi if (prop) 1070695093e3SVarun Sethi group = get_device_iommu_group(dev); 1071695093e3SVarun Sethi } 1072695093e3SVarun Sethi 1073695093e3SVarun Sethi if (!group || IS_ERR(group)) 1074695093e3SVarun Sethi return PTR_ERR(group); 1075695093e3SVarun Sethi 1076695093e3SVarun Sethi ret = iommu_group_add_device(group, dev); 1077695093e3SVarun Sethi 1078695093e3SVarun Sethi iommu_group_put(group); 1079695093e3SVarun Sethi return ret; 1080695093e3SVarun Sethi } 1081695093e3SVarun Sethi 1082695093e3SVarun Sethi static void fsl_pamu_remove_device(struct device *dev) 1083695093e3SVarun Sethi { 1084695093e3SVarun Sethi iommu_group_remove_device(dev); 1085695093e3SVarun Sethi } 1086695093e3SVarun Sethi 1087695093e3SVarun Sethi static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) 1088695093e3SVarun Sethi { 1089695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 1090695093e3SVarun Sethi unsigned long flags; 1091695093e3SVarun Sethi int ret; 1092695093e3SVarun Sethi 1093695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 1094695093e3SVarun Sethi /* Ensure domain is inactive i.e. DMA should be disabled for the domain */ 1095695093e3SVarun Sethi if (dma_domain->enabled) { 1096695093e3SVarun Sethi pr_debug("Can't set geometry attributes as domain is active\n"); 1097695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1098695093e3SVarun Sethi return -EBUSY; 1099695093e3SVarun Sethi } 1100695093e3SVarun Sethi 1101695093e3SVarun Sethi /* Ensure that the geometry has been set for the domain */ 1102695093e3SVarun Sethi if (!dma_domain->geom_size) { 1103695093e3SVarun Sethi pr_debug("Please configure geometry before setting the number of windows\n"); 1104695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1105695093e3SVarun Sethi return -EINVAL; 1106695093e3SVarun Sethi } 1107695093e3SVarun Sethi 1108695093e3SVarun Sethi /* 1109695093e3SVarun Sethi * Ensure we have valid window count i.e. it should be less than 1110695093e3SVarun Sethi * maximum permissible limit and should be a power of two. 1111695093e3SVarun Sethi */ 1112695093e3SVarun Sethi if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) { 1113695093e3SVarun Sethi pr_debug("Invalid window count\n"); 1114695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1115695093e3SVarun Sethi return -EINVAL; 1116695093e3SVarun Sethi } 1117695093e3SVarun Sethi 1118695093e3SVarun Sethi ret = pamu_set_domain_geometry(dma_domain, &domain->geometry, 1119695093e3SVarun Sethi ((w_count > 1) ? w_count : 0)); 1120695093e3SVarun Sethi if (!ret) { 1121695093e3SVarun Sethi if (dma_domain->win_arr) 1122695093e3SVarun Sethi kfree(dma_domain->win_arr); 1123695093e3SVarun Sethi dma_domain->win_arr = kzalloc(sizeof(struct dma_window) * 1124695093e3SVarun Sethi w_count, GFP_ATOMIC); 1125695093e3SVarun Sethi if (!dma_domain->win_arr) { 1126695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1127695093e3SVarun Sethi return -ENOMEM; 1128695093e3SVarun Sethi } 1129695093e3SVarun Sethi dma_domain->win_cnt = w_count; 1130695093e3SVarun Sethi } 1131695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 1132695093e3SVarun Sethi 1133695093e3SVarun Sethi return ret; 1134695093e3SVarun Sethi } 1135695093e3SVarun Sethi 1136695093e3SVarun Sethi static u32 fsl_pamu_get_windows(struct iommu_domain *domain) 1137695093e3SVarun Sethi { 1138695093e3SVarun Sethi struct fsl_dma_domain *dma_domain = domain->priv; 1139695093e3SVarun Sethi 1140695093e3SVarun Sethi return dma_domain->win_cnt; 1141695093e3SVarun Sethi } 1142695093e3SVarun Sethi 1143695093e3SVarun Sethi static struct iommu_ops fsl_pamu_ops = { 1144695093e3SVarun Sethi .domain_init = fsl_pamu_domain_init, 1145695093e3SVarun Sethi .domain_destroy = fsl_pamu_domain_destroy, 1146695093e3SVarun Sethi .attach_dev = fsl_pamu_attach_device, 1147695093e3SVarun Sethi .detach_dev = fsl_pamu_detach_device, 1148695093e3SVarun Sethi .domain_window_enable = fsl_pamu_window_enable, 1149695093e3SVarun Sethi .domain_window_disable = fsl_pamu_window_disable, 1150695093e3SVarun Sethi .domain_get_windows = fsl_pamu_get_windows, 1151695093e3SVarun Sethi .domain_set_windows = fsl_pamu_set_windows, 1152695093e3SVarun Sethi .iova_to_phys = fsl_pamu_iova_to_phys, 1153695093e3SVarun Sethi .domain_has_cap = fsl_pamu_domain_has_cap, 1154695093e3SVarun Sethi .domain_set_attr = fsl_pamu_set_domain_attr, 1155695093e3SVarun Sethi .domain_get_attr = fsl_pamu_get_domain_attr, 1156695093e3SVarun Sethi .add_device = fsl_pamu_add_device, 1157695093e3SVarun Sethi .remove_device = fsl_pamu_remove_device, 1158695093e3SVarun Sethi }; 1159695093e3SVarun Sethi 1160695093e3SVarun Sethi int pamu_domain_init() 1161695093e3SVarun Sethi { 1162695093e3SVarun Sethi int ret = 0; 1163695093e3SVarun Sethi 1164695093e3SVarun Sethi ret = iommu_init_mempool(); 1165695093e3SVarun Sethi if (ret) 1166695093e3SVarun Sethi return ret; 1167695093e3SVarun Sethi 1168695093e3SVarun Sethi bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); 1169695093e3SVarun Sethi bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); 1170695093e3SVarun Sethi 1171695093e3SVarun Sethi return ret; 1172695093e3SVarun Sethi } 1173