1d94d71cbSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2695093e3SVarun Sethi /* 3695093e3SVarun Sethi * 4695093e3SVarun Sethi * Copyright (C) 2013 Freescale Semiconductor, Inc. 5695093e3SVarun Sethi * Author: Varun Sethi <varun.sethi@freescale.com> 6695093e3SVarun Sethi */ 7695093e3SVarun Sethi 8695093e3SVarun Sethi #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__ 9695093e3SVarun Sethi 10695093e3SVarun Sethi #include "fsl_pamu_domain.h" 11695093e3SVarun Sethi 12cd70d465SEmil Medve #include <sysdev/fsl_pci.h> 13cd70d465SEmil Medve 14695093e3SVarun Sethi /* 15695093e3SVarun Sethi * Global spinlock that needs to be held while 16695093e3SVarun Sethi * configuring PAMU. 17695093e3SVarun Sethi */ 18695093e3SVarun Sethi static DEFINE_SPINLOCK(iommu_lock); 19695093e3SVarun Sethi 20695093e3SVarun Sethi static struct kmem_cache *fsl_pamu_domain_cache; 21695093e3SVarun Sethi static struct kmem_cache *iommu_devinfo_cache; 22695093e3SVarun Sethi static DEFINE_SPINLOCK(device_domain_lock); 23695093e3SVarun Sethi 243ff2dcc0SJoerg Roedel struct iommu_device pamu_iommu; /* IOMMU core code handle */ 253ff2dcc0SJoerg Roedel 268d4bfe40SJoerg Roedel static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom) 278d4bfe40SJoerg Roedel { 288d4bfe40SJoerg Roedel return container_of(dom, struct fsl_dma_domain, iommu_domain); 298d4bfe40SJoerg Roedel } 308d4bfe40SJoerg Roedel 31695093e3SVarun Sethi static int __init iommu_init_mempool(void) 32695093e3SVarun Sethi { 33695093e3SVarun Sethi fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain", 34695093e3SVarun Sethi sizeof(struct fsl_dma_domain), 35695093e3SVarun Sethi 0, 36695093e3SVarun Sethi SLAB_HWCACHE_ALIGN, 37695093e3SVarun Sethi NULL); 38695093e3SVarun Sethi if (!fsl_pamu_domain_cache) { 39695093e3SVarun Sethi pr_debug("Couldn't create fsl iommu_domain cache\n"); 40695093e3SVarun Sethi return -ENOMEM; 41695093e3SVarun Sethi } 42695093e3SVarun Sethi 43695093e3SVarun Sethi iommu_devinfo_cache = kmem_cache_create("iommu_devinfo", 44695093e3SVarun Sethi sizeof(struct device_domain_info), 45695093e3SVarun Sethi 0, 46695093e3SVarun Sethi SLAB_HWCACHE_ALIGN, 47695093e3SVarun Sethi NULL); 48695093e3SVarun Sethi if (!iommu_devinfo_cache) { 49695093e3SVarun Sethi pr_debug("Couldn't create devinfo cache\n"); 50695093e3SVarun Sethi kmem_cache_destroy(fsl_pamu_domain_cache); 51695093e3SVarun Sethi return -ENOMEM; 52695093e3SVarun Sethi } 53695093e3SVarun Sethi 54695093e3SVarun Sethi return 0; 55695093e3SVarun Sethi } 56695093e3SVarun Sethi 57695093e3SVarun Sethi static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) 58695093e3SVarun Sethi { 59695093e3SVarun Sethi u32 win_cnt = dma_domain->win_cnt; 60cd70d465SEmil Medve struct dma_window *win_ptr = &dma_domain->win_arr[0]; 61695093e3SVarun Sethi struct iommu_domain_geometry *geom; 62695093e3SVarun Sethi 638d4bfe40SJoerg Roedel geom = &dma_domain->iommu_domain.geometry; 64695093e3SVarun Sethi 65695093e3SVarun Sethi if (!win_cnt || !dma_domain->geom_size) { 66695093e3SVarun Sethi pr_debug("Number of windows/geometry not configured for the domain\n"); 67695093e3SVarun Sethi return 0; 68695093e3SVarun Sethi } 69695093e3SVarun Sethi 70695093e3SVarun Sethi if (win_cnt > 1) { 71695093e3SVarun Sethi u64 subwin_size; 72695093e3SVarun Sethi dma_addr_t subwin_iova; 73695093e3SVarun Sethi u32 wnd; 74695093e3SVarun Sethi 75695093e3SVarun Sethi subwin_size = dma_domain->geom_size >> ilog2(win_cnt); 76695093e3SVarun Sethi subwin_iova = iova & ~(subwin_size - 1); 77695093e3SVarun Sethi wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size); 78695093e3SVarun Sethi win_ptr = &dma_domain->win_arr[wnd]; 79695093e3SVarun Sethi } 80695093e3SVarun Sethi 81695093e3SVarun Sethi if (win_ptr->valid) 82cd70d465SEmil Medve return win_ptr->paddr + (iova & (win_ptr->size - 1)); 83695093e3SVarun Sethi 84695093e3SVarun Sethi return 0; 85695093e3SVarun Sethi } 86695093e3SVarun Sethi 87695093e3SVarun Sethi static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) 88695093e3SVarun Sethi { 89cd70d465SEmil Medve struct dma_window *sub_win_ptr = &dma_domain->win_arr[0]; 90695093e3SVarun Sethi int i, ret; 91695093e3SVarun Sethi unsigned long rpn, flags; 92695093e3SVarun Sethi 93695093e3SVarun Sethi for (i = 0; i < dma_domain->win_cnt; i++) { 94695093e3SVarun Sethi if (sub_win_ptr[i].valid) { 95cd70d465SEmil Medve rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT; 96695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 97695093e3SVarun Sethi ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i, 98695093e3SVarun Sethi sub_win_ptr[i].size, 99695093e3SVarun Sethi ~(u32)0, 100695093e3SVarun Sethi rpn, 101695093e3SVarun Sethi dma_domain->snoop_id, 102695093e3SVarun Sethi dma_domain->stash_id, 103695093e3SVarun Sethi (i > 0) ? 1 : 0, 104695093e3SVarun Sethi sub_win_ptr[i].prot); 105695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 106695093e3SVarun Sethi if (ret) { 107cd70d465SEmil Medve pr_debug("SPAACE configuration failed for liodn %d\n", 108695093e3SVarun Sethi liodn); 109695093e3SVarun Sethi return ret; 110695093e3SVarun Sethi } 111695093e3SVarun Sethi } 112695093e3SVarun Sethi } 113695093e3SVarun Sethi 114695093e3SVarun Sethi return ret; 115695093e3SVarun Sethi } 116695093e3SVarun Sethi 117695093e3SVarun Sethi static int map_win(int liodn, struct fsl_dma_domain *dma_domain) 118695093e3SVarun Sethi { 119695093e3SVarun Sethi int ret; 120695093e3SVarun Sethi struct dma_window *wnd = &dma_domain->win_arr[0]; 1218d4bfe40SJoerg Roedel phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; 122695093e3SVarun Sethi unsigned long flags; 123695093e3SVarun Sethi 124695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 125695093e3SVarun Sethi ret = pamu_config_ppaace(liodn, wnd_addr, 126695093e3SVarun Sethi wnd->size, 127695093e3SVarun Sethi ~(u32)0, 128695093e3SVarun Sethi wnd->paddr >> PAMU_PAGE_SHIFT, 129695093e3SVarun Sethi dma_domain->snoop_id, dma_domain->stash_id, 130695093e3SVarun Sethi 0, wnd->prot); 131695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 132695093e3SVarun Sethi if (ret) 133cd70d465SEmil Medve pr_debug("PAACE configuration failed for liodn %d\n", liodn); 134695093e3SVarun Sethi 135695093e3SVarun Sethi return ret; 136695093e3SVarun Sethi } 137695093e3SVarun Sethi 138695093e3SVarun Sethi /* Map the DMA window corresponding to the LIODN */ 139695093e3SVarun Sethi static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) 140695093e3SVarun Sethi { 141695093e3SVarun Sethi if (dma_domain->win_cnt > 1) 142695093e3SVarun Sethi return map_subwins(liodn, dma_domain); 143695093e3SVarun Sethi else 144695093e3SVarun Sethi return map_win(liodn, dma_domain); 145695093e3SVarun Sethi } 146695093e3SVarun Sethi 147695093e3SVarun Sethi /* Update window/subwindow mapping for the LIODN */ 148695093e3SVarun Sethi static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr) 149695093e3SVarun Sethi { 150695093e3SVarun Sethi int ret; 151695093e3SVarun Sethi struct dma_window *wnd = &dma_domain->win_arr[wnd_nr]; 152695093e3SVarun Sethi unsigned long flags; 153695093e3SVarun Sethi 154695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 155695093e3SVarun Sethi if (dma_domain->win_cnt > 1) { 156695093e3SVarun Sethi ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr, 157695093e3SVarun Sethi wnd->size, 158695093e3SVarun Sethi ~(u32)0, 159695093e3SVarun Sethi wnd->paddr >> PAMU_PAGE_SHIFT, 160695093e3SVarun Sethi dma_domain->snoop_id, 161695093e3SVarun Sethi dma_domain->stash_id, 162695093e3SVarun Sethi (wnd_nr > 0) ? 1 : 0, 163695093e3SVarun Sethi wnd->prot); 164695093e3SVarun Sethi if (ret) 165cd70d465SEmil Medve pr_debug("Subwindow reconfiguration failed for liodn %d\n", 166cd70d465SEmil Medve liodn); 167695093e3SVarun Sethi } else { 168695093e3SVarun Sethi phys_addr_t wnd_addr; 169695093e3SVarun Sethi 1708d4bfe40SJoerg Roedel wnd_addr = dma_domain->iommu_domain.geometry.aperture_start; 171695093e3SVarun Sethi 172695093e3SVarun Sethi ret = pamu_config_ppaace(liodn, wnd_addr, 173695093e3SVarun Sethi wnd->size, 174695093e3SVarun Sethi ~(u32)0, 175695093e3SVarun Sethi wnd->paddr >> PAMU_PAGE_SHIFT, 176695093e3SVarun Sethi dma_domain->snoop_id, dma_domain->stash_id, 177695093e3SVarun Sethi 0, wnd->prot); 178695093e3SVarun Sethi if (ret) 179cd70d465SEmil Medve pr_debug("Window reconfiguration failed for liodn %d\n", 180cd70d465SEmil Medve liodn); 181695093e3SVarun Sethi } 182695093e3SVarun Sethi 183695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 184695093e3SVarun Sethi 185695093e3SVarun Sethi return ret; 186695093e3SVarun Sethi } 187695093e3SVarun Sethi 188695093e3SVarun Sethi static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, 189695093e3SVarun Sethi u32 val) 190695093e3SVarun Sethi { 191695093e3SVarun Sethi int ret = 0, i; 192695093e3SVarun Sethi unsigned long flags; 193695093e3SVarun Sethi 194695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 195695093e3SVarun Sethi if (!dma_domain->win_arr) { 196cd70d465SEmil Medve pr_debug("Windows not configured, stash destination update failed for liodn %d\n", 197cd70d465SEmil Medve liodn); 198695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 199695093e3SVarun Sethi return -EINVAL; 200695093e3SVarun Sethi } 201695093e3SVarun Sethi 202695093e3SVarun Sethi for (i = 0; i < dma_domain->win_cnt; i++) { 203695093e3SVarun Sethi ret = pamu_update_paace_stash(liodn, i, val); 204695093e3SVarun Sethi if (ret) { 205cd70d465SEmil Medve pr_debug("Failed to update SPAACE %d field for liodn %d\n ", 206cd70d465SEmil Medve i, liodn); 207695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 208695093e3SVarun Sethi return ret; 209695093e3SVarun Sethi } 210695093e3SVarun Sethi } 211695093e3SVarun Sethi 212695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 213695093e3SVarun Sethi 214695093e3SVarun Sethi return ret; 215695093e3SVarun Sethi } 216695093e3SVarun Sethi 217695093e3SVarun Sethi /* Set the geometry parameters for a LIODN */ 218695093e3SVarun Sethi static int pamu_set_liodn(int liodn, struct device *dev, 219695093e3SVarun Sethi struct fsl_dma_domain *dma_domain, 220695093e3SVarun Sethi struct iommu_domain_geometry *geom_attr, 221695093e3SVarun Sethi u32 win_cnt) 222695093e3SVarun Sethi { 223695093e3SVarun Sethi phys_addr_t window_addr, window_size; 224695093e3SVarun Sethi phys_addr_t subwin_size; 225695093e3SVarun Sethi int ret = 0, i; 226695093e3SVarun Sethi u32 omi_index = ~(u32)0; 227695093e3SVarun Sethi unsigned long flags; 228695093e3SVarun Sethi 229695093e3SVarun Sethi /* 230695093e3SVarun Sethi * Configure the omi_index at the geometry setup time. 231695093e3SVarun Sethi * This is a static value which depends on the type of 232695093e3SVarun Sethi * device and would not change thereafter. 233695093e3SVarun Sethi */ 234695093e3SVarun Sethi get_ome_index(&omi_index, dev); 235695093e3SVarun Sethi 236695093e3SVarun Sethi window_addr = geom_attr->aperture_start; 237695093e3SVarun Sethi window_size = dma_domain->geom_size; 238695093e3SVarun Sethi 239695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 240695093e3SVarun Sethi ret = pamu_disable_liodn(liodn); 241695093e3SVarun Sethi if (!ret) 242695093e3SVarun Sethi ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index, 243695093e3SVarun Sethi 0, dma_domain->snoop_id, 244695093e3SVarun Sethi dma_domain->stash_id, win_cnt, 0); 245695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 246695093e3SVarun Sethi if (ret) { 247cd70d465SEmil Medve pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n", 248cd70d465SEmil Medve liodn, win_cnt); 249695093e3SVarun Sethi return ret; 250695093e3SVarun Sethi } 251695093e3SVarun Sethi 252695093e3SVarun Sethi if (win_cnt > 1) { 253695093e3SVarun Sethi subwin_size = window_size >> ilog2(win_cnt); 254695093e3SVarun Sethi for (i = 0; i < win_cnt; i++) { 255695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 256695093e3SVarun Sethi ret = pamu_disable_spaace(liodn, i); 257695093e3SVarun Sethi if (!ret) 258695093e3SVarun Sethi ret = pamu_config_spaace(liodn, win_cnt, i, 259695093e3SVarun Sethi subwin_size, omi_index, 260695093e3SVarun Sethi 0, dma_domain->snoop_id, 261695093e3SVarun Sethi dma_domain->stash_id, 262695093e3SVarun Sethi 0, 0); 263695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 264695093e3SVarun Sethi if (ret) { 265cd70d465SEmil Medve pr_debug("SPAACE configuration failed for liodn %d\n", 266cd70d465SEmil Medve liodn); 267695093e3SVarun Sethi return ret; 268695093e3SVarun Sethi } 269695093e3SVarun Sethi } 270695093e3SVarun Sethi } 271695093e3SVarun Sethi 272695093e3SVarun Sethi return ret; 273695093e3SVarun Sethi } 274695093e3SVarun Sethi 275695093e3SVarun Sethi static int check_size(u64 size, dma_addr_t iova) 276695093e3SVarun Sethi { 277695093e3SVarun Sethi /* 278695093e3SVarun Sethi * Size must be a power of two and at least be equal 279695093e3SVarun Sethi * to PAMU page size. 280695093e3SVarun Sethi */ 281d033f48fSVarun Sethi if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) { 282cd70d465SEmil Medve pr_debug("Size too small or not a power of two\n"); 283695093e3SVarun Sethi return -EINVAL; 284695093e3SVarun Sethi } 285695093e3SVarun Sethi 286695093e3SVarun Sethi /* iova must be page size aligned */ 287695093e3SVarun Sethi if (iova & (size - 1)) { 288cd70d465SEmil Medve pr_debug("Address is not aligned with window size\n"); 289695093e3SVarun Sethi return -EINVAL; 290695093e3SVarun Sethi } 291695093e3SVarun Sethi 292695093e3SVarun Sethi return 0; 293695093e3SVarun Sethi } 294695093e3SVarun Sethi 295695093e3SVarun Sethi static struct fsl_dma_domain *iommu_alloc_dma_domain(void) 296695093e3SVarun Sethi { 297695093e3SVarun Sethi struct fsl_dma_domain *domain; 298695093e3SVarun Sethi 299695093e3SVarun Sethi domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL); 300695093e3SVarun Sethi if (!domain) 301695093e3SVarun Sethi return NULL; 302695093e3SVarun Sethi 303695093e3SVarun Sethi domain->stash_id = ~(u32)0; 304695093e3SVarun Sethi domain->snoop_id = ~(u32)0; 305695093e3SVarun Sethi domain->win_cnt = pamu_get_max_subwin_cnt(); 306695093e3SVarun Sethi domain->geom_size = 0; 307695093e3SVarun Sethi 308695093e3SVarun Sethi INIT_LIST_HEAD(&domain->devices); 309695093e3SVarun Sethi 310695093e3SVarun Sethi spin_lock_init(&domain->domain_lock); 311695093e3SVarun Sethi 312695093e3SVarun Sethi return domain; 313695093e3SVarun Sethi } 314695093e3SVarun Sethi 315695093e3SVarun Sethi static void remove_device_ref(struct device_domain_info *info, u32 win_cnt) 316695093e3SVarun Sethi { 317695093e3SVarun Sethi unsigned long flags; 318695093e3SVarun Sethi 319695093e3SVarun Sethi list_del(&info->link); 320695093e3SVarun Sethi spin_lock_irqsave(&iommu_lock, flags); 321695093e3SVarun Sethi if (win_cnt > 1) 322695093e3SVarun Sethi pamu_free_subwins(info->liodn); 323695093e3SVarun Sethi pamu_disable_liodn(info->liodn); 324695093e3SVarun Sethi spin_unlock_irqrestore(&iommu_lock, flags); 325695093e3SVarun Sethi spin_lock_irqsave(&device_domain_lock, flags); 3262263d818SJoerg Roedel dev_iommu_priv_set(info->dev, NULL); 327695093e3SVarun Sethi kmem_cache_free(iommu_devinfo_cache, info); 328695093e3SVarun Sethi spin_unlock_irqrestore(&device_domain_lock, flags); 329695093e3SVarun Sethi } 330695093e3SVarun Sethi 331695093e3SVarun Sethi static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain) 332695093e3SVarun Sethi { 333695093e3SVarun Sethi struct device_domain_info *info, *tmp; 334695093e3SVarun Sethi unsigned long flags; 335695093e3SVarun Sethi 336695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 337695093e3SVarun Sethi /* Remove the device from the domain device list */ 338695093e3SVarun Sethi list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) { 339695093e3SVarun Sethi if (!dev || (info->dev == dev)) 340695093e3SVarun Sethi remove_device_ref(info, dma_domain->win_cnt); 341695093e3SVarun Sethi } 342695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 343695093e3SVarun Sethi } 344695093e3SVarun Sethi 345695093e3SVarun Sethi static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev) 346695093e3SVarun Sethi { 347695093e3SVarun Sethi struct device_domain_info *info, *old_domain_info; 348695093e3SVarun Sethi unsigned long flags; 349695093e3SVarun Sethi 350695093e3SVarun Sethi spin_lock_irqsave(&device_domain_lock, flags); 351695093e3SVarun Sethi /* 352695093e3SVarun Sethi * Check here if the device is already attached to domain or not. 353695093e3SVarun Sethi * If the device is already attached to a domain detach it. 354695093e3SVarun Sethi */ 3552263d818SJoerg Roedel old_domain_info = dev_iommu_priv_get(dev); 356695093e3SVarun Sethi if (old_domain_info && old_domain_info->domain != dma_domain) { 357695093e3SVarun Sethi spin_unlock_irqrestore(&device_domain_lock, flags); 358695093e3SVarun Sethi detach_device(dev, old_domain_info->domain); 359695093e3SVarun Sethi spin_lock_irqsave(&device_domain_lock, flags); 360695093e3SVarun Sethi } 361695093e3SVarun Sethi 362695093e3SVarun Sethi info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC); 363695093e3SVarun Sethi 364695093e3SVarun Sethi info->dev = dev; 365695093e3SVarun Sethi info->liodn = liodn; 366695093e3SVarun Sethi info->domain = dma_domain; 367695093e3SVarun Sethi 368695093e3SVarun Sethi list_add(&info->link, &dma_domain->devices); 369695093e3SVarun Sethi /* 370695093e3SVarun Sethi * In case of devices with multiple LIODNs just store 371695093e3SVarun Sethi * the info for the first LIODN as all 372695093e3SVarun Sethi * LIODNs share the same domain 373695093e3SVarun Sethi */ 3742263d818SJoerg Roedel if (!dev_iommu_priv_get(dev)) 3752263d818SJoerg Roedel dev_iommu_priv_set(dev, info); 376695093e3SVarun Sethi spin_unlock_irqrestore(&device_domain_lock, flags); 377695093e3SVarun Sethi } 378695093e3SVarun Sethi 379695093e3SVarun Sethi static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, 380695093e3SVarun Sethi dma_addr_t iova) 381695093e3SVarun Sethi { 3828d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 383695093e3SVarun Sethi 384cd70d465SEmil Medve if (iova < domain->geometry.aperture_start || 385cd70d465SEmil Medve iova > domain->geometry.aperture_end) 386695093e3SVarun Sethi return 0; 387695093e3SVarun Sethi 388695093e3SVarun Sethi return get_phys_addr(dma_domain, iova); 389695093e3SVarun Sethi } 390695093e3SVarun Sethi 391b7eb6785SJoerg Roedel static bool fsl_pamu_capable(enum iommu_cap cap) 392695093e3SVarun Sethi { 393695093e3SVarun Sethi return cap == IOMMU_CAP_CACHE_COHERENCY; 394695093e3SVarun Sethi } 395695093e3SVarun Sethi 3968d4bfe40SJoerg Roedel static void fsl_pamu_domain_free(struct iommu_domain *domain) 397695093e3SVarun Sethi { 3988d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 399695093e3SVarun Sethi 400695093e3SVarun Sethi /* remove all the devices from the device list */ 401695093e3SVarun Sethi detach_device(NULL, dma_domain); 402695093e3SVarun Sethi 403695093e3SVarun Sethi dma_domain->enabled = 0; 404695093e3SVarun Sethi dma_domain->mapped = 0; 405695093e3SVarun Sethi 406695093e3SVarun Sethi kmem_cache_free(fsl_pamu_domain_cache, dma_domain); 407695093e3SVarun Sethi } 408695093e3SVarun Sethi 4098d4bfe40SJoerg Roedel static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type) 410695093e3SVarun Sethi { 411695093e3SVarun Sethi struct fsl_dma_domain *dma_domain; 412695093e3SVarun Sethi 4138d4bfe40SJoerg Roedel if (type != IOMMU_DOMAIN_UNMANAGED) 4148d4bfe40SJoerg Roedel return NULL; 4158d4bfe40SJoerg Roedel 416695093e3SVarun Sethi dma_domain = iommu_alloc_dma_domain(); 417695093e3SVarun Sethi if (!dma_domain) { 418695093e3SVarun Sethi pr_debug("dma_domain allocation failed\n"); 4198d4bfe40SJoerg Roedel return NULL; 420695093e3SVarun Sethi } 421695093e3SVarun Sethi /* defaul geometry 64 GB i.e. maximum system address */ 4228d4bfe40SJoerg Roedel dma_domain->iommu_domain. geometry.aperture_start = 0; 4238d4bfe40SJoerg Roedel dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1; 4248d4bfe40SJoerg Roedel dma_domain->iommu_domain.geometry.force_aperture = true; 425695093e3SVarun Sethi 4268d4bfe40SJoerg Roedel return &dma_domain->iommu_domain; 427695093e3SVarun Sethi } 428695093e3SVarun Sethi 429695093e3SVarun Sethi /* Configure geometry settings for all LIODNs associated with domain */ 430695093e3SVarun Sethi static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain, 431695093e3SVarun Sethi struct iommu_domain_geometry *geom_attr, 432695093e3SVarun Sethi u32 win_cnt) 433695093e3SVarun Sethi { 434695093e3SVarun Sethi struct device_domain_info *info; 435695093e3SVarun Sethi int ret = 0; 436695093e3SVarun Sethi 437695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 438695093e3SVarun Sethi ret = pamu_set_liodn(info->liodn, info->dev, dma_domain, 439695093e3SVarun Sethi geom_attr, win_cnt); 440695093e3SVarun Sethi if (ret) 441695093e3SVarun Sethi break; 442695093e3SVarun Sethi } 443695093e3SVarun Sethi 444695093e3SVarun Sethi return ret; 445695093e3SVarun Sethi } 446695093e3SVarun Sethi 447695093e3SVarun Sethi /* Update stash destination for all LIODNs associated with the domain */ 448695093e3SVarun Sethi static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val) 449695093e3SVarun Sethi { 450695093e3SVarun Sethi struct device_domain_info *info; 451695093e3SVarun Sethi int ret = 0; 452695093e3SVarun Sethi 453695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 454695093e3SVarun Sethi ret = update_liodn_stash(info->liodn, dma_domain, val); 455695093e3SVarun Sethi if (ret) 456695093e3SVarun Sethi break; 457695093e3SVarun Sethi } 458695093e3SVarun Sethi 459695093e3SVarun Sethi return ret; 460695093e3SVarun Sethi } 461695093e3SVarun Sethi 462695093e3SVarun Sethi /* Update domain mappings for all LIODNs associated with the domain */ 463695093e3SVarun Sethi static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 464695093e3SVarun Sethi { 465695093e3SVarun Sethi struct device_domain_info *info; 466695093e3SVarun Sethi int ret = 0; 467695093e3SVarun Sethi 468695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 469695093e3SVarun Sethi ret = update_liodn(info->liodn, dma_domain, wnd_nr); 470695093e3SVarun Sethi if (ret) 471695093e3SVarun Sethi break; 472695093e3SVarun Sethi } 473695093e3SVarun Sethi return ret; 474695093e3SVarun Sethi } 475695093e3SVarun Sethi 476695093e3SVarun Sethi static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr) 477695093e3SVarun Sethi { 478695093e3SVarun Sethi struct device_domain_info *info; 479695093e3SVarun Sethi int ret = 0; 480695093e3SVarun Sethi 481695093e3SVarun Sethi list_for_each_entry(info, &dma_domain->devices, link) { 482695093e3SVarun Sethi if (dma_domain->win_cnt == 1 && dma_domain->enabled) { 483695093e3SVarun Sethi ret = pamu_disable_liodn(info->liodn); 484695093e3SVarun Sethi if (!ret) 485695093e3SVarun Sethi dma_domain->enabled = 0; 486695093e3SVarun Sethi } else { 487695093e3SVarun Sethi ret = pamu_disable_spaace(info->liodn, wnd_nr); 488695093e3SVarun Sethi } 489695093e3SVarun Sethi } 490695093e3SVarun Sethi 491695093e3SVarun Sethi return ret; 492695093e3SVarun Sethi } 493695093e3SVarun Sethi 494695093e3SVarun Sethi static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr) 495695093e3SVarun Sethi { 4968d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 497695093e3SVarun Sethi unsigned long flags; 498695093e3SVarun Sethi int ret; 499695093e3SVarun Sethi 500695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 501695093e3SVarun Sethi if (!dma_domain->win_arr) { 502695093e3SVarun Sethi pr_debug("Number of windows not configured\n"); 503695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 504695093e3SVarun Sethi return; 505695093e3SVarun Sethi } 506695093e3SVarun Sethi 507695093e3SVarun Sethi if (wnd_nr >= dma_domain->win_cnt) { 508695093e3SVarun Sethi pr_debug("Invalid window index\n"); 509695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 510695093e3SVarun Sethi return; 511695093e3SVarun Sethi } 512695093e3SVarun Sethi 513695093e3SVarun Sethi if (dma_domain->win_arr[wnd_nr].valid) { 514695093e3SVarun Sethi ret = disable_domain_win(dma_domain, wnd_nr); 515695093e3SVarun Sethi if (!ret) { 516695093e3SVarun Sethi dma_domain->win_arr[wnd_nr].valid = 0; 517695093e3SVarun Sethi dma_domain->mapped--; 518695093e3SVarun Sethi } 519695093e3SVarun Sethi } 520695093e3SVarun Sethi 521695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 522695093e3SVarun Sethi } 523695093e3SVarun Sethi 524695093e3SVarun Sethi static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, 525695093e3SVarun Sethi phys_addr_t paddr, u64 size, int prot) 526695093e3SVarun Sethi { 5278d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 528695093e3SVarun Sethi struct dma_window *wnd; 529695093e3SVarun Sethi int pamu_prot = 0; 530695093e3SVarun Sethi int ret; 531695093e3SVarun Sethi unsigned long flags; 532695093e3SVarun Sethi u64 win_size; 533695093e3SVarun Sethi 534695093e3SVarun Sethi if (prot & IOMMU_READ) 535695093e3SVarun Sethi pamu_prot |= PAACE_AP_PERMS_QUERY; 536695093e3SVarun Sethi if (prot & IOMMU_WRITE) 537695093e3SVarun Sethi pamu_prot |= PAACE_AP_PERMS_UPDATE; 538695093e3SVarun Sethi 539695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 540695093e3SVarun Sethi if (!dma_domain->win_arr) { 541695093e3SVarun Sethi pr_debug("Number of windows not configured\n"); 542695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 543695093e3SVarun Sethi return -ENODEV; 544695093e3SVarun Sethi } 545695093e3SVarun Sethi 546695093e3SVarun Sethi if (wnd_nr >= dma_domain->win_cnt) { 547695093e3SVarun Sethi pr_debug("Invalid window index\n"); 548695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 549695093e3SVarun Sethi return -EINVAL; 550695093e3SVarun Sethi } 551695093e3SVarun Sethi 552695093e3SVarun Sethi win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt); 553695093e3SVarun Sethi if (size > win_size) { 554695093e3SVarun Sethi pr_debug("Invalid window size\n"); 555695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 556695093e3SVarun Sethi return -EINVAL; 557695093e3SVarun Sethi } 558695093e3SVarun Sethi 559695093e3SVarun Sethi if (dma_domain->win_cnt == 1) { 560695093e3SVarun Sethi if (dma_domain->enabled) { 561695093e3SVarun Sethi pr_debug("Disable the window before updating the mapping\n"); 562695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 563695093e3SVarun Sethi return -EBUSY; 564695093e3SVarun Sethi } 565695093e3SVarun Sethi 566695093e3SVarun Sethi ret = check_size(size, domain->geometry.aperture_start); 567695093e3SVarun Sethi if (ret) { 568695093e3SVarun Sethi pr_debug("Aperture start not aligned to the size\n"); 569695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 570695093e3SVarun Sethi return -EINVAL; 571695093e3SVarun Sethi } 572695093e3SVarun Sethi } 573695093e3SVarun Sethi 574695093e3SVarun Sethi wnd = &dma_domain->win_arr[wnd_nr]; 575695093e3SVarun Sethi if (!wnd->valid) { 576695093e3SVarun Sethi wnd->paddr = paddr; 577695093e3SVarun Sethi wnd->size = size; 578695093e3SVarun Sethi wnd->prot = pamu_prot; 579695093e3SVarun Sethi 580695093e3SVarun Sethi ret = update_domain_mapping(dma_domain, wnd_nr); 581695093e3SVarun Sethi if (!ret) { 582695093e3SVarun Sethi wnd->valid = 1; 583695093e3SVarun Sethi dma_domain->mapped++; 584695093e3SVarun Sethi } 585695093e3SVarun Sethi } else { 586695093e3SVarun Sethi pr_debug("Disable the window before updating the mapping\n"); 587695093e3SVarun Sethi ret = -EBUSY; 588695093e3SVarun Sethi } 589695093e3SVarun Sethi 590695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 591695093e3SVarun Sethi 592695093e3SVarun Sethi return ret; 593695093e3SVarun Sethi } 594695093e3SVarun Sethi 595695093e3SVarun Sethi /* 596695093e3SVarun Sethi * Attach the LIODN to the DMA domain and configure the geometry 597695093e3SVarun Sethi * and window mappings. 598695093e3SVarun Sethi */ 599695093e3SVarun Sethi static int handle_attach_device(struct fsl_dma_domain *dma_domain, 600695093e3SVarun Sethi struct device *dev, const u32 *liodn, 601695093e3SVarun Sethi int num) 602695093e3SVarun Sethi { 603695093e3SVarun Sethi unsigned long flags; 6048d4bfe40SJoerg Roedel struct iommu_domain *domain = &dma_domain->iommu_domain; 605695093e3SVarun Sethi int ret = 0; 606695093e3SVarun Sethi int i; 607695093e3SVarun Sethi 608695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 609695093e3SVarun Sethi for (i = 0; i < num; i++) { 610695093e3SVarun Sethi /* Ensure that LIODN value is valid */ 611695093e3SVarun Sethi if (liodn[i] >= PAACE_NUMBER_ENTRIES) { 6126bd4f1c7SRob Herring pr_debug("Invalid liodn %d, attach device failed for %pOF\n", 6136bd4f1c7SRob Herring liodn[i], dev->of_node); 614695093e3SVarun Sethi ret = -EINVAL; 615695093e3SVarun Sethi break; 616695093e3SVarun Sethi } 617695093e3SVarun Sethi 618695093e3SVarun Sethi attach_device(dma_domain, liodn[i], dev); 619695093e3SVarun Sethi /* 620695093e3SVarun Sethi * Check if geometry has already been configured 621695093e3SVarun Sethi * for the domain. If yes, set the geometry for 622695093e3SVarun Sethi * the LIODN. 623695093e3SVarun Sethi */ 624695093e3SVarun Sethi if (dma_domain->win_arr) { 625695093e3SVarun Sethi u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0; 626cd70d465SEmil Medve 627695093e3SVarun Sethi ret = pamu_set_liodn(liodn[i], dev, dma_domain, 628cd70d465SEmil Medve &domain->geometry, win_cnt); 629695093e3SVarun Sethi if (ret) 630695093e3SVarun Sethi break; 631695093e3SVarun Sethi if (dma_domain->mapped) { 632695093e3SVarun Sethi /* 633695093e3SVarun Sethi * Create window/subwindow mapping for 634695093e3SVarun Sethi * the LIODN. 635695093e3SVarun Sethi */ 636695093e3SVarun Sethi ret = map_liodn(liodn[i], dma_domain); 637695093e3SVarun Sethi if (ret) 638695093e3SVarun Sethi break; 639695093e3SVarun Sethi } 640695093e3SVarun Sethi } 641695093e3SVarun Sethi } 642695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 643695093e3SVarun Sethi 644695093e3SVarun Sethi return ret; 645695093e3SVarun Sethi } 646695093e3SVarun Sethi 647695093e3SVarun Sethi static int fsl_pamu_attach_device(struct iommu_domain *domain, 648695093e3SVarun Sethi struct device *dev) 649695093e3SVarun Sethi { 6508d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 651695093e3SVarun Sethi const u32 *liodn; 652695093e3SVarun Sethi u32 liodn_cnt; 653695093e3SVarun Sethi int len, ret = 0; 654695093e3SVarun Sethi struct pci_dev *pdev = NULL; 655695093e3SVarun Sethi struct pci_controller *pci_ctl; 656695093e3SVarun Sethi 657695093e3SVarun Sethi /* 658695093e3SVarun Sethi * Use LIODN of the PCI controller while attaching a 659695093e3SVarun Sethi * PCI device. 660695093e3SVarun Sethi */ 661b3eb76d1SYijing Wang if (dev_is_pci(dev)) { 662695093e3SVarun Sethi pdev = to_pci_dev(dev); 663695093e3SVarun Sethi pci_ctl = pci_bus_to_host(pdev->bus); 664695093e3SVarun Sethi /* 665695093e3SVarun Sethi * make dev point to pci controller device 666695093e3SVarun Sethi * so we can get the LIODN programmed by 667695093e3SVarun Sethi * u-boot. 668695093e3SVarun Sethi */ 669695093e3SVarun Sethi dev = pci_ctl->parent; 670695093e3SVarun Sethi } 671695093e3SVarun Sethi 672695093e3SVarun Sethi liodn = of_get_property(dev->of_node, "fsl,liodn", &len); 673695093e3SVarun Sethi if (liodn) { 674695093e3SVarun Sethi liodn_cnt = len / sizeof(u32); 675cd70d465SEmil Medve ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt); 676695093e3SVarun Sethi } else { 6776bd4f1c7SRob Herring pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); 678695093e3SVarun Sethi ret = -EINVAL; 679695093e3SVarun Sethi } 680695093e3SVarun Sethi 681695093e3SVarun Sethi return ret; 682695093e3SVarun Sethi } 683695093e3SVarun Sethi 684695093e3SVarun Sethi static void fsl_pamu_detach_device(struct iommu_domain *domain, 685695093e3SVarun Sethi struct device *dev) 686695093e3SVarun Sethi { 6878d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 688695093e3SVarun Sethi const u32 *prop; 689695093e3SVarun Sethi int len; 690695093e3SVarun Sethi struct pci_dev *pdev = NULL; 691695093e3SVarun Sethi struct pci_controller *pci_ctl; 692695093e3SVarun Sethi 693695093e3SVarun Sethi /* 694695093e3SVarun Sethi * Use LIODN of the PCI controller while detaching a 695695093e3SVarun Sethi * PCI device. 696695093e3SVarun Sethi */ 697b3eb76d1SYijing Wang if (dev_is_pci(dev)) { 698695093e3SVarun Sethi pdev = to_pci_dev(dev); 699695093e3SVarun Sethi pci_ctl = pci_bus_to_host(pdev->bus); 700695093e3SVarun Sethi /* 701695093e3SVarun Sethi * make dev point to pci controller device 702695093e3SVarun Sethi * so we can get the LIODN programmed by 703695093e3SVarun Sethi * u-boot. 704695093e3SVarun Sethi */ 705695093e3SVarun Sethi dev = pci_ctl->parent; 706695093e3SVarun Sethi } 707695093e3SVarun Sethi 708695093e3SVarun Sethi prop = of_get_property(dev->of_node, "fsl,liodn", &len); 709695093e3SVarun Sethi if (prop) 710695093e3SVarun Sethi detach_device(dev, dma_domain); 711695093e3SVarun Sethi else 7126bd4f1c7SRob Herring pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node); 713695093e3SVarun Sethi } 714695093e3SVarun Sethi 715695093e3SVarun Sethi static int configure_domain_geometry(struct iommu_domain *domain, void *data) 716695093e3SVarun Sethi { 717695093e3SVarun Sethi struct iommu_domain_geometry *geom_attr = data; 7188d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 719695093e3SVarun Sethi dma_addr_t geom_size; 720695093e3SVarun Sethi unsigned long flags; 721695093e3SVarun Sethi 722695093e3SVarun Sethi geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1; 723695093e3SVarun Sethi /* 724695093e3SVarun Sethi * Sanity check the geometry size. Also, we do not support 725695093e3SVarun Sethi * DMA outside of the geometry. 726695093e3SVarun Sethi */ 727695093e3SVarun Sethi if (check_size(geom_size, geom_attr->aperture_start) || 728695093e3SVarun Sethi !geom_attr->force_aperture) { 729695093e3SVarun Sethi pr_debug("Invalid PAMU geometry attributes\n"); 730695093e3SVarun Sethi return -EINVAL; 731695093e3SVarun Sethi } 732695093e3SVarun Sethi 733695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 734695093e3SVarun Sethi if (dma_domain->enabled) { 735695093e3SVarun Sethi pr_debug("Can't set geometry attributes as domain is active\n"); 736695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 737695093e3SVarun Sethi return -EBUSY; 738695093e3SVarun Sethi } 739695093e3SVarun Sethi 740695093e3SVarun Sethi /* Copy the domain geometry information */ 741695093e3SVarun Sethi memcpy(&domain->geometry, geom_attr, 742695093e3SVarun Sethi sizeof(struct iommu_domain_geometry)); 743695093e3SVarun Sethi dma_domain->geom_size = geom_size; 744695093e3SVarun Sethi 745695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 746695093e3SVarun Sethi 747695093e3SVarun Sethi return 0; 748695093e3SVarun Sethi } 749695093e3SVarun Sethi 750695093e3SVarun Sethi /* Set the domain stash attribute */ 751695093e3SVarun Sethi static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) 752695093e3SVarun Sethi { 753695093e3SVarun Sethi struct pamu_stash_attribute *stash_attr = data; 754695093e3SVarun Sethi unsigned long flags; 755695093e3SVarun Sethi int ret; 756695093e3SVarun Sethi 757695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 758695093e3SVarun Sethi 759695093e3SVarun Sethi memcpy(&dma_domain->dma_stash, stash_attr, 760695093e3SVarun Sethi sizeof(struct pamu_stash_attribute)); 761695093e3SVarun Sethi 762695093e3SVarun Sethi dma_domain->stash_id = get_stash_id(stash_attr->cache, 763695093e3SVarun Sethi stash_attr->cpu); 764695093e3SVarun Sethi if (dma_domain->stash_id == ~(u32)0) { 765695093e3SVarun Sethi pr_debug("Invalid stash attributes\n"); 766695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 767695093e3SVarun Sethi return -EINVAL; 768695093e3SVarun Sethi } 769695093e3SVarun Sethi 770695093e3SVarun Sethi ret = update_domain_stash(dma_domain, dma_domain->stash_id); 771695093e3SVarun Sethi 772695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 773695093e3SVarun Sethi 774695093e3SVarun Sethi return ret; 775695093e3SVarun Sethi } 776695093e3SVarun Sethi 777695093e3SVarun Sethi /* Configure domain dma state i.e. enable/disable DMA */ 778695093e3SVarun Sethi static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable) 779695093e3SVarun Sethi { 780695093e3SVarun Sethi struct device_domain_info *info; 781695093e3SVarun Sethi unsigned long flags; 782695093e3SVarun Sethi int ret; 783695093e3SVarun Sethi 784695093e3SVarun Sethi spin_lock_irqsave(&dma_domain->domain_lock, flags); 785695093e3SVarun Sethi 786695093e3SVarun Sethi if (enable && !dma_domain->mapped) { 787695093e3SVarun Sethi pr_debug("Can't enable DMA domain without valid mapping\n"); 788695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 789695093e3SVarun Sethi return -ENODEV; 790695093e3SVarun Sethi } 791695093e3SVarun Sethi 792695093e3SVarun Sethi dma_domain->enabled = enable; 793cd70d465SEmil Medve list_for_each_entry(info, &dma_domain->devices, link) { 794695093e3SVarun Sethi ret = (enable) ? pamu_enable_liodn(info->liodn) : 795695093e3SVarun Sethi pamu_disable_liodn(info->liodn); 796695093e3SVarun Sethi if (ret) 797695093e3SVarun Sethi pr_debug("Unable to set dma state for liodn %d", 798695093e3SVarun Sethi info->liodn); 799695093e3SVarun Sethi } 800695093e3SVarun Sethi spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 801695093e3SVarun Sethi 802695093e3SVarun Sethi return 0; 803695093e3SVarun Sethi } 804695093e3SVarun Sethi 8055131e08cSRobin Murphy static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) 8065131e08cSRobin Murphy { 8075131e08cSRobin Murphy struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 8085131e08cSRobin Murphy unsigned long flags; 8095131e08cSRobin Murphy int ret; 8105131e08cSRobin Murphy 8115131e08cSRobin Murphy spin_lock_irqsave(&dma_domain->domain_lock, flags); 8125131e08cSRobin Murphy /* Ensure domain is inactive i.e. DMA should be disabled for the domain */ 8135131e08cSRobin Murphy if (dma_domain->enabled) { 8145131e08cSRobin Murphy pr_debug("Can't set geometry attributes as domain is active\n"); 8155131e08cSRobin Murphy spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 8165131e08cSRobin Murphy return -EBUSY; 8175131e08cSRobin Murphy } 8185131e08cSRobin Murphy 8195131e08cSRobin Murphy /* Ensure that the geometry has been set for the domain */ 8205131e08cSRobin Murphy if (!dma_domain->geom_size) { 8215131e08cSRobin Murphy pr_debug("Please configure geometry before setting the number of windows\n"); 8225131e08cSRobin Murphy spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 8235131e08cSRobin Murphy return -EINVAL; 8245131e08cSRobin Murphy } 8255131e08cSRobin Murphy 8265131e08cSRobin Murphy /* 8275131e08cSRobin Murphy * Ensure we have valid window count i.e. it should be less than 8285131e08cSRobin Murphy * maximum permissible limit and should be a power of two. 8295131e08cSRobin Murphy */ 8305131e08cSRobin Murphy if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) { 8315131e08cSRobin Murphy pr_debug("Invalid window count\n"); 8325131e08cSRobin Murphy spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 8335131e08cSRobin Murphy return -EINVAL; 8345131e08cSRobin Murphy } 8355131e08cSRobin Murphy 8365131e08cSRobin Murphy ret = pamu_set_domain_geometry(dma_domain, &domain->geometry, 8375131e08cSRobin Murphy w_count > 1 ? w_count : 0); 8385131e08cSRobin Murphy if (!ret) { 8395131e08cSRobin Murphy kfree(dma_domain->win_arr); 8405131e08cSRobin Murphy dma_domain->win_arr = kcalloc(w_count, 8415131e08cSRobin Murphy sizeof(*dma_domain->win_arr), 8425131e08cSRobin Murphy GFP_ATOMIC); 8435131e08cSRobin Murphy if (!dma_domain->win_arr) { 8445131e08cSRobin Murphy spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 8455131e08cSRobin Murphy return -ENOMEM; 8465131e08cSRobin Murphy } 8475131e08cSRobin Murphy dma_domain->win_cnt = w_count; 8485131e08cSRobin Murphy } 8495131e08cSRobin Murphy spin_unlock_irqrestore(&dma_domain->domain_lock, flags); 8505131e08cSRobin Murphy 8515131e08cSRobin Murphy return ret; 8525131e08cSRobin Murphy } 8535131e08cSRobin Murphy 854695093e3SVarun Sethi static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, 855695093e3SVarun Sethi enum iommu_attr attr_type, void *data) 856695093e3SVarun Sethi { 8578d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 858695093e3SVarun Sethi int ret = 0; 859695093e3SVarun Sethi 860695093e3SVarun Sethi switch (attr_type) { 861695093e3SVarun Sethi case DOMAIN_ATTR_GEOMETRY: 862695093e3SVarun Sethi ret = configure_domain_geometry(domain, data); 863695093e3SVarun Sethi break; 864695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_STASH: 865695093e3SVarun Sethi ret = configure_domain_stash(dma_domain, data); 866695093e3SVarun Sethi break; 867695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_ENABLE: 868695093e3SVarun Sethi ret = configure_domain_dma_state(dma_domain, *(int *)data); 869695093e3SVarun Sethi break; 870701d8a62SRobin Murphy case DOMAIN_ATTR_WINDOWS: 8715131e08cSRobin Murphy ret = fsl_pamu_set_windows(domain, *(u32 *)data); 872701d8a62SRobin Murphy break; 873695093e3SVarun Sethi default: 874695093e3SVarun Sethi pr_debug("Unsupported attribute type\n"); 875695093e3SVarun Sethi ret = -EINVAL; 876695093e3SVarun Sethi break; 877cd70d465SEmil Medve } 878695093e3SVarun Sethi 879695093e3SVarun Sethi return ret; 880695093e3SVarun Sethi } 881695093e3SVarun Sethi 882695093e3SVarun Sethi static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, 883695093e3SVarun Sethi enum iommu_attr attr_type, void *data) 884695093e3SVarun Sethi { 8858d4bfe40SJoerg Roedel struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain); 886695093e3SVarun Sethi int ret = 0; 887695093e3SVarun Sethi 888695093e3SVarun Sethi switch (attr_type) { 889695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_STASH: 890cd70d465SEmil Medve memcpy(data, &dma_domain->dma_stash, 891695093e3SVarun Sethi sizeof(struct pamu_stash_attribute)); 892695093e3SVarun Sethi break; 893695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMU_ENABLE: 894695093e3SVarun Sethi *(int *)data = dma_domain->enabled; 895695093e3SVarun Sethi break; 896695093e3SVarun Sethi case DOMAIN_ATTR_FSL_PAMUV1: 897695093e3SVarun Sethi *(int *)data = DOMAIN_ATTR_FSL_PAMUV1; 898695093e3SVarun Sethi break; 899701d8a62SRobin Murphy case DOMAIN_ATTR_WINDOWS: 9005131e08cSRobin Murphy *(u32 *)data = dma_domain->win_cnt; 901701d8a62SRobin Murphy break; 902695093e3SVarun Sethi default: 903695093e3SVarun Sethi pr_debug("Unsupported attribute type\n"); 904695093e3SVarun Sethi ret = -EINVAL; 905695093e3SVarun Sethi break; 906cd70d465SEmil Medve } 907695093e3SVarun Sethi 908695093e3SVarun Sethi return ret; 909695093e3SVarun Sethi } 910695093e3SVarun Sethi 911695093e3SVarun Sethi static struct iommu_group *get_device_iommu_group(struct device *dev) 912695093e3SVarun Sethi { 913695093e3SVarun Sethi struct iommu_group *group; 914695093e3SVarun Sethi 915695093e3SVarun Sethi group = iommu_group_get(dev); 916695093e3SVarun Sethi if (!group) 917695093e3SVarun Sethi group = iommu_group_alloc(); 918695093e3SVarun Sethi 919695093e3SVarun Sethi return group; 920695093e3SVarun Sethi } 921695093e3SVarun Sethi 922695093e3SVarun Sethi static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl) 923695093e3SVarun Sethi { 924695093e3SVarun Sethi u32 version; 925695093e3SVarun Sethi 926695093e3SVarun Sethi /* Check the PCI controller version number by readding BRR1 register */ 927695093e3SVarun Sethi version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2)); 928695093e3SVarun Sethi version &= PCI_FSL_BRR1_VER; 929695093e3SVarun Sethi /* If PCI controller version is >= 0x204 we can partition endpoints */ 930cd70d465SEmil Medve return version >= 0x204; 931695093e3SVarun Sethi } 932695093e3SVarun Sethi 933695093e3SVarun Sethi /* Get iommu group information from peer devices or devices on the parent bus */ 934695093e3SVarun Sethi static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev) 935695093e3SVarun Sethi { 936695093e3SVarun Sethi struct pci_dev *tmp; 937695093e3SVarun Sethi struct iommu_group *group; 938695093e3SVarun Sethi struct pci_bus *bus = pdev->bus; 939695093e3SVarun Sethi 940695093e3SVarun Sethi /* 941695093e3SVarun Sethi * Traverese the pci bus device list to get 942695093e3SVarun Sethi * the shared iommu group. 943695093e3SVarun Sethi */ 944695093e3SVarun Sethi while (bus) { 945695093e3SVarun Sethi list_for_each_entry(tmp, &bus->devices, bus_list) { 946695093e3SVarun Sethi if (tmp == pdev) 947695093e3SVarun Sethi continue; 948695093e3SVarun Sethi group = iommu_group_get(&tmp->dev); 949695093e3SVarun Sethi if (group) 950695093e3SVarun Sethi return group; 951695093e3SVarun Sethi } 952695093e3SVarun Sethi 953695093e3SVarun Sethi bus = bus->parent; 954695093e3SVarun Sethi } 955695093e3SVarun Sethi 956695093e3SVarun Sethi return NULL; 957695093e3SVarun Sethi } 958695093e3SVarun Sethi 959695093e3SVarun Sethi static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) 960695093e3SVarun Sethi { 961695093e3SVarun Sethi struct pci_controller *pci_ctl; 962bc46c229SColin Ian King bool pci_endpt_partitioning; 963695093e3SVarun Sethi struct iommu_group *group = NULL; 964695093e3SVarun Sethi 965695093e3SVarun Sethi pci_ctl = pci_bus_to_host(pdev->bus); 966bc46c229SColin Ian King pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl); 967695093e3SVarun Sethi /* We can partition PCIe devices so assign device group to the device */ 968bc46c229SColin Ian King if (pci_endpt_partitioning) { 969d5e58297SJoerg Roedel group = pci_device_group(&pdev->dev); 970695093e3SVarun Sethi 971695093e3SVarun Sethi /* 972695093e3SVarun Sethi * PCIe controller is not a paritionable entity 973695093e3SVarun Sethi * free the controller device iommu_group. 974695093e3SVarun Sethi */ 975695093e3SVarun Sethi if (pci_ctl->parent->iommu_group) 976695093e3SVarun Sethi iommu_group_remove_device(pci_ctl->parent); 977695093e3SVarun Sethi } else { 978695093e3SVarun Sethi /* 979695093e3SVarun Sethi * All devices connected to the controller will share the 980695093e3SVarun Sethi * PCI controllers device group. If this is the first 981695093e3SVarun Sethi * device to be probed for the pci controller, copy the 982695093e3SVarun Sethi * device group information from the PCI controller device 983695093e3SVarun Sethi * node and remove the PCI controller iommu group. 984695093e3SVarun Sethi * For subsequent devices, the iommu group information can 985695093e3SVarun Sethi * be obtained from sibling devices (i.e. from the bus_devices 986695093e3SVarun Sethi * link list). 987695093e3SVarun Sethi */ 988695093e3SVarun Sethi if (pci_ctl->parent->iommu_group) { 989695093e3SVarun Sethi group = get_device_iommu_group(pci_ctl->parent); 990695093e3SVarun Sethi iommu_group_remove_device(pci_ctl->parent); 991cd70d465SEmil Medve } else { 992695093e3SVarun Sethi group = get_shared_pci_device_group(pdev); 993695093e3SVarun Sethi } 994cd70d465SEmil Medve } 995695093e3SVarun Sethi 9963170447cSVarun Sethi if (!group) 9973170447cSVarun Sethi group = ERR_PTR(-ENODEV); 9983170447cSVarun Sethi 999695093e3SVarun Sethi return group; 1000695093e3SVarun Sethi } 1001695093e3SVarun Sethi 1002d5e58297SJoerg Roedel static struct iommu_group *fsl_pamu_device_group(struct device *dev) 1003695093e3SVarun Sethi { 10043170447cSVarun Sethi struct iommu_group *group = ERR_PTR(-ENODEV); 1005d5e58297SJoerg Roedel int len; 1006695093e3SVarun Sethi 1007695093e3SVarun Sethi /* 1008695093e3SVarun Sethi * For platform devices we allocate a separate group for 1009695093e3SVarun Sethi * each of the devices. 1010695093e3SVarun Sethi */ 1011d5e58297SJoerg Roedel if (dev_is_pci(dev)) 1012d5e58297SJoerg Roedel group = get_pci_device_group(to_pci_dev(dev)); 1013d5e58297SJoerg Roedel else if (of_get_property(dev->of_node, "fsl,liodn", &len)) 1014695093e3SVarun Sethi group = get_device_iommu_group(dev); 1015d5e58297SJoerg Roedel 1016d5e58297SJoerg Roedel return group; 1017695093e3SVarun Sethi } 1018695093e3SVarun Sethi 101952dd3ca4SJoerg Roedel static struct iommu_device *fsl_pamu_probe_device(struct device *dev) 1020d5e58297SJoerg Roedel { 102152dd3ca4SJoerg Roedel return &pamu_iommu; 1022695093e3SVarun Sethi } 1023695093e3SVarun Sethi 102452dd3ca4SJoerg Roedel static void fsl_pamu_release_device(struct device *dev) 1025695093e3SVarun Sethi { 1026695093e3SVarun Sethi } 1027695093e3SVarun Sethi 1028b22f6434SThierry Reding static const struct iommu_ops fsl_pamu_ops = { 1029b7eb6785SJoerg Roedel .capable = fsl_pamu_capable, 10308d4bfe40SJoerg Roedel .domain_alloc = fsl_pamu_domain_alloc, 10318d4bfe40SJoerg Roedel .domain_free = fsl_pamu_domain_free, 1032695093e3SVarun Sethi .attach_dev = fsl_pamu_attach_device, 1033695093e3SVarun Sethi .detach_dev = fsl_pamu_detach_device, 1034695093e3SVarun Sethi .domain_window_enable = fsl_pamu_window_enable, 1035695093e3SVarun Sethi .domain_window_disable = fsl_pamu_window_disable, 1036695093e3SVarun Sethi .iova_to_phys = fsl_pamu_iova_to_phys, 1037695093e3SVarun Sethi .domain_set_attr = fsl_pamu_set_domain_attr, 1038695093e3SVarun Sethi .domain_get_attr = fsl_pamu_get_domain_attr, 103952dd3ca4SJoerg Roedel .probe_device = fsl_pamu_probe_device, 104052dd3ca4SJoerg Roedel .release_device = fsl_pamu_release_device, 1041d5e58297SJoerg Roedel .device_group = fsl_pamu_device_group, 1042695093e3SVarun Sethi }; 1043695093e3SVarun Sethi 1044cd70d465SEmil Medve int __init pamu_domain_init(void) 1045695093e3SVarun Sethi { 1046695093e3SVarun Sethi int ret = 0; 1047695093e3SVarun Sethi 1048695093e3SVarun Sethi ret = iommu_init_mempool(); 1049695093e3SVarun Sethi if (ret) 1050695093e3SVarun Sethi return ret; 1051695093e3SVarun Sethi 10523ff2dcc0SJoerg Roedel ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0"); 10533ff2dcc0SJoerg Roedel if (ret) 10543ff2dcc0SJoerg Roedel return ret; 10553ff2dcc0SJoerg Roedel 10563ff2dcc0SJoerg Roedel iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops); 10573ff2dcc0SJoerg Roedel 10583ff2dcc0SJoerg Roedel ret = iommu_device_register(&pamu_iommu); 10593ff2dcc0SJoerg Roedel if (ret) { 10603ff2dcc0SJoerg Roedel iommu_device_sysfs_remove(&pamu_iommu); 10613ff2dcc0SJoerg Roedel pr_err("Can't register iommu device\n"); 10623ff2dcc0SJoerg Roedel return ret; 10633ff2dcc0SJoerg Roedel } 10643ff2dcc0SJoerg Roedel 1065695093e3SVarun Sethi bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); 1066695093e3SVarun Sethi bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); 1067695093e3SVarun Sethi 1068695093e3SVarun Sethi return ret; 1069695093e3SVarun Sethi } 1070