1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ 3 4 #include <linux/err.h> 5 #include <linux/slab.h> 6 #include <linux/dma-buf.h> 7 #include <linux/dma-mapping.h> 8 9 #include <drm/panfrost_drm.h> 10 #include "panfrost_device.h" 11 #include "panfrost_gem.h" 12 #include "panfrost_mmu.h" 13 14 /* Called DRM core on the last userspace/kernel unreference of the 15 * BO. 16 */ 17 static void panfrost_gem_free_object(struct drm_gem_object *obj) 18 { 19 struct panfrost_gem_object *bo = to_panfrost_bo(obj); 20 struct panfrost_device *pfdev = obj->dev->dev_private; 21 22 /* 23 * Make sure the BO is no longer inserted in the shrinker list before 24 * taking care of the destruction itself. If we don't do that we have a 25 * race condition between this function and what's done in 26 * panfrost_gem_shrinker_scan(). 27 */ 28 mutex_lock(&pfdev->shrinker_lock); 29 list_del_init(&bo->base.madv_list); 30 mutex_unlock(&pfdev->shrinker_lock); 31 32 /* 33 * If we still have mappings attached to the BO, there's a problem in 34 * our refcounting. 35 */ 36 WARN_ON_ONCE(!list_empty(&bo->mappings.list)); 37 38 if (bo->sgts) { 39 int i; 40 int n_sgt = bo->base.base.size / SZ_2M; 41 42 for (i = 0; i < n_sgt; i++) { 43 if (bo->sgts[i].sgl) { 44 dma_unmap_sg(pfdev->dev, bo->sgts[i].sgl, 45 bo->sgts[i].nents, DMA_BIDIRECTIONAL); 46 sg_free_table(&bo->sgts[i]); 47 } 48 } 49 kfree(bo->sgts); 50 } 51 52 drm_gem_shmem_free_object(obj); 53 } 54 55 struct panfrost_gem_mapping * 56 panfrost_gem_mapping_get(struct panfrost_gem_object *bo, 57 struct panfrost_file_priv *priv) 58 { 59 struct panfrost_gem_mapping *iter, *mapping = NULL; 60 61 mutex_lock(&bo->mappings.lock); 62 list_for_each_entry(iter, &bo->mappings.list, node) { 63 if (iter->mmu == &priv->mmu) { 64 kref_get(&iter->refcount); 65 mapping = iter; 66 break; 67 } 68 } 69 mutex_unlock(&bo->mappings.lock); 70 71 return mapping; 72 } 73 74 static void 75 panfrost_gem_teardown_mapping(struct panfrost_gem_mapping *mapping) 76 { 77 struct panfrost_file_priv *priv; 78 79 if (mapping->active) 80 panfrost_mmu_unmap(mapping); 81 82 priv = container_of(mapping->mmu, struct panfrost_file_priv, mmu); 83 spin_lock(&priv->mm_lock); 84 if (drm_mm_node_allocated(&mapping->mmnode)) 85 drm_mm_remove_node(&mapping->mmnode); 86 spin_unlock(&priv->mm_lock); 87 } 88 89 static void panfrost_gem_mapping_release(struct kref *kref) 90 { 91 struct panfrost_gem_mapping *mapping; 92 93 mapping = container_of(kref, struct panfrost_gem_mapping, refcount); 94 95 panfrost_gem_teardown_mapping(mapping); 96 drm_gem_object_put_unlocked(&mapping->obj->base.base); 97 kfree(mapping); 98 } 99 100 void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping) 101 { 102 if (!mapping) 103 return; 104 105 kref_put(&mapping->refcount, panfrost_gem_mapping_release); 106 } 107 108 void panfrost_gem_teardown_mappings(struct panfrost_gem_object *bo) 109 { 110 struct panfrost_gem_mapping *mapping; 111 112 mutex_lock(&bo->mappings.lock); 113 list_for_each_entry(mapping, &bo->mappings.list, node) 114 panfrost_gem_teardown_mapping(mapping); 115 mutex_unlock(&bo->mappings.lock); 116 } 117 118 int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_priv) 119 { 120 int ret; 121 size_t size = obj->size; 122 u64 align; 123 struct panfrost_gem_object *bo = to_panfrost_bo(obj); 124 unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0; 125 struct panfrost_file_priv *priv = file_priv->driver_priv; 126 struct panfrost_gem_mapping *mapping; 127 128 mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); 129 if (!mapping) 130 return -ENOMEM; 131 132 INIT_LIST_HEAD(&mapping->node); 133 kref_init(&mapping->refcount); 134 drm_gem_object_get(obj); 135 mapping->obj = bo; 136 137 /* 138 * Executable buffers cannot cross a 16MB boundary as the program 139 * counter is 24-bits. We assume executable buffers will be less than 140 * 16MB and aligning executable buffers to their size will avoid 141 * crossing a 16MB boundary. 142 */ 143 if (!bo->noexec) 144 align = size >> PAGE_SHIFT; 145 else 146 align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0; 147 148 mapping->mmu = &priv->mmu; 149 spin_lock(&priv->mm_lock); 150 ret = drm_mm_insert_node_generic(&priv->mm, &mapping->mmnode, 151 size >> PAGE_SHIFT, align, color, 0); 152 spin_unlock(&priv->mm_lock); 153 if (ret) 154 goto err; 155 156 if (!bo->is_heap) { 157 ret = panfrost_mmu_map(mapping); 158 if (ret) 159 goto err; 160 } 161 162 mutex_lock(&bo->mappings.lock); 163 WARN_ON(bo->base.madv != PANFROST_MADV_WILLNEED); 164 list_add_tail(&mapping->node, &bo->mappings.list); 165 mutex_unlock(&bo->mappings.lock); 166 167 err: 168 if (ret) 169 panfrost_gem_mapping_put(mapping); 170 return ret; 171 } 172 173 void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv) 174 { 175 struct panfrost_file_priv *priv = file_priv->driver_priv; 176 struct panfrost_gem_object *bo = to_panfrost_bo(obj); 177 struct panfrost_gem_mapping *mapping = NULL, *iter; 178 179 mutex_lock(&bo->mappings.lock); 180 list_for_each_entry(iter, &bo->mappings.list, node) { 181 if (iter->mmu == &priv->mmu) { 182 mapping = iter; 183 list_del(&iter->node); 184 break; 185 } 186 } 187 mutex_unlock(&bo->mappings.lock); 188 189 panfrost_gem_mapping_put(mapping); 190 } 191 192 static int panfrost_gem_pin(struct drm_gem_object *obj) 193 { 194 if (to_panfrost_bo(obj)->is_heap) 195 return -EINVAL; 196 197 return drm_gem_shmem_pin(obj); 198 } 199 200 static const struct drm_gem_object_funcs panfrost_gem_funcs = { 201 .free = panfrost_gem_free_object, 202 .open = panfrost_gem_open, 203 .close = panfrost_gem_close, 204 .print_info = drm_gem_shmem_print_info, 205 .pin = panfrost_gem_pin, 206 .unpin = drm_gem_shmem_unpin, 207 .get_sg_table = drm_gem_shmem_get_sg_table, 208 .vmap = drm_gem_shmem_vmap, 209 .vunmap = drm_gem_shmem_vunmap, 210 .mmap = drm_gem_shmem_mmap, 211 }; 212 213 /** 214 * panfrost_gem_create_object - Implementation of driver->gem_create_object. 215 * @dev: DRM device 216 * @size: Size in bytes of the memory the object will reference 217 * 218 * This lets the GEM helpers allocate object structs for us, and keep 219 * our BO stats correct. 220 */ 221 struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size) 222 { 223 struct panfrost_gem_object *obj; 224 225 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 226 if (!obj) 227 return NULL; 228 229 INIT_LIST_HEAD(&obj->mappings.list); 230 mutex_init(&obj->mappings.lock); 231 obj->base.base.funcs = &panfrost_gem_funcs; 232 233 return &obj->base.base; 234 } 235 236 struct panfrost_gem_object * 237 panfrost_gem_create_with_handle(struct drm_file *file_priv, 238 struct drm_device *dev, size_t size, 239 u32 flags, 240 uint32_t *handle) 241 { 242 int ret; 243 struct drm_gem_shmem_object *shmem; 244 struct panfrost_gem_object *bo; 245 246 /* Round up heap allocations to 2MB to keep fault handling simple */ 247 if (flags & PANFROST_BO_HEAP) 248 size = roundup(size, SZ_2M); 249 250 shmem = drm_gem_shmem_create(dev, size); 251 if (IS_ERR(shmem)) 252 return ERR_CAST(shmem); 253 254 bo = to_panfrost_bo(&shmem->base); 255 bo->noexec = !!(flags & PANFROST_BO_NOEXEC); 256 bo->is_heap = !!(flags & PANFROST_BO_HEAP); 257 258 /* 259 * Allocate an id of idr table where the obj is registered 260 * and handle has the id what user can see. 261 */ 262 ret = drm_gem_handle_create(file_priv, &shmem->base, handle); 263 /* drop reference from allocate - handle holds it now. */ 264 drm_gem_object_put_unlocked(&shmem->base); 265 if (ret) 266 return ERR_PTR(ret); 267 268 return bo; 269 } 270 271 struct drm_gem_object * 272 panfrost_gem_prime_import_sg_table(struct drm_device *dev, 273 struct dma_buf_attachment *attach, 274 struct sg_table *sgt) 275 { 276 struct drm_gem_object *obj; 277 struct panfrost_gem_object *bo; 278 279 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); 280 if (IS_ERR(obj)) 281 return ERR_CAST(obj); 282 283 bo = to_panfrost_bo(obj); 284 bo->noexec = true; 285 286 return obj; 287 } 288