1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2019 Collabora Ltd */ 3 4 #include <drm/drm_file.h> 5 #include <drm/drm_gem_shmem_helper.h> 6 #include <drm/panfrost_drm.h> 7 #include <linux/completion.h> 8 #include <linux/dma-buf-map.h> 9 #include <linux/iopoll.h> 10 #include <linux/pm_runtime.h> 11 #include <linux/slab.h> 12 #include <linux/uaccess.h> 13 14 #include "panfrost_device.h" 15 #include "panfrost_features.h" 16 #include "panfrost_gem.h" 17 #include "panfrost_issues.h" 18 #include "panfrost_job.h" 19 #include "panfrost_mmu.h" 20 #include "panfrost_perfcnt.h" 21 #include "panfrost_regs.h" 22 23 #define COUNTERS_PER_BLOCK 64 24 #define BYTES_PER_COUNTER 4 25 #define BLOCKS_PER_COREGROUP 8 26 #define V4_SHADERS_PER_COREGROUP 4 27 28 struct panfrost_perfcnt { 29 struct panfrost_gem_mapping *mapping; 30 size_t bosize; 31 void *buf; 32 struct panfrost_file_priv *user; 33 struct mutex lock; 34 struct completion dump_comp; 35 }; 36 37 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev) 38 { 39 complete(&pfdev->perfcnt->dump_comp); 40 } 41 42 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev) 43 { 44 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES); 45 } 46 47 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev) 48 { 49 u64 gpuva; 50 int ret; 51 52 reinit_completion(&pfdev->perfcnt->dump_comp); 53 gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT; 54 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, lower_32_bits(gpuva)); 55 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, upper_32_bits(gpuva)); 56 gpu_write(pfdev, GPU_INT_CLEAR, 57 GPU_IRQ_CLEAN_CACHES_COMPLETED | 58 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); 59 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE); 60 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp, 61 msecs_to_jiffies(1000)); 62 if (!ret) 63 ret = -ETIMEDOUT; 64 else if (ret > 0) 65 ret = 0; 66 67 return ret; 68 } 69 70 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, 71 struct drm_file *file_priv, 72 unsigned int counterset) 73 { 74 struct panfrost_file_priv *user = file_priv->driver_priv; 75 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 76 struct dma_buf_map map; 77 struct drm_gem_shmem_object *bo; 78 u32 cfg, as; 79 int ret; 80 81 if (user == perfcnt->user) 82 return 0; 83 else if (perfcnt->user) 84 return -EBUSY; 85 86 ret = pm_runtime_get_sync(pfdev->dev); 87 if (ret < 0) 88 goto err_put_pm; 89 90 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize); 91 if (IS_ERR(bo)) { 92 ret = PTR_ERR(bo); 93 goto err_put_pm; 94 } 95 96 /* Map the perfcnt buf in the address space attached to file_priv. */ 97 ret = panfrost_gem_open(&bo->base, file_priv); 98 if (ret) 99 goto err_put_bo; 100 101 perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base), 102 user); 103 if (!perfcnt->mapping) { 104 ret = -EINVAL; 105 goto err_close_bo; 106 } 107 108 ret = drm_gem_shmem_vmap(&bo->base, &map); 109 if (ret) 110 goto err_put_mapping; 111 perfcnt->buf = map.vaddr; 112 113 /* 114 * Invalidate the cache and clear the counters to start from a fresh 115 * state. 116 */ 117 reinit_completion(&pfdev->perfcnt->dump_comp); 118 gpu_write(pfdev, GPU_INT_CLEAR, 119 GPU_IRQ_CLEAN_CACHES_COMPLETED | 120 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); 121 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR); 122 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES); 123 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp, 124 msecs_to_jiffies(1000)); 125 if (!ret) { 126 ret = -ETIMEDOUT; 127 goto err_vunmap; 128 } 129 130 perfcnt->user = user; 131 132 as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu); 133 cfg = GPU_PERFCNT_CFG_AS(as) | 134 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL); 135 136 /* 137 * Bifrost GPUs have 2 set of counters, but we're only interested by 138 * the first one for now. 139 */ 140 if (panfrost_model_is_bifrost(pfdev)) 141 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset); 142 143 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff); 144 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff); 145 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff); 146 147 /* 148 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW 149 * counters. 150 */ 151 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) 152 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 153 else 154 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); 155 156 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg); 157 158 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) 159 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); 160 161 /* The BO ref is retained by the mapping. */ 162 drm_gem_object_put(&bo->base); 163 164 return 0; 165 166 err_vunmap: 167 drm_gem_shmem_vunmap(&bo->base, &map); 168 err_put_mapping: 169 panfrost_gem_mapping_put(perfcnt->mapping); 170 err_close_bo: 171 panfrost_gem_close(&bo->base, file_priv); 172 err_put_bo: 173 drm_gem_object_put(&bo->base); 174 err_put_pm: 175 pm_runtime_put(pfdev->dev); 176 return ret; 177 } 178 179 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, 180 struct drm_file *file_priv) 181 { 182 struct panfrost_file_priv *user = file_priv->driver_priv; 183 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 184 struct dma_buf_map map = DMA_BUF_MAP_INIT_VADDR(perfcnt->buf); 185 186 if (user != perfcnt->user) 187 return -EINVAL; 188 189 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0); 190 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0); 191 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0); 192 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 193 gpu_write(pfdev, GPU_PERFCNT_CFG, 194 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 195 196 perfcnt->user = NULL; 197 drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, &map); 198 perfcnt->buf = NULL; 199 panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv); 200 panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu); 201 panfrost_gem_mapping_put(perfcnt->mapping); 202 perfcnt->mapping = NULL; 203 pm_runtime_mark_last_busy(pfdev->dev); 204 pm_runtime_put_autosuspend(pfdev->dev); 205 206 return 0; 207 } 208 209 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, 210 struct drm_file *file_priv) 211 { 212 struct panfrost_device *pfdev = dev->dev_private; 213 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 214 struct drm_panfrost_perfcnt_enable *req = data; 215 int ret; 216 217 ret = panfrost_unstable_ioctl_check(); 218 if (ret) 219 return ret; 220 221 /* Only Bifrost GPUs have 2 set of counters. */ 222 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0)) 223 return -EINVAL; 224 225 mutex_lock(&perfcnt->lock); 226 if (req->enable) 227 ret = panfrost_perfcnt_enable_locked(pfdev, file_priv, 228 req->counterset); 229 else 230 ret = panfrost_perfcnt_disable_locked(pfdev, file_priv); 231 mutex_unlock(&perfcnt->lock); 232 233 return ret; 234 } 235 236 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data, 237 struct drm_file *file_priv) 238 { 239 struct panfrost_device *pfdev = dev->dev_private; 240 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 241 struct drm_panfrost_perfcnt_dump *req = data; 242 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr; 243 int ret; 244 245 ret = panfrost_unstable_ioctl_check(); 246 if (ret) 247 return ret; 248 249 mutex_lock(&perfcnt->lock); 250 if (perfcnt->user != file_priv->driver_priv) { 251 ret = -EINVAL; 252 goto out; 253 } 254 255 ret = panfrost_perfcnt_dump_locked(pfdev); 256 if (ret) 257 goto out; 258 259 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize)) 260 ret = -EFAULT; 261 262 out: 263 mutex_unlock(&perfcnt->lock); 264 265 return ret; 266 } 267 268 void panfrost_perfcnt_close(struct drm_file *file_priv) 269 { 270 struct panfrost_file_priv *pfile = file_priv->driver_priv; 271 struct panfrost_device *pfdev = pfile->pfdev; 272 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 273 274 pm_runtime_get_sync(pfdev->dev); 275 mutex_lock(&perfcnt->lock); 276 if (perfcnt->user == pfile) 277 panfrost_perfcnt_disable_locked(pfdev, file_priv); 278 mutex_unlock(&perfcnt->lock); 279 pm_runtime_mark_last_busy(pfdev->dev); 280 pm_runtime_put_autosuspend(pfdev->dev); 281 } 282 283 int panfrost_perfcnt_init(struct panfrost_device *pfdev) 284 { 285 struct panfrost_perfcnt *perfcnt; 286 size_t size; 287 288 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) { 289 unsigned int ncoregroups; 290 291 ncoregroups = hweight64(pfdev->features.l2_present); 292 size = ncoregroups * BLOCKS_PER_COREGROUP * 293 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; 294 } else { 295 unsigned int nl2c, ncores; 296 297 /* 298 * TODO: define a macro to extract the number of l2 caches from 299 * mem_features. 300 */ 301 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1; 302 303 /* 304 * shader_present might be sparse, but the counters layout 305 * forces to dump unused regions too, hence the fls64() call 306 * instead of hweight64(). 307 */ 308 ncores = fls64(pfdev->features.shader_present); 309 310 /* 311 * There's always one JM and one Tiler block, hence the '+ 2' 312 * here. 313 */ 314 size = (nl2c + ncores + 2) * 315 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; 316 } 317 318 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL); 319 if (!perfcnt) 320 return -ENOMEM; 321 322 perfcnt->bosize = size; 323 324 /* Start with everything disabled. */ 325 gpu_write(pfdev, GPU_PERFCNT_CFG, 326 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 327 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); 328 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); 329 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); 330 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 331 332 init_completion(&perfcnt->dump_comp); 333 mutex_init(&perfcnt->lock); 334 pfdev->perfcnt = perfcnt; 335 336 return 0; 337 } 338 339 void panfrost_perfcnt_fini(struct panfrost_device *pfdev) 340 { 341 /* Disable everything before leaving. */ 342 gpu_write(pfdev, GPU_PERFCNT_CFG, 343 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 344 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); 345 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); 346 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); 347 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 348 } 349