1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright 2019 Collabora Ltd */ 3 4 #include <drm/drm_file.h> 5 #include <drm/drm_gem_shmem_helper.h> 6 #include <drm/panfrost_drm.h> 7 #include <linux/completion.h> 8 #include <linux/iopoll.h> 9 #include <linux/pm_runtime.h> 10 #include <linux/slab.h> 11 #include <linux/uaccess.h> 12 13 #include "panfrost_device.h" 14 #include "panfrost_features.h" 15 #include "panfrost_gem.h" 16 #include "panfrost_issues.h" 17 #include "panfrost_job.h" 18 #include "panfrost_mmu.h" 19 #include "panfrost_perfcnt.h" 20 #include "panfrost_regs.h" 21 22 #define COUNTERS_PER_BLOCK 64 23 #define BYTES_PER_COUNTER 4 24 #define BLOCKS_PER_COREGROUP 8 25 #define V4_SHADERS_PER_COREGROUP 4 26 27 struct panfrost_perfcnt { 28 struct panfrost_gem_mapping *mapping; 29 size_t bosize; 30 void *buf; 31 struct panfrost_file_priv *user; 32 struct mutex lock; 33 struct completion dump_comp; 34 }; 35 36 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev) 37 { 38 complete(&pfdev->perfcnt->dump_comp); 39 } 40 41 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev) 42 { 43 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES); 44 } 45 46 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev) 47 { 48 u64 gpuva; 49 int ret; 50 51 reinit_completion(&pfdev->perfcnt->dump_comp); 52 gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT; 53 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva); 54 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32); 55 gpu_write(pfdev, GPU_INT_CLEAR, 56 GPU_IRQ_CLEAN_CACHES_COMPLETED | 57 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); 58 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE); 59 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp, 60 msecs_to_jiffies(1000)); 61 if (!ret) 62 ret = -ETIMEDOUT; 63 else if (ret > 0) 64 ret = 0; 65 66 return ret; 67 } 68 69 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, 70 struct drm_file *file_priv, 71 unsigned int counterset) 72 { 73 struct panfrost_file_priv *user = file_priv->driver_priv; 74 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 75 struct drm_gem_shmem_object *bo; 76 u32 cfg, as; 77 int ret; 78 79 if (user == perfcnt->user) 80 return 0; 81 else if (perfcnt->user) 82 return -EBUSY; 83 84 ret = pm_runtime_get_sync(pfdev->dev); 85 if (ret < 0) 86 goto err_put_pm; 87 88 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize); 89 if (IS_ERR(bo)) { 90 ret = PTR_ERR(bo); 91 goto err_put_pm; 92 } 93 94 /* Map the perfcnt buf in the address space attached to file_priv. */ 95 ret = panfrost_gem_open(&bo->base, file_priv); 96 if (ret) 97 goto err_put_bo; 98 99 perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base), 100 user); 101 if (!perfcnt->mapping) { 102 ret = -EINVAL; 103 goto err_close_bo; 104 } 105 106 perfcnt->buf = drm_gem_shmem_vmap(&bo->base); 107 if (IS_ERR(perfcnt->buf)) { 108 ret = PTR_ERR(perfcnt->buf); 109 goto err_put_mapping; 110 } 111 112 /* 113 * Invalidate the cache and clear the counters to start from a fresh 114 * state. 115 */ 116 reinit_completion(&pfdev->perfcnt->dump_comp); 117 gpu_write(pfdev, GPU_INT_CLEAR, 118 GPU_IRQ_CLEAN_CACHES_COMPLETED | 119 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); 120 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR); 121 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES); 122 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp, 123 msecs_to_jiffies(1000)); 124 if (!ret) { 125 ret = -ETIMEDOUT; 126 goto err_vunmap; 127 } 128 129 perfcnt->user = user; 130 131 as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu); 132 cfg = GPU_PERFCNT_CFG_AS(as) | 133 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL); 134 135 /* 136 * Bifrost GPUs have 2 set of counters, but we're only interested by 137 * the first one for now. 138 */ 139 if (panfrost_model_is_bifrost(pfdev)) 140 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset); 141 142 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff); 143 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff); 144 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff); 145 146 /* 147 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW 148 * counters. 149 */ 150 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) 151 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 152 else 153 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); 154 155 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg); 156 157 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) 158 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); 159 160 /* The BO ref is retained by the mapping. */ 161 drm_gem_object_put(&bo->base); 162 163 return 0; 164 165 err_vunmap: 166 drm_gem_shmem_vunmap(&bo->base, perfcnt->buf); 167 err_put_mapping: 168 panfrost_gem_mapping_put(perfcnt->mapping); 169 err_close_bo: 170 panfrost_gem_close(&bo->base, file_priv); 171 err_put_bo: 172 drm_gem_object_put(&bo->base); 173 err_put_pm: 174 pm_runtime_put(pfdev->dev); 175 return ret; 176 } 177 178 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, 179 struct drm_file *file_priv) 180 { 181 struct panfrost_file_priv *user = file_priv->driver_priv; 182 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 183 184 if (user != perfcnt->user) 185 return -EINVAL; 186 187 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0); 188 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0); 189 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0); 190 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 191 gpu_write(pfdev, GPU_PERFCNT_CFG, 192 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 193 194 perfcnt->user = NULL; 195 drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, perfcnt->buf); 196 perfcnt->buf = NULL; 197 panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv); 198 panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu); 199 panfrost_gem_mapping_put(perfcnt->mapping); 200 perfcnt->mapping = NULL; 201 pm_runtime_mark_last_busy(pfdev->dev); 202 pm_runtime_put_autosuspend(pfdev->dev); 203 204 return 0; 205 } 206 207 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, 208 struct drm_file *file_priv) 209 { 210 struct panfrost_device *pfdev = dev->dev_private; 211 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 212 struct drm_panfrost_perfcnt_enable *req = data; 213 int ret; 214 215 ret = panfrost_unstable_ioctl_check(); 216 if (ret) 217 return ret; 218 219 /* Only Bifrost GPUs have 2 set of counters. */ 220 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0)) 221 return -EINVAL; 222 223 mutex_lock(&perfcnt->lock); 224 if (req->enable) 225 ret = panfrost_perfcnt_enable_locked(pfdev, file_priv, 226 req->counterset); 227 else 228 ret = panfrost_perfcnt_disable_locked(pfdev, file_priv); 229 mutex_unlock(&perfcnt->lock); 230 231 return ret; 232 } 233 234 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data, 235 struct drm_file *file_priv) 236 { 237 struct panfrost_device *pfdev = dev->dev_private; 238 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 239 struct drm_panfrost_perfcnt_dump *req = data; 240 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr; 241 int ret; 242 243 ret = panfrost_unstable_ioctl_check(); 244 if (ret) 245 return ret; 246 247 mutex_lock(&perfcnt->lock); 248 if (perfcnt->user != file_priv->driver_priv) { 249 ret = -EINVAL; 250 goto out; 251 } 252 253 ret = panfrost_perfcnt_dump_locked(pfdev); 254 if (ret) 255 goto out; 256 257 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize)) 258 ret = -EFAULT; 259 260 out: 261 mutex_unlock(&perfcnt->lock); 262 263 return ret; 264 } 265 266 void panfrost_perfcnt_close(struct drm_file *file_priv) 267 { 268 struct panfrost_file_priv *pfile = file_priv->driver_priv; 269 struct panfrost_device *pfdev = pfile->pfdev; 270 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; 271 272 pm_runtime_get_sync(pfdev->dev); 273 mutex_lock(&perfcnt->lock); 274 if (perfcnt->user == pfile) 275 panfrost_perfcnt_disable_locked(pfdev, file_priv); 276 mutex_unlock(&perfcnt->lock); 277 pm_runtime_mark_last_busy(pfdev->dev); 278 pm_runtime_put_autosuspend(pfdev->dev); 279 } 280 281 int panfrost_perfcnt_init(struct panfrost_device *pfdev) 282 { 283 struct panfrost_perfcnt *perfcnt; 284 size_t size; 285 286 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) { 287 unsigned int ncoregroups; 288 289 ncoregroups = hweight64(pfdev->features.l2_present); 290 size = ncoregroups * BLOCKS_PER_COREGROUP * 291 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; 292 } else { 293 unsigned int nl2c, ncores; 294 295 /* 296 * TODO: define a macro to extract the number of l2 caches from 297 * mem_features. 298 */ 299 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1; 300 301 /* 302 * shader_present might be sparse, but the counters layout 303 * forces to dump unused regions too, hence the fls64() call 304 * instead of hweight64(). 305 */ 306 ncores = fls64(pfdev->features.shader_present); 307 308 /* 309 * There's always one JM and one Tiler block, hence the '+ 2' 310 * here. 311 */ 312 size = (nl2c + ncores + 2) * 313 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; 314 } 315 316 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL); 317 if (!perfcnt) 318 return -ENOMEM; 319 320 perfcnt->bosize = size; 321 322 /* Start with everything disabled. */ 323 gpu_write(pfdev, GPU_PERFCNT_CFG, 324 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 325 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); 326 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); 327 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); 328 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 329 330 init_completion(&perfcnt->dump_comp); 331 mutex_init(&perfcnt->lock); 332 pfdev->perfcnt = perfcnt; 333 334 return 0; 335 } 336 337 void panfrost_perfcnt_fini(struct panfrost_device *pfdev) 338 { 339 /* Disable everything before leaving. */ 340 gpu_write(pfdev, GPU_PERFCNT_CFG, 341 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); 342 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); 343 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); 344 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); 345 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); 346 } 347