1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2019 Collabora Ltd */
3 
4 #include <drm/drm_file.h>
5 #include <drm/drm_gem_shmem_helper.h>
6 #include <drm/panfrost_drm.h>
7 #include <linux/completion.h>
8 #include <linux/dma-buf-map.h>
9 #include <linux/iopoll.h>
10 #include <linux/pm_runtime.h>
11 #include <linux/slab.h>
12 #include <linux/uaccess.h>
13 
14 #include "panfrost_device.h"
15 #include "panfrost_features.h"
16 #include "panfrost_gem.h"
17 #include "panfrost_issues.h"
18 #include "panfrost_job.h"
19 #include "panfrost_mmu.h"
20 #include "panfrost_perfcnt.h"
21 #include "panfrost_regs.h"
22 
23 #define COUNTERS_PER_BLOCK		64
24 #define BYTES_PER_COUNTER		4
25 #define BLOCKS_PER_COREGROUP		8
26 #define V4_SHADERS_PER_COREGROUP	4
27 
28 struct panfrost_perfcnt {
29 	struct panfrost_gem_mapping *mapping;
30 	size_t bosize;
31 	void *buf;
32 	struct panfrost_file_priv *user;
33 	struct mutex lock;
34 	struct completion dump_comp;
35 };
36 
37 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
38 {
39 	complete(&pfdev->perfcnt->dump_comp);
40 }
41 
42 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
43 {
44 	gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
45 }
46 
47 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
48 {
49 	u64 gpuva;
50 	int ret;
51 
52 	reinit_completion(&pfdev->perfcnt->dump_comp);
53 	gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
54 	gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
55 	gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
56 	gpu_write(pfdev, GPU_INT_CLEAR,
57 		  GPU_IRQ_CLEAN_CACHES_COMPLETED |
58 		  GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
59 	gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
60 	ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
61 							msecs_to_jiffies(1000));
62 	if (!ret)
63 		ret = -ETIMEDOUT;
64 	else if (ret > 0)
65 		ret = 0;
66 
67 	return ret;
68 }
69 
70 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
71 					  struct drm_file *file_priv,
72 					  unsigned int counterset)
73 {
74 	struct panfrost_file_priv *user = file_priv->driver_priv;
75 	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
76 	struct dma_buf_map map;
77 	struct drm_gem_shmem_object *bo;
78 	u32 cfg, as;
79 	int ret;
80 
81 	if (user == perfcnt->user)
82 		return 0;
83 	else if (perfcnt->user)
84 		return -EBUSY;
85 
86 	ret = pm_runtime_get_sync(pfdev->dev);
87 	if (ret < 0)
88 		goto err_put_pm;
89 
90 	bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
91 	if (IS_ERR(bo)) {
92 		ret = PTR_ERR(bo);
93 		goto err_put_pm;
94 	}
95 
96 	/* Map the perfcnt buf in the address space attached to file_priv. */
97 	ret = panfrost_gem_open(&bo->base, file_priv);
98 	if (ret)
99 		goto err_put_bo;
100 
101 	perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
102 						    user);
103 	if (!perfcnt->mapping) {
104 		ret = -EINVAL;
105 		goto err_close_bo;
106 	}
107 
108 	ret = drm_gem_shmem_vmap(&bo->base, &map);
109 	if (ret)
110 		goto err_put_mapping;
111 	perfcnt->buf = map.vaddr;
112 
113 	/*
114 	 * Invalidate the cache and clear the counters to start from a fresh
115 	 * state.
116 	 */
117 	reinit_completion(&pfdev->perfcnt->dump_comp);
118 	gpu_write(pfdev, GPU_INT_CLEAR,
119 		  GPU_IRQ_CLEAN_CACHES_COMPLETED |
120 		  GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
121 	gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
122 	gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
123 	ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
124 					  msecs_to_jiffies(1000));
125 	if (!ret) {
126 		ret = -ETIMEDOUT;
127 		goto err_vunmap;
128 	}
129 
130 	perfcnt->user = user;
131 
132 	as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
133 	cfg = GPU_PERFCNT_CFG_AS(as) |
134 	      GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
135 
136 	/*
137 	 * Bifrost GPUs have 2 set of counters, but we're only interested by
138 	 * the first one for now.
139 	 */
140 	if (panfrost_model_is_bifrost(pfdev))
141 		cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
142 
143 	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
144 	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
145 	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
146 
147 	/*
148 	 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
149 	 * counters.
150 	 */
151 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
152 		gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
153 	else
154 		gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
155 
156 	gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
157 
158 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
159 		gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
160 
161 	/* The BO ref is retained by the mapping. */
162 	drm_gem_object_put(&bo->base);
163 
164 	return 0;
165 
166 err_vunmap:
167 	drm_gem_shmem_vunmap(&bo->base, &map);
168 err_put_mapping:
169 	panfrost_gem_mapping_put(perfcnt->mapping);
170 err_close_bo:
171 	panfrost_gem_close(&bo->base, file_priv);
172 err_put_bo:
173 	drm_gem_object_put(&bo->base);
174 err_put_pm:
175 	pm_runtime_put(pfdev->dev);
176 	return ret;
177 }
178 
179 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
180 					   struct drm_file *file_priv)
181 {
182 	struct panfrost_file_priv *user = file_priv->driver_priv;
183 	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
184 	struct dma_buf_map map = DMA_BUF_MAP_INIT_VADDR(perfcnt->buf);
185 
186 	if (user != perfcnt->user)
187 		return -EINVAL;
188 
189 	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
190 	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
191 	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
192 	gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
193 	gpu_write(pfdev, GPU_PERFCNT_CFG,
194 		  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
195 
196 	perfcnt->user = NULL;
197 	drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, &map);
198 	perfcnt->buf = NULL;
199 	panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
200 	panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
201 	panfrost_gem_mapping_put(perfcnt->mapping);
202 	perfcnt->mapping = NULL;
203 	pm_runtime_mark_last_busy(pfdev->dev);
204 	pm_runtime_put_autosuspend(pfdev->dev);
205 
206 	return 0;
207 }
208 
209 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
210 				  struct drm_file *file_priv)
211 {
212 	struct panfrost_device *pfdev = dev->dev_private;
213 	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
214 	struct drm_panfrost_perfcnt_enable *req = data;
215 	int ret;
216 
217 	ret = panfrost_unstable_ioctl_check();
218 	if (ret)
219 		return ret;
220 
221 	/* Only Bifrost GPUs have 2 set of counters. */
222 	if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
223 		return -EINVAL;
224 
225 	mutex_lock(&perfcnt->lock);
226 	if (req->enable)
227 		ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
228 						     req->counterset);
229 	else
230 		ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
231 	mutex_unlock(&perfcnt->lock);
232 
233 	return ret;
234 }
235 
236 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
237 				struct drm_file *file_priv)
238 {
239 	struct panfrost_device *pfdev = dev->dev_private;
240 	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
241 	struct drm_panfrost_perfcnt_dump *req = data;
242 	void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
243 	int ret;
244 
245 	ret = panfrost_unstable_ioctl_check();
246 	if (ret)
247 		return ret;
248 
249 	mutex_lock(&perfcnt->lock);
250 	if (perfcnt->user != file_priv->driver_priv) {
251 		ret = -EINVAL;
252 		goto out;
253 	}
254 
255 	ret = panfrost_perfcnt_dump_locked(pfdev);
256 	if (ret)
257 		goto out;
258 
259 	if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
260 		ret = -EFAULT;
261 
262 out:
263 	mutex_unlock(&perfcnt->lock);
264 
265 	return ret;
266 }
267 
268 void panfrost_perfcnt_close(struct drm_file *file_priv)
269 {
270 	struct panfrost_file_priv *pfile = file_priv->driver_priv;
271 	struct panfrost_device *pfdev = pfile->pfdev;
272 	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
273 
274 	pm_runtime_get_sync(pfdev->dev);
275 	mutex_lock(&perfcnt->lock);
276 	if (perfcnt->user == pfile)
277 		panfrost_perfcnt_disable_locked(pfdev, file_priv);
278 	mutex_unlock(&perfcnt->lock);
279 	pm_runtime_mark_last_busy(pfdev->dev);
280 	pm_runtime_put_autosuspend(pfdev->dev);
281 }
282 
283 int panfrost_perfcnt_init(struct panfrost_device *pfdev)
284 {
285 	struct panfrost_perfcnt *perfcnt;
286 	size_t size;
287 
288 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
289 		unsigned int ncoregroups;
290 
291 		ncoregroups = hweight64(pfdev->features.l2_present);
292 		size = ncoregroups * BLOCKS_PER_COREGROUP *
293 		       COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
294 	} else {
295 		unsigned int nl2c, ncores;
296 
297 		/*
298 		 * TODO: define a macro to extract the number of l2 caches from
299 		 * mem_features.
300 		 */
301 		nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
302 
303 		/*
304 		 * shader_present might be sparse, but the counters layout
305 		 * forces to dump unused regions too, hence the fls64() call
306 		 * instead of hweight64().
307 		 */
308 		ncores = fls64(pfdev->features.shader_present);
309 
310 		/*
311 		 * There's always one JM and one Tiler block, hence the '+ 2'
312 		 * here.
313 		 */
314 		size = (nl2c + ncores + 2) *
315 		       COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
316 	}
317 
318 	perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
319 	if (!perfcnt)
320 		return -ENOMEM;
321 
322 	perfcnt->bosize = size;
323 
324 	/* Start with everything disabled. */
325 	gpu_write(pfdev, GPU_PERFCNT_CFG,
326 		  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
327 	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
328 	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
329 	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
330 	gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
331 
332 	init_completion(&perfcnt->dump_comp);
333 	mutex_init(&perfcnt->lock);
334 	pfdev->perfcnt = perfcnt;
335 
336 	return 0;
337 }
338 
339 void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
340 {
341 	/* Disable everything before leaving. */
342 	gpu_write(pfdev, GPU_PERFCNT_CFG,
343 		  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
344 	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
345 	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
346 	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
347 	gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
348 }
349