1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: monk liu <monk.liu@amd.com>
23  */
24 
25 #include <drm/drmP.h>
26 #include <drm/drm_auth.h>
27 #include "amdgpu.h"
28 #include "amdgpu_sched.h"
29 
30 static int amdgpu_ctx_priority_permit(struct drm_file *filp,
31 				      enum amd_sched_priority priority)
32 {
33 	/* NORMAL and below are accessible by everyone */
34 	if (priority <= AMD_SCHED_PRIORITY_NORMAL)
35 		return 0;
36 
37 	if (capable(CAP_SYS_NICE))
38 		return 0;
39 
40 	if (drm_is_current_master(filp))
41 		return 0;
42 
43 	return -EACCES;
44 }
45 
46 static int amdgpu_ctx_init(struct amdgpu_device *adev,
47 			   enum amd_sched_priority priority,
48 			   struct drm_file *filp,
49 			   struct amdgpu_ctx *ctx)
50 {
51 	unsigned i, j;
52 	int r;
53 
54 	if (priority < 0 || priority >= AMD_SCHED_PRIORITY_MAX)
55 		return -EINVAL;
56 
57 	r = amdgpu_ctx_priority_permit(filp, priority);
58 	if (r)
59 		return r;
60 
61 	memset(ctx, 0, sizeof(*ctx));
62 	ctx->adev = adev;
63 	kref_init(&ctx->refcount);
64 	spin_lock_init(&ctx->ring_lock);
65 	ctx->fences = kcalloc(amdgpu_sched_jobs * AMDGPU_MAX_RINGS,
66 			      sizeof(struct dma_fence*), GFP_KERNEL);
67 	if (!ctx->fences)
68 		return -ENOMEM;
69 
70 	mutex_init(&ctx->lock);
71 
72 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
73 		ctx->rings[i].sequence = 1;
74 		ctx->rings[i].fences = &ctx->fences[amdgpu_sched_jobs * i];
75 	}
76 
77 	ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
78 	ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
79 	ctx->init_priority = priority;
80 	ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
81 
82 	/* create context entity for each ring */
83 	for (i = 0; i < adev->num_rings; i++) {
84 		struct amdgpu_ring *ring = adev->rings[i];
85 		struct amd_sched_rq *rq;
86 
87 		rq = &ring->sched.sched_rq[priority];
88 
89 		if (ring == &adev->gfx.kiq.ring)
90 			continue;
91 
92 		r = amd_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
93 					  rq, amdgpu_sched_jobs);
94 		if (r)
95 			goto failed;
96 	}
97 
98 	r = amdgpu_queue_mgr_init(adev, &ctx->queue_mgr);
99 	if (r)
100 		goto failed;
101 
102 	return 0;
103 
104 failed:
105 	for (j = 0; j < i; j++)
106 		amd_sched_entity_fini(&adev->rings[j]->sched,
107 				      &ctx->rings[j].entity);
108 	kfree(ctx->fences);
109 	ctx->fences = NULL;
110 	return r;
111 }
112 
113 static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
114 {
115 	struct amdgpu_device *adev = ctx->adev;
116 	unsigned i, j;
117 
118 	if (!adev)
119 		return;
120 
121 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
122 		for (j = 0; j < amdgpu_sched_jobs; ++j)
123 			dma_fence_put(ctx->rings[i].fences[j]);
124 	kfree(ctx->fences);
125 	ctx->fences = NULL;
126 
127 	for (i = 0; i < adev->num_rings; i++)
128 		amd_sched_entity_fini(&adev->rings[i]->sched,
129 				      &ctx->rings[i].entity);
130 
131 	amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr);
132 
133 	mutex_destroy(&ctx->lock);
134 }
135 
136 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
137 			    struct amdgpu_fpriv *fpriv,
138 			    struct drm_file *filp,
139 			    enum amd_sched_priority priority,
140 			    uint32_t *id)
141 {
142 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
143 	struct amdgpu_ctx *ctx;
144 	int r;
145 
146 	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
147 	if (!ctx)
148 		return -ENOMEM;
149 
150 	mutex_lock(&mgr->lock);
151 	r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
152 	if (r < 0) {
153 		mutex_unlock(&mgr->lock);
154 		kfree(ctx);
155 		return r;
156 	}
157 
158 	*id = (uint32_t)r;
159 	r = amdgpu_ctx_init(adev, priority, filp, ctx);
160 	if (r) {
161 		idr_remove(&mgr->ctx_handles, *id);
162 		*id = 0;
163 		kfree(ctx);
164 	}
165 	mutex_unlock(&mgr->lock);
166 	return r;
167 }
168 
169 static void amdgpu_ctx_do_release(struct kref *ref)
170 {
171 	struct amdgpu_ctx *ctx;
172 
173 	ctx = container_of(ref, struct amdgpu_ctx, refcount);
174 
175 	amdgpu_ctx_fini(ctx);
176 
177 	kfree(ctx);
178 }
179 
180 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
181 {
182 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
183 	struct amdgpu_ctx *ctx;
184 
185 	mutex_lock(&mgr->lock);
186 	ctx = idr_remove(&mgr->ctx_handles, id);
187 	if (ctx)
188 		kref_put(&ctx->refcount, amdgpu_ctx_do_release);
189 	mutex_unlock(&mgr->lock);
190 	return ctx ? 0 : -EINVAL;
191 }
192 
193 static int amdgpu_ctx_query(struct amdgpu_device *adev,
194 			    struct amdgpu_fpriv *fpriv, uint32_t id,
195 			    union drm_amdgpu_ctx_out *out)
196 {
197 	struct amdgpu_ctx *ctx;
198 	struct amdgpu_ctx_mgr *mgr;
199 	unsigned reset_counter;
200 
201 	if (!fpriv)
202 		return -EINVAL;
203 
204 	mgr = &fpriv->ctx_mgr;
205 	mutex_lock(&mgr->lock);
206 	ctx = idr_find(&mgr->ctx_handles, id);
207 	if (!ctx) {
208 		mutex_unlock(&mgr->lock);
209 		return -EINVAL;
210 	}
211 
212 	/* TODO: these two are always zero */
213 	out->state.flags = 0x0;
214 	out->state.hangs = 0x0;
215 
216 	/* determine if a GPU reset has occured since the last call */
217 	reset_counter = atomic_read(&adev->gpu_reset_counter);
218 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
219 	if (ctx->reset_counter == reset_counter)
220 		out->state.reset_status = AMDGPU_CTX_NO_RESET;
221 	else
222 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
223 	ctx->reset_counter = reset_counter;
224 
225 	mutex_unlock(&mgr->lock);
226 	return 0;
227 }
228 
229 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
230 		     struct drm_file *filp)
231 {
232 	int r;
233 	uint32_t id;
234 	enum amd_sched_priority priority;
235 
236 	union drm_amdgpu_ctx *args = data;
237 	struct amdgpu_device *adev = dev->dev_private;
238 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
239 
240 	r = 0;
241 	id = args->in.ctx_id;
242 	priority = amdgpu_to_sched_priority(args->in.priority);
243 
244 	/* For backwards compatibility reasons, we need to accept
245 	 * ioctls with garbage in the priority field */
246 	if (priority == AMD_SCHED_PRIORITY_INVALID)
247 		priority = AMD_SCHED_PRIORITY_NORMAL;
248 
249 	switch (args->in.op) {
250 	case AMDGPU_CTX_OP_ALLOC_CTX:
251 		r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
252 		args->out.alloc.ctx_id = id;
253 		break;
254 	case AMDGPU_CTX_OP_FREE_CTX:
255 		r = amdgpu_ctx_free(fpriv, id);
256 		break;
257 	case AMDGPU_CTX_OP_QUERY_STATE:
258 		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
259 		break;
260 	default:
261 		return -EINVAL;
262 	}
263 
264 	return r;
265 }
266 
267 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
268 {
269 	struct amdgpu_ctx *ctx;
270 	struct amdgpu_ctx_mgr *mgr;
271 
272 	if (!fpriv)
273 		return NULL;
274 
275 	mgr = &fpriv->ctx_mgr;
276 
277 	mutex_lock(&mgr->lock);
278 	ctx = idr_find(&mgr->ctx_handles, id);
279 	if (ctx)
280 		kref_get(&ctx->refcount);
281 	mutex_unlock(&mgr->lock);
282 	return ctx;
283 }
284 
285 int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
286 {
287 	if (ctx == NULL)
288 		return -EINVAL;
289 
290 	kref_put(&ctx->refcount, amdgpu_ctx_do_release);
291 	return 0;
292 }
293 
294 int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
295 			      struct dma_fence *fence, uint64_t* handler)
296 {
297 	struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
298 	uint64_t seq = cring->sequence;
299 	unsigned idx = 0;
300 	struct dma_fence *other = NULL;
301 
302 	idx = seq & (amdgpu_sched_jobs - 1);
303 	other = cring->fences[idx];
304 	if (other)
305 		BUG_ON(!dma_fence_is_signaled(other));
306 
307 	dma_fence_get(fence);
308 
309 	spin_lock(&ctx->ring_lock);
310 	cring->fences[idx] = fence;
311 	cring->sequence++;
312 	spin_unlock(&ctx->ring_lock);
313 
314 	dma_fence_put(other);
315 	if (handler)
316 		*handler = seq;
317 
318 	return 0;
319 }
320 
321 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
322 				       struct amdgpu_ring *ring, uint64_t seq)
323 {
324 	struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
325 	struct dma_fence *fence;
326 
327 	spin_lock(&ctx->ring_lock);
328 
329 	if (seq == ~0ull)
330 		seq = ctx->rings[ring->idx].sequence - 1;
331 
332 	if (seq >= cring->sequence) {
333 		spin_unlock(&ctx->ring_lock);
334 		return ERR_PTR(-EINVAL);
335 	}
336 
337 
338 	if (seq + amdgpu_sched_jobs < cring->sequence) {
339 		spin_unlock(&ctx->ring_lock);
340 		return NULL;
341 	}
342 
343 	fence = dma_fence_get(cring->fences[seq & (amdgpu_sched_jobs - 1)]);
344 	spin_unlock(&ctx->ring_lock);
345 
346 	return fence;
347 }
348 
349 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
350 				  enum amd_sched_priority priority)
351 {
352 	int i;
353 	struct amdgpu_device *adev = ctx->adev;
354 	struct amd_sched_rq *rq;
355 	struct amd_sched_entity *entity;
356 	struct amdgpu_ring *ring;
357 	enum amd_sched_priority ctx_prio;
358 
359 	ctx->override_priority = priority;
360 
361 	ctx_prio = (ctx->override_priority == AMD_SCHED_PRIORITY_UNSET) ?
362 			ctx->init_priority : ctx->override_priority;
363 
364 	for (i = 0; i < adev->num_rings; i++) {
365 		ring = adev->rings[i];
366 		entity = &ctx->rings[i].entity;
367 		rq = &ring->sched.sched_rq[ctx_prio];
368 
369 		if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
370 			continue;
371 
372 		amd_sched_entity_set_rq(entity, rq);
373 	}
374 }
375 
376 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id)
377 {
378 	struct amdgpu_ctx_ring *cring = &ctx->rings[ring_id];
379 	unsigned idx = cring->sequence & (amdgpu_sched_jobs - 1);
380 	struct dma_fence *other = cring->fences[idx];
381 
382 	if (other) {
383 		signed long r;
384 		r = dma_fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
385 		if (r < 0) {
386 			DRM_ERROR("Error (%ld) waiting for fence!\n", r);
387 			return r;
388 		}
389 	}
390 
391 	return 0;
392 }
393 
394 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
395 {
396 	mutex_init(&mgr->lock);
397 	idr_init(&mgr->ctx_handles);
398 }
399 
400 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
401 {
402 	struct amdgpu_ctx *ctx;
403 	struct idr *idp;
404 	uint32_t id;
405 
406 	idp = &mgr->ctx_handles;
407 
408 	idr_for_each_entry(idp, ctx, id) {
409 		if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1)
410 			DRM_ERROR("ctx %p is still alive\n", ctx);
411 	}
412 
413 	idr_destroy(&mgr->ctx_handles);
414 	mutex_destroy(&mgr->lock);
415 }
416