1 /*
2  * Copyright (C) 2013 Red Hat
3  * Author: Rob Clark <robdclark@gmail.com>
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms of the GNU General Public License version 2 as published by
7  * the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program.  If not, see <http://www.gnu.org/licenses/>.
16  */
17 
18 #include "msm_drv.h"
19 #include "msm_gpu.h"
20 #include "msm_gem.h"
21 
22 /*
23  * Cmdstream submission:
24  */
25 
26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
27 #define BO_VALID    0x8000   /* is current addr in cmdstream correct/valid? */
28 #define BO_LOCKED   0x4000
29 #define BO_PINNED   0x2000
30 
31 static struct msm_gem_submit *submit_create(struct drm_device *dev,
32 		struct msm_gpu *gpu, int nr_bos, int nr_cmds)
33 {
34 	struct msm_gem_submit *submit;
35 	int sz = sizeof(*submit) + (nr_bos * sizeof(submit->bos[0])) +
36 			(nr_cmds * sizeof(*submit->cmd));
37 
38 	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
39 	if (!submit)
40 		return NULL;
41 
42 	submit->dev = dev;
43 	submit->gpu = gpu;
44 	submit->fence = NULL;
45 	submit->pid = get_pid(task_pid(current));
46 	submit->cmd = (void *)&submit->bos[nr_bos];
47 
48 	/* initially, until copy_from_user() and bo lookup succeeds: */
49 	submit->nr_bos = 0;
50 	submit->nr_cmds = 0;
51 
52 	INIT_LIST_HEAD(&submit->node);
53 	INIT_LIST_HEAD(&submit->bo_list);
54 	ww_acquire_init(&submit->ticket, &reservation_ww_class);
55 
56 	return submit;
57 }
58 
59 void msm_gem_submit_free(struct msm_gem_submit *submit)
60 {
61 	fence_put(submit->fence);
62 	list_del(&submit->node);
63 	put_pid(submit->pid);
64 	kfree(submit);
65 }
66 
67 static inline unsigned long __must_check
68 copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
69 {
70 	if (access_ok(VERIFY_READ, from, n))
71 		return __copy_from_user_inatomic(to, from, n);
72 	return -EFAULT;
73 }
74 
75 static int submit_lookup_objects(struct msm_gem_submit *submit,
76 		struct drm_msm_gem_submit *args, struct drm_file *file)
77 {
78 	unsigned i;
79 	int ret = 0;
80 
81 	spin_lock(&file->table_lock);
82 	pagefault_disable();
83 
84 	for (i = 0; i < args->nr_bos; i++) {
85 		struct drm_msm_gem_submit_bo submit_bo;
86 		struct drm_gem_object *obj;
87 		struct msm_gem_object *msm_obj;
88 		void __user *userptr =
89 			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
90 
91 		/* make sure we don't have garbage flags, in case we hit
92 		 * error path before flags is initialized:
93 		 */
94 		submit->bos[i].flags = 0;
95 
96 		ret = copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo));
97 		if (unlikely(ret)) {
98 			pagefault_enable();
99 			spin_unlock(&file->table_lock);
100 			ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
101 			if (ret)
102 				goto out;
103 			spin_lock(&file->table_lock);
104 			pagefault_disable();
105 		}
106 
107 		if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
108 			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
109 			ret = -EINVAL;
110 			goto out_unlock;
111 		}
112 
113 		submit->bos[i].flags = submit_bo.flags;
114 		/* in validate_objects() we figure out if this is true: */
115 		submit->bos[i].iova  = submit_bo.presumed;
116 
117 		/* normally use drm_gem_object_lookup(), but for bulk lookup
118 		 * all under single table_lock just hit object_idr directly:
119 		 */
120 		obj = idr_find(&file->object_idr, submit_bo.handle);
121 		if (!obj) {
122 			DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
123 			ret = -EINVAL;
124 			goto out_unlock;
125 		}
126 
127 		msm_obj = to_msm_bo(obj);
128 
129 		if (!list_empty(&msm_obj->submit_entry)) {
130 			DRM_ERROR("handle %u at index %u already on submit list\n",
131 					submit_bo.handle, i);
132 			ret = -EINVAL;
133 			goto out_unlock;
134 		}
135 
136 		drm_gem_object_reference(obj);
137 
138 		submit->bos[i].obj = msm_obj;
139 
140 		list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
141 	}
142 
143 out_unlock:
144 	pagefault_enable();
145 	spin_unlock(&file->table_lock);
146 
147 out:
148 	submit->nr_bos = i;
149 
150 	return ret;
151 }
152 
153 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
154 {
155 	struct msm_gem_object *msm_obj = submit->bos[i].obj;
156 
157 	if (submit->bos[i].flags & BO_PINNED)
158 		msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
159 
160 	if (submit->bos[i].flags & BO_LOCKED)
161 		ww_mutex_unlock(&msm_obj->resv->lock);
162 
163 	if (!(submit->bos[i].flags & BO_VALID))
164 		submit->bos[i].iova = 0;
165 
166 	submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
167 }
168 
169 /* This is where we make sure all the bo's are reserved and pin'd: */
170 static int submit_lock_objects(struct msm_gem_submit *submit)
171 {
172 	int contended, slow_locked = -1, i, ret = 0;
173 
174 retry:
175 	for (i = 0; i < submit->nr_bos; i++) {
176 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
177 
178 		if (slow_locked == i)
179 			slow_locked = -1;
180 
181 		contended = i;
182 
183 		if (!(submit->bos[i].flags & BO_LOCKED)) {
184 			ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
185 					&submit->ticket);
186 			if (ret)
187 				goto fail;
188 			submit->bos[i].flags |= BO_LOCKED;
189 		}
190 	}
191 
192 	ww_acquire_done(&submit->ticket);
193 
194 	return 0;
195 
196 fail:
197 	for (; i >= 0; i--)
198 		submit_unlock_unpin_bo(submit, i);
199 
200 	if (slow_locked > 0)
201 		submit_unlock_unpin_bo(submit, slow_locked);
202 
203 	if (ret == -EDEADLK) {
204 		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
205 		/* we lost out in a seqno race, lock and retry.. */
206 		ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
207 				&submit->ticket);
208 		if (!ret) {
209 			submit->bos[contended].flags |= BO_LOCKED;
210 			slow_locked = contended;
211 			goto retry;
212 		}
213 	}
214 
215 	return ret;
216 }
217 
218 static int submit_fence_sync(struct msm_gem_submit *submit)
219 {
220 	int i, ret = 0;
221 
222 	for (i = 0; i < submit->nr_bos; i++) {
223 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
224 		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
225 
226 		ret = msm_gem_sync_object(&msm_obj->base, submit->gpu->fctx, write);
227 		if (ret)
228 			break;
229 	}
230 
231 	return ret;
232 }
233 
234 static int submit_pin_objects(struct msm_gem_submit *submit)
235 {
236 	int i, ret = 0;
237 
238 	submit->valid = true;
239 
240 	for (i = 0; i < submit->nr_bos; i++) {
241 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
242 		uint32_t iova;
243 
244 		/* if locking succeeded, pin bo: */
245 		ret = msm_gem_get_iova_locked(&msm_obj->base,
246 				submit->gpu->id, &iova);
247 
248 		if (ret)
249 			break;
250 
251 		submit->bos[i].flags |= BO_PINNED;
252 
253 		if (iova == submit->bos[i].iova) {
254 			submit->bos[i].flags |= BO_VALID;
255 		} else {
256 			submit->bos[i].iova = iova;
257 			/* iova changed, so address in cmdstream is not valid: */
258 			submit->bos[i].flags &= ~BO_VALID;
259 			submit->valid = false;
260 		}
261 	}
262 
263 	return ret;
264 }
265 
266 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
267 		struct msm_gem_object **obj, uint32_t *iova, bool *valid)
268 {
269 	if (idx >= submit->nr_bos) {
270 		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
271 				idx, submit->nr_bos);
272 		return -EINVAL;
273 	}
274 
275 	if (obj)
276 		*obj = submit->bos[idx].obj;
277 	if (iova)
278 		*iova = submit->bos[idx].iova;
279 	if (valid)
280 		*valid = !!(submit->bos[idx].flags & BO_VALID);
281 
282 	return 0;
283 }
284 
285 /* process the reloc's and patch up the cmdstream as needed: */
286 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
287 		uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
288 {
289 	uint32_t i, last_offset = 0;
290 	uint32_t *ptr;
291 	int ret;
292 
293 	if (offset % 4) {
294 		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
295 		return -EINVAL;
296 	}
297 
298 	/* For now, just map the entire thing.  Eventually we probably
299 	 * to do it page-by-page, w/ kmap() if not vmap()d..
300 	 */
301 	ptr = msm_gem_get_vaddr_locked(&obj->base);
302 
303 	if (IS_ERR(ptr)) {
304 		ret = PTR_ERR(ptr);
305 		DBG("failed to map: %d", ret);
306 		return ret;
307 	}
308 
309 	for (i = 0; i < nr_relocs; i++) {
310 		struct drm_msm_gem_submit_reloc submit_reloc;
311 		void __user *userptr =
312 			u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
313 		uint32_t iova, off;
314 		bool valid;
315 
316 		ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
317 		if (ret)
318 			return -EFAULT;
319 
320 		if (submit_reloc.submit_offset % 4) {
321 			DRM_ERROR("non-aligned reloc offset: %u\n",
322 					submit_reloc.submit_offset);
323 			return -EINVAL;
324 		}
325 
326 		/* offset in dwords: */
327 		off = submit_reloc.submit_offset / 4;
328 
329 		if ((off >= (obj->base.size / 4)) ||
330 				(off < last_offset)) {
331 			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
332 			return -EINVAL;
333 		}
334 
335 		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
336 		if (ret)
337 			return ret;
338 
339 		if (valid)
340 			continue;
341 
342 		iova += submit_reloc.reloc_offset;
343 
344 		if (submit_reloc.shift < 0)
345 			iova >>= -submit_reloc.shift;
346 		else
347 			iova <<= submit_reloc.shift;
348 
349 		ptr[off] = iova | submit_reloc.or;
350 
351 		last_offset = off;
352 	}
353 
354 	msm_gem_put_vaddr_locked(&obj->base);
355 
356 	return 0;
357 }
358 
359 static void submit_cleanup(struct msm_gem_submit *submit)
360 {
361 	unsigned i;
362 
363 	for (i = 0; i < submit->nr_bos; i++) {
364 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
365 		submit_unlock_unpin_bo(submit, i);
366 		list_del_init(&msm_obj->submit_entry);
367 		drm_gem_object_unreference(&msm_obj->base);
368 	}
369 
370 	ww_acquire_fini(&submit->ticket);
371 }
372 
373 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
374 		struct drm_file *file)
375 {
376 	struct msm_drm_private *priv = dev->dev_private;
377 	struct drm_msm_gem_submit *args = data;
378 	struct msm_file_private *ctx = file->driver_priv;
379 	struct msm_gem_submit *submit;
380 	struct msm_gpu *gpu = priv->gpu;
381 	unsigned i;
382 	int ret;
383 
384 	if (!gpu)
385 		return -ENXIO;
386 
387 	/* for now, we just have 3d pipe.. eventually this would need to
388 	 * be more clever to dispatch to appropriate gpu module:
389 	 */
390 	if (args->pipe != MSM_PIPE_3D0)
391 		return -EINVAL;
392 
393 	ret = mutex_lock_interruptible(&dev->struct_mutex);
394 	if (ret)
395 		return ret;
396 
397 	priv->struct_mutex_task = current;
398 
399 	submit = submit_create(dev, gpu, args->nr_bos, args->nr_cmds);
400 	if (!submit) {
401 		ret = -ENOMEM;
402 		goto out_unlock;
403 	}
404 
405 	ret = submit_lookup_objects(submit, args, file);
406 	if (ret)
407 		goto out;
408 
409 	ret = submit_lock_objects(submit);
410 	if (ret)
411 		goto out;
412 
413 	ret = submit_fence_sync(submit);
414 	if (ret)
415 		goto out;
416 
417 	ret = submit_pin_objects(submit);
418 	if (ret)
419 		goto out;
420 
421 	for (i = 0; i < args->nr_cmds; i++) {
422 		struct drm_msm_gem_submit_cmd submit_cmd;
423 		void __user *userptr =
424 			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
425 		struct msm_gem_object *msm_obj;
426 		uint32_t iova;
427 
428 		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
429 		if (ret) {
430 			ret = -EFAULT;
431 			goto out;
432 		}
433 
434 		/* validate input from userspace: */
435 		switch (submit_cmd.type) {
436 		case MSM_SUBMIT_CMD_BUF:
437 		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
438 		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
439 			break;
440 		default:
441 			DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
442 			ret = -EINVAL;
443 			goto out;
444 		}
445 
446 		ret = submit_bo(submit, submit_cmd.submit_idx,
447 				&msm_obj, &iova, NULL);
448 		if (ret)
449 			goto out;
450 
451 		if (submit_cmd.size % 4) {
452 			DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
453 					submit_cmd.size);
454 			ret = -EINVAL;
455 			goto out;
456 		}
457 
458 		if ((submit_cmd.size + submit_cmd.submit_offset) >=
459 				msm_obj->base.size) {
460 			DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
461 			ret = -EINVAL;
462 			goto out;
463 		}
464 
465 		submit->cmd[i].type = submit_cmd.type;
466 		submit->cmd[i].size = submit_cmd.size / 4;
467 		submit->cmd[i].iova = iova + submit_cmd.submit_offset;
468 		submit->cmd[i].idx  = submit_cmd.submit_idx;
469 
470 		if (submit->valid)
471 			continue;
472 
473 		ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
474 				submit_cmd.nr_relocs, submit_cmd.relocs);
475 		if (ret)
476 			goto out;
477 	}
478 
479 	submit->nr_cmds = i;
480 
481 	ret = msm_gpu_submit(gpu, submit, ctx);
482 
483 	args->fence = submit->fence->seqno;
484 
485 out:
486 	submit_cleanup(submit);
487 	if (ret)
488 		msm_gem_submit_free(submit);
489 out_unlock:
490 	priv->struct_mutex_task = NULL;
491 	mutex_unlock(&dev->struct_mutex);
492 	return ret;
493 }
494