1 /*
2  * Copyright (C) 2015 Etnaviv Project
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms of the GNU General Public License version 2 as published by
6  * the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program.  If not, see <http://www.gnu.org/licenses/>.
15  */
16 
17 #include <linux/reservation.h>
18 #include "etnaviv_cmdbuf.h"
19 #include "etnaviv_drv.h"
20 #include "etnaviv_gpu.h"
21 #include "etnaviv_gem.h"
22 
23 /*
24  * Cmdstream submission:
25  */
26 
27 #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE)
28 /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */
29 #define BO_LOCKED   0x4000
30 #define BO_PINNED   0x2000
31 
32 static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
33 		struct etnaviv_gpu *gpu, size_t nr)
34 {
35 	struct etnaviv_gem_submit *submit;
36 	size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit));
37 
38 	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
39 	if (submit) {
40 		submit->dev = dev;
41 		submit->gpu = gpu;
42 
43 		/* initially, until copy_from_user() and bo lookup succeeds: */
44 		submit->nr_bos = 0;
45 
46 		ww_acquire_init(&submit->ticket, &reservation_ww_class);
47 	}
48 
49 	return submit;
50 }
51 
52 static int submit_lookup_objects(struct etnaviv_gem_submit *submit,
53 	struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos,
54 	unsigned nr_bos)
55 {
56 	struct drm_etnaviv_gem_submit_bo *bo;
57 	unsigned i;
58 	int ret = 0;
59 
60 	spin_lock(&file->table_lock);
61 
62 	for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) {
63 		struct drm_gem_object *obj;
64 
65 		if (bo->flags & BO_INVALID_FLAGS) {
66 			DRM_ERROR("invalid flags: %x\n", bo->flags);
67 			ret = -EINVAL;
68 			goto out_unlock;
69 		}
70 
71 		submit->bos[i].flags = bo->flags;
72 
73 		/* normally use drm_gem_object_lookup(), but for bulk lookup
74 		 * all under single table_lock just hit object_idr directly:
75 		 */
76 		obj = idr_find(&file->object_idr, bo->handle);
77 		if (!obj) {
78 			DRM_ERROR("invalid handle %u at index %u\n",
79 				  bo->handle, i);
80 			ret = -EINVAL;
81 			goto out_unlock;
82 		}
83 
84 		/*
85 		 * Take a refcount on the object. The file table lock
86 		 * prevents the object_idr's refcount on this being dropped.
87 		 */
88 		drm_gem_object_reference(obj);
89 
90 		submit->bos[i].obj = to_etnaviv_bo(obj);
91 	}
92 
93 out_unlock:
94 	submit->nr_bos = i;
95 	spin_unlock(&file->table_lock);
96 
97 	return ret;
98 }
99 
100 static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)
101 {
102 	if (submit->bos[i].flags & BO_LOCKED) {
103 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
104 
105 		ww_mutex_unlock(&etnaviv_obj->resv->lock);
106 		submit->bos[i].flags &= ~BO_LOCKED;
107 	}
108 }
109 
110 static int submit_lock_objects(struct etnaviv_gem_submit *submit)
111 {
112 	int contended, slow_locked = -1, i, ret = 0;
113 
114 retry:
115 	for (i = 0; i < submit->nr_bos; i++) {
116 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
117 
118 		if (slow_locked == i)
119 			slow_locked = -1;
120 
121 		contended = i;
122 
123 		if (!(submit->bos[i].flags & BO_LOCKED)) {
124 			ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock,
125 					&submit->ticket);
126 			if (ret == -EALREADY)
127 				DRM_ERROR("BO at index %u already on submit list\n",
128 					  i);
129 			if (ret)
130 				goto fail;
131 			submit->bos[i].flags |= BO_LOCKED;
132 		}
133 	}
134 
135 	ww_acquire_done(&submit->ticket);
136 
137 	return 0;
138 
139 fail:
140 	for (; i >= 0; i--)
141 		submit_unlock_object(submit, i);
142 
143 	if (slow_locked > 0)
144 		submit_unlock_object(submit, slow_locked);
145 
146 	if (ret == -EDEADLK) {
147 		struct etnaviv_gem_object *etnaviv_obj;
148 
149 		etnaviv_obj = submit->bos[contended].obj;
150 
151 		/* we lost out in a seqno race, lock and retry.. */
152 		ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock,
153 				&submit->ticket);
154 		if (!ret) {
155 			submit->bos[contended].flags |= BO_LOCKED;
156 			slow_locked = contended;
157 			goto retry;
158 		}
159 	}
160 
161 	return ret;
162 }
163 
164 static int submit_fence_sync(const struct etnaviv_gem_submit *submit)
165 {
166 	unsigned int context = submit->gpu->fence_context;
167 	int i, ret = 0;
168 
169 	for (i = 0; i < submit->nr_bos; i++) {
170 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
171 		bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE;
172 
173 		ret = etnaviv_gpu_fence_sync_obj(etnaviv_obj, context, write);
174 		if (ret)
175 			break;
176 	}
177 
178 	return ret;
179 }
180 
181 static void submit_unpin_objects(struct etnaviv_gem_submit *submit)
182 {
183 	int i;
184 
185 	for (i = 0; i < submit->nr_bos; i++) {
186 		if (submit->bos[i].flags & BO_PINNED)
187 			etnaviv_gem_mapping_unreference(submit->bos[i].mapping);
188 
189 		submit->bos[i].mapping = NULL;
190 		submit->bos[i].flags &= ~BO_PINNED;
191 	}
192 }
193 
194 static int submit_pin_objects(struct etnaviv_gem_submit *submit)
195 {
196 	int i, ret = 0;
197 
198 	for (i = 0; i < submit->nr_bos; i++) {
199 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
200 		struct etnaviv_vram_mapping *mapping;
201 
202 		mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base,
203 						  submit->gpu);
204 		if (IS_ERR(mapping)) {
205 			ret = PTR_ERR(mapping);
206 			break;
207 		}
208 
209 		submit->bos[i].flags |= BO_PINNED;
210 		submit->bos[i].mapping = mapping;
211 	}
212 
213 	return ret;
214 }
215 
216 static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx,
217 	struct etnaviv_gem_submit_bo **bo)
218 {
219 	if (idx >= submit->nr_bos) {
220 		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
221 				idx, submit->nr_bos);
222 		return -EINVAL;
223 	}
224 
225 	*bo = &submit->bos[idx];
226 
227 	return 0;
228 }
229 
230 /* process the reloc's and patch up the cmdstream as needed: */
231 static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream,
232 		u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs,
233 		u32 nr_relocs)
234 {
235 	u32 i, last_offset = 0;
236 	u32 *ptr = stream;
237 	int ret;
238 
239 	for (i = 0; i < nr_relocs; i++) {
240 		const struct drm_etnaviv_gem_submit_reloc *r = relocs + i;
241 		struct etnaviv_gem_submit_bo *bo;
242 		u32 off;
243 
244 		if (unlikely(r->flags)) {
245 			DRM_ERROR("invalid reloc flags\n");
246 			return -EINVAL;
247 		}
248 
249 		if (r->submit_offset % 4) {
250 			DRM_ERROR("non-aligned reloc offset: %u\n",
251 				  r->submit_offset);
252 			return -EINVAL;
253 		}
254 
255 		/* offset in dwords: */
256 		off = r->submit_offset / 4;
257 
258 		if ((off >= size ) ||
259 				(off < last_offset)) {
260 			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
261 			return -EINVAL;
262 		}
263 
264 		ret = submit_bo(submit, r->reloc_idx, &bo);
265 		if (ret)
266 			return ret;
267 
268 		if (r->reloc_offset >= bo->obj->base.size - sizeof(*ptr)) {
269 			DRM_ERROR("relocation %u outside object", i);
270 			return -EINVAL;
271 		}
272 
273 		ptr[off] = bo->mapping->iova + r->reloc_offset;
274 
275 		last_offset = off;
276 	}
277 
278 	return 0;
279 }
280 
281 static void submit_cleanup(struct etnaviv_gem_submit *submit)
282 {
283 	unsigned i;
284 
285 	for (i = 0; i < submit->nr_bos; i++) {
286 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
287 
288 		submit_unlock_object(submit, i);
289 		drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
290 	}
291 
292 	ww_acquire_fini(&submit->ticket);
293 	kfree(submit);
294 }
295 
296 int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
297 		struct drm_file *file)
298 {
299 	struct etnaviv_drm_private *priv = dev->dev_private;
300 	struct drm_etnaviv_gem_submit *args = data;
301 	struct drm_etnaviv_gem_submit_reloc *relocs;
302 	struct drm_etnaviv_gem_submit_bo *bos;
303 	struct etnaviv_gem_submit *submit;
304 	struct etnaviv_cmdbuf *cmdbuf;
305 	struct etnaviv_gpu *gpu;
306 	void *stream;
307 	int ret;
308 
309 	if (args->pipe >= ETNA_MAX_PIPES)
310 		return -EINVAL;
311 
312 	gpu = priv->gpu[args->pipe];
313 	if (!gpu)
314 		return -ENXIO;
315 
316 	if (args->stream_size % 4) {
317 		DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
318 			  args->stream_size);
319 		return -EINVAL;
320 	}
321 
322 	if (args->exec_state != ETNA_PIPE_3D &&
323 	    args->exec_state != ETNA_PIPE_2D &&
324 	    args->exec_state != ETNA_PIPE_VG) {
325 		DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state);
326 		return -EINVAL;
327 	}
328 
329 	/*
330 	 * Copy the command submission and bo array to kernel space in
331 	 * one go, and do this outside of any locks.
332 	 */
333 	bos = drm_malloc_ab(args->nr_bos, sizeof(*bos));
334 	relocs = drm_malloc_ab(args->nr_relocs, sizeof(*relocs));
335 	stream = drm_malloc_ab(1, args->stream_size);
336 	cmdbuf = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc,
337 				    ALIGN(args->stream_size, 8) + 8,
338 				    args->nr_bos);
339 	if (!bos || !relocs || !stream || !cmdbuf) {
340 		ret = -ENOMEM;
341 		goto err_submit_cmds;
342 	}
343 
344 	cmdbuf->exec_state = args->exec_state;
345 	cmdbuf->ctx = file->driver_priv;
346 
347 	ret = copy_from_user(bos, u64_to_user_ptr(args->bos),
348 			     args->nr_bos * sizeof(*bos));
349 	if (ret) {
350 		ret = -EFAULT;
351 		goto err_submit_cmds;
352 	}
353 
354 	ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs),
355 			     args->nr_relocs * sizeof(*relocs));
356 	if (ret) {
357 		ret = -EFAULT;
358 		goto err_submit_cmds;
359 	}
360 
361 	ret = copy_from_user(stream, u64_to_user_ptr(args->stream),
362 			     args->stream_size);
363 	if (ret) {
364 		ret = -EFAULT;
365 		goto err_submit_cmds;
366 	}
367 
368 	submit = submit_create(dev, gpu, args->nr_bos);
369 	if (!submit) {
370 		ret = -ENOMEM;
371 		goto err_submit_cmds;
372 	}
373 
374 	ret = submit_lookup_objects(submit, file, bos, args->nr_bos);
375 	if (ret)
376 		goto err_submit_objects;
377 
378 	ret = submit_lock_objects(submit);
379 	if (ret)
380 		goto err_submit_objects;
381 
382 	if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4,
383 				      relocs, args->nr_relocs)) {
384 		ret = -EINVAL;
385 		goto err_submit_objects;
386 	}
387 
388 	ret = submit_fence_sync(submit);
389 	if (ret)
390 		goto err_submit_objects;
391 
392 	ret = submit_pin_objects(submit);
393 	if (ret)
394 		goto out;
395 
396 	ret = submit_reloc(submit, stream, args->stream_size / 4,
397 			   relocs, args->nr_relocs);
398 	if (ret)
399 		goto out;
400 
401 	memcpy(cmdbuf->vaddr, stream, args->stream_size);
402 	cmdbuf->user_size = ALIGN(args->stream_size, 8);
403 
404 	ret = etnaviv_gpu_submit(gpu, submit, cmdbuf);
405 	if (ret == 0)
406 		cmdbuf = NULL;
407 
408 	args->fence = submit->fence;
409 
410 out:
411 	submit_unpin_objects(submit);
412 
413 	/*
414 	 * If we're returning -EAGAIN, it may be due to the userptr code
415 	 * wanting to run its workqueue outside of any locks. Flush our
416 	 * workqueue to ensure that it is run in a timely manner.
417 	 */
418 	if (ret == -EAGAIN)
419 		flush_workqueue(priv->wq);
420 
421 err_submit_objects:
422 	submit_cleanup(submit);
423 
424 err_submit_cmds:
425 	/* if we still own the cmdbuf */
426 	if (cmdbuf)
427 		etnaviv_cmdbuf_free(cmdbuf);
428 	if (stream)
429 		drm_free_large(stream);
430 	if (bos)
431 		drm_free_large(bos);
432 	if (relocs)
433 		drm_free_large(relocs);
434 
435 	return ret;
436 }
437