1 /*
2  * Copyright (C) 2008 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include "nouveau_drv.h"
28 #include "nouveau_dma.h"
29 #include "nouveau_fence.h"
30 #include "nouveau_abi16.h"
31 
32 #include "nouveau_ttm.h"
33 #include "nouveau_gem.h"
34 #include "nouveau_mem.h"
35 #include "nouveau_vmm.h"
36 
37 #include <nvif/class.h>
38 
39 void
40 nouveau_gem_object_del(struct drm_gem_object *gem)
41 {
42 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
43 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
44 	struct device *dev = drm->dev->dev;
45 	int ret;
46 
47 	ret = pm_runtime_get_sync(dev);
48 	if (WARN_ON(ret < 0 && ret != -EACCES))
49 		return;
50 
51 	if (gem->import_attach)
52 		drm_prime_gem_destroy(gem, nvbo->bo.sg);
53 
54 	ttm_bo_put(&nvbo->bo);
55 
56 	pm_runtime_mark_last_busy(dev);
57 	pm_runtime_put_autosuspend(dev);
58 }
59 
60 int
61 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
62 {
63 	struct nouveau_cli *cli = nouveau_cli(file_priv);
64 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
65 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
66 	struct device *dev = drm->dev->dev;
67 	struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
68 	struct nouveau_vma *vma;
69 	int ret;
70 
71 	if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
72 		return 0;
73 
74 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
75 	if (ret)
76 		return ret;
77 
78 	ret = pm_runtime_get_sync(dev);
79 	if (ret < 0 && ret != -EACCES)
80 		goto out;
81 
82 	ret = nouveau_vma_new(nvbo, vmm, &vma);
83 	pm_runtime_mark_last_busy(dev);
84 	pm_runtime_put_autosuspend(dev);
85 out:
86 	ttm_bo_unreserve(&nvbo->bo);
87 	return ret;
88 }
89 
90 struct nouveau_gem_object_unmap {
91 	struct nouveau_cli_work work;
92 	struct nouveau_vma *vma;
93 };
94 
95 static void
96 nouveau_gem_object_delete(struct nouveau_vma *vma)
97 {
98 	nouveau_fence_unref(&vma->fence);
99 	nouveau_vma_del(&vma);
100 }
101 
102 static void
103 nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
104 {
105 	struct nouveau_gem_object_unmap *work =
106 		container_of(w, typeof(*work), work);
107 	nouveau_gem_object_delete(work->vma);
108 	kfree(work);
109 }
110 
111 static void
112 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
113 {
114 	struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
115 	struct nouveau_gem_object_unmap *work;
116 
117 	list_del_init(&vma->head);
118 
119 	if (!fence) {
120 		nouveau_gem_object_delete(vma);
121 		return;
122 	}
123 
124 	if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) {
125 		WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
126 		nouveau_gem_object_delete(vma);
127 		return;
128 	}
129 
130 	work->work.func = nouveau_gem_object_delete_work;
131 	work->vma = vma;
132 	nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
133 }
134 
135 void
136 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
137 {
138 	struct nouveau_cli *cli = nouveau_cli(file_priv);
139 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
140 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
141 	struct device *dev = drm->dev->dev;
142 	struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm;
143 	struct nouveau_vma *vma;
144 	int ret;
145 
146 	if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
147 		return;
148 
149 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
150 	if (ret)
151 		return;
152 
153 	vma = nouveau_vma_find(nvbo, vmm);
154 	if (vma) {
155 		if (--vma->refs == 0) {
156 			ret = pm_runtime_get_sync(dev);
157 			if (!WARN_ON(ret < 0 && ret != -EACCES)) {
158 				nouveau_gem_object_unmap(nvbo, vma);
159 				pm_runtime_mark_last_busy(dev);
160 				pm_runtime_put_autosuspend(dev);
161 			}
162 		}
163 	}
164 	ttm_bo_unreserve(&nvbo->bo);
165 }
166 
167 int
168 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
169 		uint32_t tile_mode, uint32_t tile_flags,
170 		struct nouveau_bo **pnvbo)
171 {
172 	struct nouveau_drm *drm = cli->drm;
173 	struct nouveau_bo *nvbo;
174 	u32 flags = 0;
175 	int ret;
176 
177 	if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
178 		flags |= TTM_PL_FLAG_VRAM;
179 	if (domain & NOUVEAU_GEM_DOMAIN_GART)
180 		flags |= TTM_PL_FLAG_TT;
181 	if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
182 		flags |= TTM_PL_FLAG_SYSTEM;
183 
184 	if (domain & NOUVEAU_GEM_DOMAIN_COHERENT)
185 		flags |= TTM_PL_FLAG_UNCACHED;
186 
187 	nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode,
188 				tile_flags);
189 	if (IS_ERR(nvbo))
190 		return PTR_ERR(nvbo);
191 
192 	/* Initialize the embedded gem-object. We return a single gem-reference
193 	 * to the caller, instead of a normal nouveau_bo ttm reference. */
194 	ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size);
195 	if (ret) {
196 		nouveau_bo_ref(NULL, &nvbo);
197 		return ret;
198 	}
199 
200 	ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL);
201 	if (ret) {
202 		nouveau_bo_ref(NULL, &nvbo);
203 		return ret;
204 	}
205 
206 	/* we restrict allowed domains on nv50+ to only the types
207 	 * that were requested at creation time.  not possibly on
208 	 * earlier chips without busting the ABI.
209 	 */
210 	nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
211 			      NOUVEAU_GEM_DOMAIN_GART;
212 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
213 		nvbo->valid_domains &= domain;
214 
215 	nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp;
216 	*pnvbo = nvbo;
217 	return 0;
218 }
219 
220 static int
221 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
222 		 struct drm_nouveau_gem_info *rep)
223 {
224 	struct nouveau_cli *cli = nouveau_cli(file_priv);
225 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
226 	struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
227 	struct nouveau_vma *vma;
228 
229 	if (is_power_of_2(nvbo->valid_domains))
230 		rep->domain = nvbo->valid_domains;
231 	else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
232 		rep->domain = NOUVEAU_GEM_DOMAIN_GART;
233 	else
234 		rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
235 	rep->offset = nvbo->bo.offset;
236 	if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
237 		vma = nouveau_vma_find(nvbo, vmm);
238 		if (!vma)
239 			return -EINVAL;
240 
241 		rep->offset = vma->addr;
242 	}
243 
244 	rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
245 	rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node);
246 	rep->tile_mode = nvbo->mode;
247 	rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
248 	if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
249 		rep->tile_flags |= nvbo->kind << 8;
250 	else
251 	if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
252 		rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
253 	else
254 		rep->tile_flags |= nvbo->zeta;
255 	return 0;
256 }
257 
258 int
259 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
260 		      struct drm_file *file_priv)
261 {
262 	struct nouveau_cli *cli = nouveau_cli(file_priv);
263 	struct drm_nouveau_gem_new *req = data;
264 	struct nouveau_bo *nvbo = NULL;
265 	int ret = 0;
266 
267 	ret = nouveau_gem_new(cli, req->info.size, req->align,
268 			      req->info.domain, req->info.tile_mode,
269 			      req->info.tile_flags, &nvbo);
270 	if (ret)
271 		return ret;
272 
273 	ret = drm_gem_handle_create(file_priv, &nvbo->bo.base,
274 				    &req->info.handle);
275 	if (ret == 0) {
276 		ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info);
277 		if (ret)
278 			drm_gem_handle_delete(file_priv, req->info.handle);
279 	}
280 
281 	/* drop reference from allocate - handle holds it now */
282 	drm_gem_object_put_unlocked(&nvbo->bo.base);
283 	return ret;
284 }
285 
286 static int
287 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
288 		       uint32_t write_domains, uint32_t valid_domains)
289 {
290 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
291 	struct ttm_buffer_object *bo = &nvbo->bo;
292 	uint32_t domains = valid_domains & nvbo->valid_domains &
293 		(write_domains ? write_domains : read_domains);
294 	uint32_t pref_flags = 0, valid_flags = 0;
295 
296 	if (!domains)
297 		return -EINVAL;
298 
299 	if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
300 		valid_flags |= TTM_PL_FLAG_VRAM;
301 
302 	if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
303 		valid_flags |= TTM_PL_FLAG_TT;
304 
305 	if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
306 	    bo->mem.mem_type == TTM_PL_VRAM)
307 		pref_flags |= TTM_PL_FLAG_VRAM;
308 
309 	else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
310 		 bo->mem.mem_type == TTM_PL_TT)
311 		pref_flags |= TTM_PL_FLAG_TT;
312 
313 	else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
314 		pref_flags |= TTM_PL_FLAG_VRAM;
315 
316 	else
317 		pref_flags |= TTM_PL_FLAG_TT;
318 
319 	nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
320 
321 	return 0;
322 }
323 
324 struct validate_op {
325 	struct list_head list;
326 	struct ww_acquire_ctx ticket;
327 };
328 
329 static void
330 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan,
331 			struct nouveau_fence *fence,
332 			struct drm_nouveau_gem_pushbuf_bo *pbbo)
333 {
334 	struct nouveau_bo *nvbo;
335 	struct drm_nouveau_gem_pushbuf_bo *b;
336 
337 	while (!list_empty(&op->list)) {
338 		nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
339 		b = &pbbo[nvbo->pbbo_index];
340 
341 		if (likely(fence)) {
342 			nouveau_bo_fence(nvbo, fence, !!b->write_domains);
343 
344 			if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
345 				struct nouveau_vma *vma =
346 					(void *)(unsigned long)b->user_priv;
347 				nouveau_fence_unref(&vma->fence);
348 				dma_fence_get(&fence->base);
349 				vma->fence = fence;
350 			}
351 		}
352 
353 		if (unlikely(nvbo->validate_mapped)) {
354 			ttm_bo_kunmap(&nvbo->kmap);
355 			nvbo->validate_mapped = false;
356 		}
357 
358 		list_del(&nvbo->entry);
359 		nvbo->reserved_by = NULL;
360 		ttm_bo_unreserve(&nvbo->bo);
361 		drm_gem_object_put_unlocked(&nvbo->bo.base);
362 	}
363 }
364 
365 static void
366 validate_fini(struct validate_op *op, struct nouveau_channel *chan,
367 	      struct nouveau_fence *fence,
368 	      struct drm_nouveau_gem_pushbuf_bo *pbbo)
369 {
370 	validate_fini_no_ticket(op, chan, fence, pbbo);
371 	ww_acquire_fini(&op->ticket);
372 }
373 
374 static int
375 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
376 	      struct drm_nouveau_gem_pushbuf_bo *pbbo,
377 	      int nr_buffers, struct validate_op *op)
378 {
379 	struct nouveau_cli *cli = nouveau_cli(file_priv);
380 	int trycnt = 0;
381 	int ret = -EINVAL, i;
382 	struct nouveau_bo *res_bo = NULL;
383 	LIST_HEAD(gart_list);
384 	LIST_HEAD(vram_list);
385 	LIST_HEAD(both_list);
386 
387 	ww_acquire_init(&op->ticket, &reservation_ww_class);
388 retry:
389 	if (++trycnt > 100000) {
390 		NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
391 		return -EINVAL;
392 	}
393 
394 	for (i = 0; i < nr_buffers; i++) {
395 		struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
396 		struct drm_gem_object *gem;
397 		struct nouveau_bo *nvbo;
398 
399 		gem = drm_gem_object_lookup(file_priv, b->handle);
400 		if (!gem) {
401 			NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
402 			ret = -ENOENT;
403 			break;
404 		}
405 		nvbo = nouveau_gem_object(gem);
406 		if (nvbo == res_bo) {
407 			res_bo = NULL;
408 			drm_gem_object_put_unlocked(gem);
409 			continue;
410 		}
411 
412 		if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
413 			NV_PRINTK(err, cli, "multiple instances of buffer %d on "
414 				      "validation list\n", b->handle);
415 			drm_gem_object_put_unlocked(gem);
416 			ret = -EINVAL;
417 			break;
418 		}
419 
420 		ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
421 		if (ret) {
422 			list_splice_tail_init(&vram_list, &op->list);
423 			list_splice_tail_init(&gart_list, &op->list);
424 			list_splice_tail_init(&both_list, &op->list);
425 			validate_fini_no_ticket(op, chan, NULL, NULL);
426 			if (unlikely(ret == -EDEADLK)) {
427 				ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
428 							      &op->ticket);
429 				if (!ret)
430 					res_bo = nvbo;
431 			}
432 			if (unlikely(ret)) {
433 				if (ret != -ERESTARTSYS)
434 					NV_PRINTK(err, cli, "fail reserve\n");
435 				break;
436 			}
437 		}
438 
439 		if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
440 			struct nouveau_vmm *vmm = chan->vmm;
441 			struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
442 			if (!vma) {
443 				NV_PRINTK(err, cli, "vma not found!\n");
444 				ret = -EINVAL;
445 				break;
446 			}
447 
448 			b->user_priv = (uint64_t)(unsigned long)vma;
449 		} else {
450 			b->user_priv = (uint64_t)(unsigned long)nvbo;
451 		}
452 
453 		nvbo->reserved_by = file_priv;
454 		nvbo->pbbo_index = i;
455 		if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
456 		    (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
457 			list_add_tail(&nvbo->entry, &both_list);
458 		else
459 		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
460 			list_add_tail(&nvbo->entry, &vram_list);
461 		else
462 		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
463 			list_add_tail(&nvbo->entry, &gart_list);
464 		else {
465 			NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
466 				 b->valid_domains);
467 			list_add_tail(&nvbo->entry, &both_list);
468 			ret = -EINVAL;
469 			break;
470 		}
471 		if (nvbo == res_bo)
472 			goto retry;
473 	}
474 
475 	ww_acquire_done(&op->ticket);
476 	list_splice_tail(&vram_list, &op->list);
477 	list_splice_tail(&gart_list, &op->list);
478 	list_splice_tail(&both_list, &op->list);
479 	if (ret)
480 		validate_fini(op, chan, NULL, NULL);
481 	return ret;
482 
483 }
484 
485 static int
486 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
487 	      struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo)
488 {
489 	struct nouveau_drm *drm = chan->drm;
490 	struct nouveau_bo *nvbo;
491 	int ret, relocs = 0;
492 
493 	list_for_each_entry(nvbo, list, entry) {
494 		struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
495 
496 		ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains,
497 					     b->write_domains,
498 					     b->valid_domains);
499 		if (unlikely(ret)) {
500 			NV_PRINTK(err, cli, "fail set_domain\n");
501 			return ret;
502 		}
503 
504 		ret = nouveau_bo_validate(nvbo, true, false);
505 		if (unlikely(ret)) {
506 			if (ret != -ERESTARTSYS)
507 				NV_PRINTK(err, cli, "fail ttm_validate\n");
508 			return ret;
509 		}
510 
511 		ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
512 		if (unlikely(ret)) {
513 			if (ret != -ERESTARTSYS)
514 				NV_PRINTK(err, cli, "fail post-validate sync\n");
515 			return ret;
516 		}
517 
518 		if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
519 			if (nvbo->bo.offset == b->presumed.offset &&
520 			    ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
521 			      b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
522 			     (nvbo->bo.mem.mem_type == TTM_PL_TT &&
523 			      b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
524 				continue;
525 
526 			if (nvbo->bo.mem.mem_type == TTM_PL_TT)
527 				b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
528 			else
529 				b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
530 			b->presumed.offset = nvbo->bo.offset;
531 			b->presumed.valid = 0;
532 			relocs++;
533 		}
534 	}
535 
536 	return relocs;
537 }
538 
539 static int
540 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
541 			     struct drm_file *file_priv,
542 			     struct drm_nouveau_gem_pushbuf_bo *pbbo,
543 			     int nr_buffers,
544 			     struct validate_op *op, bool *apply_relocs)
545 {
546 	struct nouveau_cli *cli = nouveau_cli(file_priv);
547 	int ret;
548 
549 	INIT_LIST_HEAD(&op->list);
550 
551 	if (nr_buffers == 0)
552 		return 0;
553 
554 	ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
555 	if (unlikely(ret)) {
556 		if (ret != -ERESTARTSYS)
557 			NV_PRINTK(err, cli, "validate_init\n");
558 		return ret;
559 	}
560 
561 	ret = validate_list(chan, cli, &op->list, pbbo);
562 	if (unlikely(ret < 0)) {
563 		if (ret != -ERESTARTSYS)
564 			NV_PRINTK(err, cli, "validating bo list\n");
565 		validate_fini(op, chan, NULL, NULL);
566 		return ret;
567 	}
568 	*apply_relocs = ret;
569 	return 0;
570 }
571 
572 static inline void
573 u_free(void *addr)
574 {
575 	kvfree(addr);
576 }
577 
578 static inline void *
579 u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
580 {
581 	void *mem;
582 	void __user *userptr = (void __force __user *)(uintptr_t)user;
583 
584 	size *= nmemb;
585 
586 	mem = kvmalloc(size, GFP_KERNEL);
587 	if (!mem)
588 		return ERR_PTR(-ENOMEM);
589 
590 	if (copy_from_user(mem, userptr, size)) {
591 		u_free(mem);
592 		return ERR_PTR(-EFAULT);
593 	}
594 
595 	return mem;
596 }
597 
598 static int
599 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
600 				struct drm_nouveau_gem_pushbuf *req,
601 				struct drm_nouveau_gem_pushbuf_reloc *reloc,
602 				struct drm_nouveau_gem_pushbuf_bo *bo)
603 {
604 	int ret = 0;
605 	unsigned i;
606 
607 	for (i = 0; i < req->nr_relocs; i++) {
608 		struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
609 		struct drm_nouveau_gem_pushbuf_bo *b;
610 		struct nouveau_bo *nvbo;
611 		uint32_t data;
612 
613 		if (unlikely(r->bo_index >= req->nr_buffers)) {
614 			NV_PRINTK(err, cli, "reloc bo index invalid\n");
615 			ret = -EINVAL;
616 			break;
617 		}
618 
619 		b = &bo[r->bo_index];
620 		if (b->presumed.valid)
621 			continue;
622 
623 		if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
624 			NV_PRINTK(err, cli, "reloc container bo index invalid\n");
625 			ret = -EINVAL;
626 			break;
627 		}
628 		nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
629 
630 		if (unlikely(r->reloc_bo_offset + 4 >
631 			     nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
632 			NV_PRINTK(err, cli, "reloc outside of bo\n");
633 			ret = -EINVAL;
634 			break;
635 		}
636 
637 		if (!nvbo->kmap.virtual) {
638 			ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
639 					  &nvbo->kmap);
640 			if (ret) {
641 				NV_PRINTK(err, cli, "failed kmap for reloc\n");
642 				break;
643 			}
644 			nvbo->validate_mapped = true;
645 		}
646 
647 		if (r->flags & NOUVEAU_GEM_RELOC_LOW)
648 			data = b->presumed.offset + r->data;
649 		else
650 		if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
651 			data = (b->presumed.offset + r->data) >> 32;
652 		else
653 			data = r->data;
654 
655 		if (r->flags & NOUVEAU_GEM_RELOC_OR) {
656 			if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
657 				data |= r->tor;
658 			else
659 				data |= r->vor;
660 		}
661 
662 		ret = ttm_bo_wait(&nvbo->bo, false, false);
663 		if (ret) {
664 			NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
665 			break;
666 		}
667 
668 		nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
669 	}
670 
671 	u_free(reloc);
672 	return ret;
673 }
674 
675 int
676 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
677 			  struct drm_file *file_priv)
678 {
679 	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
680 	struct nouveau_cli *cli = nouveau_cli(file_priv);
681 	struct nouveau_abi16_chan *temp;
682 	struct nouveau_drm *drm = nouveau_drm(dev);
683 	struct drm_nouveau_gem_pushbuf *req = data;
684 	struct drm_nouveau_gem_pushbuf_push *push;
685 	struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
686 	struct drm_nouveau_gem_pushbuf_bo *bo;
687 	struct nouveau_channel *chan = NULL;
688 	struct validate_op op;
689 	struct nouveau_fence *fence = NULL;
690 	int i, j, ret = 0;
691 	bool do_reloc = false, sync = false;
692 
693 	if (unlikely(!abi16))
694 		return -ENOMEM;
695 
696 	list_for_each_entry(temp, &abi16->channels, head) {
697 		if (temp->chan->chid == req->channel) {
698 			chan = temp->chan;
699 			break;
700 		}
701 	}
702 
703 	if (!chan)
704 		return nouveau_abi16_put(abi16, -ENOENT);
705 	if (unlikely(atomic_read(&chan->killed)))
706 		return nouveau_abi16_put(abi16, -ENODEV);
707 
708 	sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC;
709 
710 	req->vram_available = drm->gem.vram_available;
711 	req->gart_available = drm->gem.gart_available;
712 	if (unlikely(req->nr_push == 0))
713 		goto out_next;
714 
715 	if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
716 		NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
717 			 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
718 		return nouveau_abi16_put(abi16, -EINVAL);
719 	}
720 
721 	if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
722 		NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
723 			 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
724 		return nouveau_abi16_put(abi16, -EINVAL);
725 	}
726 
727 	if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
728 		NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
729 			 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
730 		return nouveau_abi16_put(abi16, -EINVAL);
731 	}
732 
733 	push = u_memcpya(req->push, req->nr_push, sizeof(*push));
734 	if (IS_ERR(push))
735 		return nouveau_abi16_put(abi16, PTR_ERR(push));
736 
737 	bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
738 	if (IS_ERR(bo)) {
739 		u_free(push);
740 		return nouveau_abi16_put(abi16, PTR_ERR(bo));
741 	}
742 
743 	/* Ensure all push buffers are on validate list */
744 	for (i = 0; i < req->nr_push; i++) {
745 		if (push[i].bo_index >= req->nr_buffers) {
746 			NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
747 			ret = -EINVAL;
748 			goto out_prevalid;
749 		}
750 	}
751 
752 	/* Validate buffer list */
753 revalidate:
754 	ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo,
755 					   req->nr_buffers, &op, &do_reloc);
756 	if (ret) {
757 		if (ret != -ERESTARTSYS)
758 			NV_PRINTK(err, cli, "validate: %d\n", ret);
759 		goto out_prevalid;
760 	}
761 
762 	/* Apply any relocations that are required */
763 	if (do_reloc) {
764 		if (!reloc) {
765 			validate_fini(&op, chan, NULL, bo);
766 			reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
767 			if (IS_ERR(reloc)) {
768 				ret = PTR_ERR(reloc);
769 				goto out_prevalid;
770 			}
771 
772 			goto revalidate;
773 		}
774 
775 		ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo);
776 		if (ret) {
777 			NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
778 			goto out;
779 		}
780 	}
781 
782 	if (chan->dma.ib_max) {
783 		ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
784 		if (ret) {
785 			NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
786 			goto out;
787 		}
788 
789 		for (i = 0; i < req->nr_push; i++) {
790 			struct nouveau_vma *vma = (void *)(unsigned long)
791 				bo[push[i].bo_index].user_priv;
792 
793 			nv50_dma_push(chan, vma->addr + push[i].offset,
794 				      push[i].length);
795 		}
796 	} else
797 	if (drm->client.device.info.chipset >= 0x25) {
798 		ret = RING_SPACE(chan, req->nr_push * 2);
799 		if (ret) {
800 			NV_PRINTK(err, cli, "cal_space: %d\n", ret);
801 			goto out;
802 		}
803 
804 		for (i = 0; i < req->nr_push; i++) {
805 			struct nouveau_bo *nvbo = (void *)(unsigned long)
806 				bo[push[i].bo_index].user_priv;
807 
808 			OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
809 			OUT_RING(chan, 0);
810 		}
811 	} else {
812 		ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
813 		if (ret) {
814 			NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
815 			goto out;
816 		}
817 
818 		for (i = 0; i < req->nr_push; i++) {
819 			struct nouveau_bo *nvbo = (void *)(unsigned long)
820 				bo[push[i].bo_index].user_priv;
821 			uint32_t cmd;
822 
823 			cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
824 			cmd |= 0x20000000;
825 			if (unlikely(cmd != req->suffix0)) {
826 				if (!nvbo->kmap.virtual) {
827 					ret = ttm_bo_kmap(&nvbo->bo, 0,
828 							  nvbo->bo.mem.
829 							  num_pages,
830 							  &nvbo->kmap);
831 					if (ret) {
832 						WIND_RING(chan);
833 						goto out;
834 					}
835 					nvbo->validate_mapped = true;
836 				}
837 
838 				nouveau_bo_wr32(nvbo, (push[i].offset +
839 						push[i].length - 8) / 4, cmd);
840 			}
841 
842 			OUT_RING(chan, 0x20000000 |
843 				      (nvbo->bo.offset + push[i].offset));
844 			OUT_RING(chan, 0);
845 			for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
846 				OUT_RING(chan, 0);
847 		}
848 	}
849 
850 	ret = nouveau_fence_new(chan, false, &fence);
851 	if (ret) {
852 		NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
853 		WIND_RING(chan);
854 		goto out;
855 	}
856 
857 	if (sync) {
858 		if (!(ret = nouveau_fence_wait(fence, false, false))) {
859 			if ((ret = dma_fence_get_status(&fence->base)) == 1)
860 				ret = 0;
861 		}
862 	}
863 
864 out:
865 	validate_fini(&op, chan, fence, bo);
866 	nouveau_fence_unref(&fence);
867 
868 	if (do_reloc) {
869 		struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
870 			u64_to_user_ptr(req->buffers);
871 
872 		for (i = 0; i < req->nr_buffers; i++) {
873 			if (bo[i].presumed.valid)
874 				continue;
875 
876 			if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed,
877 					 sizeof(bo[i].presumed))) {
878 				ret = -EFAULT;
879 				break;
880 			}
881 		}
882 		u_free(reloc);
883 	}
884 out_prevalid:
885 	u_free(bo);
886 	u_free(push);
887 
888 out_next:
889 	if (chan->dma.ib_max) {
890 		req->suffix0 = 0x00000000;
891 		req->suffix1 = 0x00000000;
892 	} else
893 	if (drm->client.device.info.chipset >= 0x25) {
894 		req->suffix0 = 0x00020000;
895 		req->suffix1 = 0x00000000;
896 	} else {
897 		req->suffix0 = 0x20000000 |
898 			      (chan->push.addr + ((chan->dma.cur + 2) << 2));
899 		req->suffix1 = 0x00000000;
900 	}
901 
902 	return nouveau_abi16_put(abi16, ret);
903 }
904 
905 int
906 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
907 			   struct drm_file *file_priv)
908 {
909 	struct drm_nouveau_gem_cpu_prep *req = data;
910 	struct drm_gem_object *gem;
911 	struct nouveau_bo *nvbo;
912 	bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
913 	bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
914 	long lret;
915 	int ret;
916 
917 	gem = drm_gem_object_lookup(file_priv, req->handle);
918 	if (!gem)
919 		return -ENOENT;
920 	nvbo = nouveau_gem_object(gem);
921 
922 	lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true,
923 						   no_wait ? 0 : 30 * HZ);
924 	if (!lret)
925 		ret = -EBUSY;
926 	else if (lret > 0)
927 		ret = 0;
928 	else
929 		ret = lret;
930 
931 	nouveau_bo_sync_for_cpu(nvbo);
932 	drm_gem_object_put_unlocked(gem);
933 
934 	return ret;
935 }
936 
937 int
938 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
939 			   struct drm_file *file_priv)
940 {
941 	struct drm_nouveau_gem_cpu_fini *req = data;
942 	struct drm_gem_object *gem;
943 	struct nouveau_bo *nvbo;
944 
945 	gem = drm_gem_object_lookup(file_priv, req->handle);
946 	if (!gem)
947 		return -ENOENT;
948 	nvbo = nouveau_gem_object(gem);
949 
950 	nouveau_bo_sync_for_device(nvbo);
951 	drm_gem_object_put_unlocked(gem);
952 	return 0;
953 }
954 
955 int
956 nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
957 		       struct drm_file *file_priv)
958 {
959 	struct drm_nouveau_gem_info *req = data;
960 	struct drm_gem_object *gem;
961 	int ret;
962 
963 	gem = drm_gem_object_lookup(file_priv, req->handle);
964 	if (!gem)
965 		return -ENOENT;
966 
967 	ret = nouveau_gem_info(file_priv, gem, req);
968 	drm_gem_object_put_unlocked(gem);
969 	return ret;
970 }
971 
972