1 /*
2  * Copyright (C) 2008 Ben Skeggs.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining
6  * a copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial
15  * portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  *
25  */
26 
27 #include "nouveau_drv.h"
28 #include "nouveau_dma.h"
29 #include "nouveau_fence.h"
30 #include "nouveau_abi16.h"
31 
32 #include "nouveau_ttm.h"
33 #include "nouveau_gem.h"
34 #include "nouveau_mem.h"
35 #include "nouveau_vmm.h"
36 
37 #include <nvif/class.h>
38 
39 void
40 nouveau_gem_object_del(struct drm_gem_object *gem)
41 {
42 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
43 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
44 	struct device *dev = drm->dev->dev;
45 	int ret;
46 
47 	ret = pm_runtime_get_sync(dev);
48 	if (WARN_ON(ret < 0 && ret != -EACCES))
49 		return;
50 
51 	if (gem->import_attach)
52 		drm_prime_gem_destroy(gem, nvbo->bo.sg);
53 
54 	ttm_bo_put(&nvbo->bo);
55 
56 	pm_runtime_mark_last_busy(dev);
57 	pm_runtime_put_autosuspend(dev);
58 }
59 
60 int
61 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
62 {
63 	struct nouveau_cli *cli = nouveau_cli(file_priv);
64 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
65 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
66 	struct device *dev = drm->dev->dev;
67 	struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
68 	struct nouveau_vma *vma;
69 	int ret;
70 
71 	if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
72 		return 0;
73 
74 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
75 	if (ret)
76 		return ret;
77 
78 	ret = pm_runtime_get_sync(dev);
79 	if (ret < 0 && ret != -EACCES) {
80 		pm_runtime_put_autosuspend(dev);
81 		goto out;
82 	}
83 
84 	ret = nouveau_vma_new(nvbo, vmm, &vma);
85 	pm_runtime_mark_last_busy(dev);
86 	pm_runtime_put_autosuspend(dev);
87 out:
88 	ttm_bo_unreserve(&nvbo->bo);
89 	return ret;
90 }
91 
92 struct nouveau_gem_object_unmap {
93 	struct nouveau_cli_work work;
94 	struct nouveau_vma *vma;
95 };
96 
97 static void
98 nouveau_gem_object_delete(struct nouveau_vma *vma)
99 {
100 	nouveau_fence_unref(&vma->fence);
101 	nouveau_vma_del(&vma);
102 }
103 
104 static void
105 nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
106 {
107 	struct nouveau_gem_object_unmap *work =
108 		container_of(w, typeof(*work), work);
109 	nouveau_gem_object_delete(work->vma);
110 	kfree(work);
111 }
112 
113 static void
114 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
115 {
116 	struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
117 	struct nouveau_gem_object_unmap *work;
118 
119 	list_del_init(&vma->head);
120 
121 	if (!fence) {
122 		nouveau_gem_object_delete(vma);
123 		return;
124 	}
125 
126 	if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) {
127 		WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
128 		nouveau_gem_object_delete(vma);
129 		return;
130 	}
131 
132 	work->work.func = nouveau_gem_object_delete_work;
133 	work->vma = vma;
134 	nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
135 }
136 
137 void
138 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
139 {
140 	struct nouveau_cli *cli = nouveau_cli(file_priv);
141 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
142 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
143 	struct device *dev = drm->dev->dev;
144 	struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm;
145 	struct nouveau_vma *vma;
146 	int ret;
147 
148 	if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
149 		return;
150 
151 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
152 	if (ret)
153 		return;
154 
155 	vma = nouveau_vma_find(nvbo, vmm);
156 	if (vma) {
157 		if (--vma->refs == 0) {
158 			ret = pm_runtime_get_sync(dev);
159 			if (!WARN_ON(ret < 0 && ret != -EACCES)) {
160 				nouveau_gem_object_unmap(nvbo, vma);
161 				pm_runtime_mark_last_busy(dev);
162 			}
163 			pm_runtime_put_autosuspend(dev);
164 		}
165 	}
166 	ttm_bo_unreserve(&nvbo->bo);
167 }
168 
169 int
170 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
171 		uint32_t tile_mode, uint32_t tile_flags,
172 		struct nouveau_bo **pnvbo)
173 {
174 	struct nouveau_drm *drm = cli->drm;
175 	struct nouveau_bo *nvbo;
176 	u32 flags = 0;
177 	int ret;
178 
179 	if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
180 		flags |= TTM_PL_FLAG_VRAM;
181 	if (domain & NOUVEAU_GEM_DOMAIN_GART)
182 		flags |= TTM_PL_FLAG_TT;
183 	if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
184 		flags |= TTM_PL_FLAG_SYSTEM;
185 
186 	if (domain & NOUVEAU_GEM_DOMAIN_COHERENT)
187 		flags |= TTM_PL_FLAG_UNCACHED;
188 
189 	nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode,
190 				tile_flags);
191 	if (IS_ERR(nvbo))
192 		return PTR_ERR(nvbo);
193 
194 	/* Initialize the embedded gem-object. We return a single gem-reference
195 	 * to the caller, instead of a normal nouveau_bo ttm reference. */
196 	ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size);
197 	if (ret) {
198 		nouveau_bo_ref(NULL, &nvbo);
199 		return ret;
200 	}
201 
202 	ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL);
203 	if (ret) {
204 		nouveau_bo_ref(NULL, &nvbo);
205 		return ret;
206 	}
207 
208 	/* we restrict allowed domains on nv50+ to only the types
209 	 * that were requested at creation time.  not possibly on
210 	 * earlier chips without busting the ABI.
211 	 */
212 	nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
213 			      NOUVEAU_GEM_DOMAIN_GART;
214 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
215 		nvbo->valid_domains &= domain;
216 
217 	nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp;
218 	*pnvbo = nvbo;
219 	return 0;
220 }
221 
222 static int
223 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
224 		 struct drm_nouveau_gem_info *rep)
225 {
226 	struct nouveau_cli *cli = nouveau_cli(file_priv);
227 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
228 	struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
229 	struct nouveau_vma *vma;
230 
231 	if (is_power_of_2(nvbo->valid_domains))
232 		rep->domain = nvbo->valid_domains;
233 	else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
234 		rep->domain = NOUVEAU_GEM_DOMAIN_GART;
235 	else
236 		rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
237 	rep->offset = nvbo->bo.offset;
238 	if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
239 		vma = nouveau_vma_find(nvbo, vmm);
240 		if (!vma)
241 			return -EINVAL;
242 
243 		rep->offset = vma->addr;
244 	}
245 
246 	rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
247 	rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node);
248 	rep->tile_mode = nvbo->mode;
249 	rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
250 	if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
251 		rep->tile_flags |= nvbo->kind << 8;
252 	else
253 	if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
254 		rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
255 	else
256 		rep->tile_flags |= nvbo->zeta;
257 	return 0;
258 }
259 
260 int
261 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
262 		      struct drm_file *file_priv)
263 {
264 	struct nouveau_cli *cli = nouveau_cli(file_priv);
265 	struct drm_nouveau_gem_new *req = data;
266 	struct nouveau_bo *nvbo = NULL;
267 	int ret = 0;
268 
269 	ret = nouveau_gem_new(cli, req->info.size, req->align,
270 			      req->info.domain, req->info.tile_mode,
271 			      req->info.tile_flags, &nvbo);
272 	if (ret)
273 		return ret;
274 
275 	ret = drm_gem_handle_create(file_priv, &nvbo->bo.base,
276 				    &req->info.handle);
277 	if (ret == 0) {
278 		ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info);
279 		if (ret)
280 			drm_gem_handle_delete(file_priv, req->info.handle);
281 	}
282 
283 	/* drop reference from allocate - handle holds it now */
284 	drm_gem_object_put_unlocked(&nvbo->bo.base);
285 	return ret;
286 }
287 
288 static int
289 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
290 		       uint32_t write_domains, uint32_t valid_domains)
291 {
292 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
293 	struct ttm_buffer_object *bo = &nvbo->bo;
294 	uint32_t domains = valid_domains & nvbo->valid_domains &
295 		(write_domains ? write_domains : read_domains);
296 	uint32_t pref_flags = 0, valid_flags = 0;
297 
298 	if (!domains)
299 		return -EINVAL;
300 
301 	if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
302 		valid_flags |= TTM_PL_FLAG_VRAM;
303 
304 	if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
305 		valid_flags |= TTM_PL_FLAG_TT;
306 
307 	if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
308 	    bo->mem.mem_type == TTM_PL_VRAM)
309 		pref_flags |= TTM_PL_FLAG_VRAM;
310 
311 	else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
312 		 bo->mem.mem_type == TTM_PL_TT)
313 		pref_flags |= TTM_PL_FLAG_TT;
314 
315 	else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
316 		pref_flags |= TTM_PL_FLAG_VRAM;
317 
318 	else
319 		pref_flags |= TTM_PL_FLAG_TT;
320 
321 	nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
322 
323 	return 0;
324 }
325 
326 struct validate_op {
327 	struct list_head list;
328 	struct ww_acquire_ctx ticket;
329 };
330 
331 static void
332 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan,
333 			struct nouveau_fence *fence,
334 			struct drm_nouveau_gem_pushbuf_bo *pbbo)
335 {
336 	struct nouveau_bo *nvbo;
337 	struct drm_nouveau_gem_pushbuf_bo *b;
338 
339 	while (!list_empty(&op->list)) {
340 		nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
341 		b = &pbbo[nvbo->pbbo_index];
342 
343 		if (likely(fence)) {
344 			nouveau_bo_fence(nvbo, fence, !!b->write_domains);
345 
346 			if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
347 				struct nouveau_vma *vma =
348 					(void *)(unsigned long)b->user_priv;
349 				nouveau_fence_unref(&vma->fence);
350 				dma_fence_get(&fence->base);
351 				vma->fence = fence;
352 			}
353 		}
354 
355 		if (unlikely(nvbo->validate_mapped)) {
356 			ttm_bo_kunmap(&nvbo->kmap);
357 			nvbo->validate_mapped = false;
358 		}
359 
360 		list_del(&nvbo->entry);
361 		nvbo->reserved_by = NULL;
362 		ttm_bo_unreserve(&nvbo->bo);
363 		drm_gem_object_put_unlocked(&nvbo->bo.base);
364 	}
365 }
366 
367 static void
368 validate_fini(struct validate_op *op, struct nouveau_channel *chan,
369 	      struct nouveau_fence *fence,
370 	      struct drm_nouveau_gem_pushbuf_bo *pbbo)
371 {
372 	validate_fini_no_ticket(op, chan, fence, pbbo);
373 	ww_acquire_fini(&op->ticket);
374 }
375 
376 static int
377 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
378 	      struct drm_nouveau_gem_pushbuf_bo *pbbo,
379 	      int nr_buffers, struct validate_op *op)
380 {
381 	struct nouveau_cli *cli = nouveau_cli(file_priv);
382 	int trycnt = 0;
383 	int ret = -EINVAL, i;
384 	struct nouveau_bo *res_bo = NULL;
385 	LIST_HEAD(gart_list);
386 	LIST_HEAD(vram_list);
387 	LIST_HEAD(both_list);
388 
389 	ww_acquire_init(&op->ticket, &reservation_ww_class);
390 retry:
391 	if (++trycnt > 100000) {
392 		NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
393 		return -EINVAL;
394 	}
395 
396 	for (i = 0; i < nr_buffers; i++) {
397 		struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
398 		struct drm_gem_object *gem;
399 		struct nouveau_bo *nvbo;
400 
401 		gem = drm_gem_object_lookup(file_priv, b->handle);
402 		if (!gem) {
403 			NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
404 			ret = -ENOENT;
405 			break;
406 		}
407 		nvbo = nouveau_gem_object(gem);
408 		if (nvbo == res_bo) {
409 			res_bo = NULL;
410 			drm_gem_object_put_unlocked(gem);
411 			continue;
412 		}
413 
414 		if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
415 			NV_PRINTK(err, cli, "multiple instances of buffer %d on "
416 				      "validation list\n", b->handle);
417 			drm_gem_object_put_unlocked(gem);
418 			ret = -EINVAL;
419 			break;
420 		}
421 
422 		ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
423 		if (ret) {
424 			list_splice_tail_init(&vram_list, &op->list);
425 			list_splice_tail_init(&gart_list, &op->list);
426 			list_splice_tail_init(&both_list, &op->list);
427 			validate_fini_no_ticket(op, chan, NULL, NULL);
428 			if (unlikely(ret == -EDEADLK)) {
429 				ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
430 							      &op->ticket);
431 				if (!ret)
432 					res_bo = nvbo;
433 			}
434 			if (unlikely(ret)) {
435 				if (ret != -ERESTARTSYS)
436 					NV_PRINTK(err, cli, "fail reserve\n");
437 				break;
438 			}
439 		}
440 
441 		if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
442 			struct nouveau_vmm *vmm = chan->vmm;
443 			struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
444 			if (!vma) {
445 				NV_PRINTK(err, cli, "vma not found!\n");
446 				ret = -EINVAL;
447 				break;
448 			}
449 
450 			b->user_priv = (uint64_t)(unsigned long)vma;
451 		} else {
452 			b->user_priv = (uint64_t)(unsigned long)nvbo;
453 		}
454 
455 		nvbo->reserved_by = file_priv;
456 		nvbo->pbbo_index = i;
457 		if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
458 		    (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
459 			list_add_tail(&nvbo->entry, &both_list);
460 		else
461 		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
462 			list_add_tail(&nvbo->entry, &vram_list);
463 		else
464 		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
465 			list_add_tail(&nvbo->entry, &gart_list);
466 		else {
467 			NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
468 				 b->valid_domains);
469 			list_add_tail(&nvbo->entry, &both_list);
470 			ret = -EINVAL;
471 			break;
472 		}
473 		if (nvbo == res_bo)
474 			goto retry;
475 	}
476 
477 	ww_acquire_done(&op->ticket);
478 	list_splice_tail(&vram_list, &op->list);
479 	list_splice_tail(&gart_list, &op->list);
480 	list_splice_tail(&both_list, &op->list);
481 	if (ret)
482 		validate_fini(op, chan, NULL, NULL);
483 	return ret;
484 
485 }
486 
487 static int
488 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
489 	      struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo)
490 {
491 	struct nouveau_drm *drm = chan->drm;
492 	struct nouveau_bo *nvbo;
493 	int ret, relocs = 0;
494 
495 	list_for_each_entry(nvbo, list, entry) {
496 		struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
497 
498 		ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains,
499 					     b->write_domains,
500 					     b->valid_domains);
501 		if (unlikely(ret)) {
502 			NV_PRINTK(err, cli, "fail set_domain\n");
503 			return ret;
504 		}
505 
506 		ret = nouveau_bo_validate(nvbo, true, false);
507 		if (unlikely(ret)) {
508 			if (ret != -ERESTARTSYS)
509 				NV_PRINTK(err, cli, "fail ttm_validate\n");
510 			return ret;
511 		}
512 
513 		ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
514 		if (unlikely(ret)) {
515 			if (ret != -ERESTARTSYS)
516 				NV_PRINTK(err, cli, "fail post-validate sync\n");
517 			return ret;
518 		}
519 
520 		if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
521 			if (nvbo->bo.offset == b->presumed.offset &&
522 			    ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
523 			      b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
524 			     (nvbo->bo.mem.mem_type == TTM_PL_TT &&
525 			      b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
526 				continue;
527 
528 			if (nvbo->bo.mem.mem_type == TTM_PL_TT)
529 				b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
530 			else
531 				b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
532 			b->presumed.offset = nvbo->bo.offset;
533 			b->presumed.valid = 0;
534 			relocs++;
535 		}
536 	}
537 
538 	return relocs;
539 }
540 
541 static int
542 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
543 			     struct drm_file *file_priv,
544 			     struct drm_nouveau_gem_pushbuf_bo *pbbo,
545 			     int nr_buffers,
546 			     struct validate_op *op, bool *apply_relocs)
547 {
548 	struct nouveau_cli *cli = nouveau_cli(file_priv);
549 	int ret;
550 
551 	INIT_LIST_HEAD(&op->list);
552 
553 	if (nr_buffers == 0)
554 		return 0;
555 
556 	ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
557 	if (unlikely(ret)) {
558 		if (ret != -ERESTARTSYS)
559 			NV_PRINTK(err, cli, "validate_init\n");
560 		return ret;
561 	}
562 
563 	ret = validate_list(chan, cli, &op->list, pbbo);
564 	if (unlikely(ret < 0)) {
565 		if (ret != -ERESTARTSYS)
566 			NV_PRINTK(err, cli, "validating bo list\n");
567 		validate_fini(op, chan, NULL, NULL);
568 		return ret;
569 	}
570 	*apply_relocs = ret;
571 	return 0;
572 }
573 
574 static inline void
575 u_free(void *addr)
576 {
577 	kvfree(addr);
578 }
579 
580 static inline void *
581 u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
582 {
583 	void *mem;
584 	void __user *userptr = (void __force __user *)(uintptr_t)user;
585 
586 	size *= nmemb;
587 
588 	mem = kvmalloc(size, GFP_KERNEL);
589 	if (!mem)
590 		return ERR_PTR(-ENOMEM);
591 
592 	if (copy_from_user(mem, userptr, size)) {
593 		u_free(mem);
594 		return ERR_PTR(-EFAULT);
595 	}
596 
597 	return mem;
598 }
599 
600 static int
601 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
602 				struct drm_nouveau_gem_pushbuf *req,
603 				struct drm_nouveau_gem_pushbuf_reloc *reloc,
604 				struct drm_nouveau_gem_pushbuf_bo *bo)
605 {
606 	int ret = 0;
607 	unsigned i;
608 
609 	for (i = 0; i < req->nr_relocs; i++) {
610 		struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
611 		struct drm_nouveau_gem_pushbuf_bo *b;
612 		struct nouveau_bo *nvbo;
613 		uint32_t data;
614 
615 		if (unlikely(r->bo_index >= req->nr_buffers)) {
616 			NV_PRINTK(err, cli, "reloc bo index invalid\n");
617 			ret = -EINVAL;
618 			break;
619 		}
620 
621 		b = &bo[r->bo_index];
622 		if (b->presumed.valid)
623 			continue;
624 
625 		if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
626 			NV_PRINTK(err, cli, "reloc container bo index invalid\n");
627 			ret = -EINVAL;
628 			break;
629 		}
630 		nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
631 
632 		if (unlikely(r->reloc_bo_offset + 4 >
633 			     nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
634 			NV_PRINTK(err, cli, "reloc outside of bo\n");
635 			ret = -EINVAL;
636 			break;
637 		}
638 
639 		if (!nvbo->kmap.virtual) {
640 			ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
641 					  &nvbo->kmap);
642 			if (ret) {
643 				NV_PRINTK(err, cli, "failed kmap for reloc\n");
644 				break;
645 			}
646 			nvbo->validate_mapped = true;
647 		}
648 
649 		if (r->flags & NOUVEAU_GEM_RELOC_LOW)
650 			data = b->presumed.offset + r->data;
651 		else
652 		if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
653 			data = (b->presumed.offset + r->data) >> 32;
654 		else
655 			data = r->data;
656 
657 		if (r->flags & NOUVEAU_GEM_RELOC_OR) {
658 			if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
659 				data |= r->tor;
660 			else
661 				data |= r->vor;
662 		}
663 
664 		ret = ttm_bo_wait(&nvbo->bo, false, false);
665 		if (ret) {
666 			NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
667 			break;
668 		}
669 
670 		nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
671 	}
672 
673 	u_free(reloc);
674 	return ret;
675 }
676 
677 int
678 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
679 			  struct drm_file *file_priv)
680 {
681 	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
682 	struct nouveau_cli *cli = nouveau_cli(file_priv);
683 	struct nouveau_abi16_chan *temp;
684 	struct nouveau_drm *drm = nouveau_drm(dev);
685 	struct drm_nouveau_gem_pushbuf *req = data;
686 	struct drm_nouveau_gem_pushbuf_push *push;
687 	struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
688 	struct drm_nouveau_gem_pushbuf_bo *bo;
689 	struct nouveau_channel *chan = NULL;
690 	struct validate_op op;
691 	struct nouveau_fence *fence = NULL;
692 	int i, j, ret = 0;
693 	bool do_reloc = false, sync = false;
694 
695 	if (unlikely(!abi16))
696 		return -ENOMEM;
697 
698 	list_for_each_entry(temp, &abi16->channels, head) {
699 		if (temp->chan->chid == req->channel) {
700 			chan = temp->chan;
701 			break;
702 		}
703 	}
704 
705 	if (!chan)
706 		return nouveau_abi16_put(abi16, -ENOENT);
707 	if (unlikely(atomic_read(&chan->killed)))
708 		return nouveau_abi16_put(abi16, -ENODEV);
709 
710 	sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC;
711 
712 	req->vram_available = drm->gem.vram_available;
713 	req->gart_available = drm->gem.gart_available;
714 	if (unlikely(req->nr_push == 0))
715 		goto out_next;
716 
717 	if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
718 		NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
719 			 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
720 		return nouveau_abi16_put(abi16, -EINVAL);
721 	}
722 
723 	if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
724 		NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
725 			 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
726 		return nouveau_abi16_put(abi16, -EINVAL);
727 	}
728 
729 	if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
730 		NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
731 			 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
732 		return nouveau_abi16_put(abi16, -EINVAL);
733 	}
734 
735 	push = u_memcpya(req->push, req->nr_push, sizeof(*push));
736 	if (IS_ERR(push))
737 		return nouveau_abi16_put(abi16, PTR_ERR(push));
738 
739 	bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
740 	if (IS_ERR(bo)) {
741 		u_free(push);
742 		return nouveau_abi16_put(abi16, PTR_ERR(bo));
743 	}
744 
745 	/* Ensure all push buffers are on validate list */
746 	for (i = 0; i < req->nr_push; i++) {
747 		if (push[i].bo_index >= req->nr_buffers) {
748 			NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
749 			ret = -EINVAL;
750 			goto out_prevalid;
751 		}
752 	}
753 
754 	/* Validate buffer list */
755 revalidate:
756 	ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo,
757 					   req->nr_buffers, &op, &do_reloc);
758 	if (ret) {
759 		if (ret != -ERESTARTSYS)
760 			NV_PRINTK(err, cli, "validate: %d\n", ret);
761 		goto out_prevalid;
762 	}
763 
764 	/* Apply any relocations that are required */
765 	if (do_reloc) {
766 		if (!reloc) {
767 			validate_fini(&op, chan, NULL, bo);
768 			reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
769 			if (IS_ERR(reloc)) {
770 				ret = PTR_ERR(reloc);
771 				goto out_prevalid;
772 			}
773 
774 			goto revalidate;
775 		}
776 
777 		ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo);
778 		if (ret) {
779 			NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
780 			goto out;
781 		}
782 	}
783 
784 	if (chan->dma.ib_max) {
785 		ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
786 		if (ret) {
787 			NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
788 			goto out;
789 		}
790 
791 		for (i = 0; i < req->nr_push; i++) {
792 			struct nouveau_vma *vma = (void *)(unsigned long)
793 				bo[push[i].bo_index].user_priv;
794 
795 			nv50_dma_push(chan, vma->addr + push[i].offset,
796 				      push[i].length);
797 		}
798 	} else
799 	if (drm->client.device.info.chipset >= 0x25) {
800 		ret = RING_SPACE(chan, req->nr_push * 2);
801 		if (ret) {
802 			NV_PRINTK(err, cli, "cal_space: %d\n", ret);
803 			goto out;
804 		}
805 
806 		for (i = 0; i < req->nr_push; i++) {
807 			struct nouveau_bo *nvbo = (void *)(unsigned long)
808 				bo[push[i].bo_index].user_priv;
809 
810 			OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
811 			OUT_RING(chan, 0);
812 		}
813 	} else {
814 		ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
815 		if (ret) {
816 			NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
817 			goto out;
818 		}
819 
820 		for (i = 0; i < req->nr_push; i++) {
821 			struct nouveau_bo *nvbo = (void *)(unsigned long)
822 				bo[push[i].bo_index].user_priv;
823 			uint32_t cmd;
824 
825 			cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
826 			cmd |= 0x20000000;
827 			if (unlikely(cmd != req->suffix0)) {
828 				if (!nvbo->kmap.virtual) {
829 					ret = ttm_bo_kmap(&nvbo->bo, 0,
830 							  nvbo->bo.mem.
831 							  num_pages,
832 							  &nvbo->kmap);
833 					if (ret) {
834 						WIND_RING(chan);
835 						goto out;
836 					}
837 					nvbo->validate_mapped = true;
838 				}
839 
840 				nouveau_bo_wr32(nvbo, (push[i].offset +
841 						push[i].length - 8) / 4, cmd);
842 			}
843 
844 			OUT_RING(chan, 0x20000000 |
845 				      (nvbo->bo.offset + push[i].offset));
846 			OUT_RING(chan, 0);
847 			for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
848 				OUT_RING(chan, 0);
849 		}
850 	}
851 
852 	ret = nouveau_fence_new(chan, false, &fence);
853 	if (ret) {
854 		NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
855 		WIND_RING(chan);
856 		goto out;
857 	}
858 
859 	if (sync) {
860 		if (!(ret = nouveau_fence_wait(fence, false, false))) {
861 			if ((ret = dma_fence_get_status(&fence->base)) == 1)
862 				ret = 0;
863 		}
864 	}
865 
866 out:
867 	validate_fini(&op, chan, fence, bo);
868 	nouveau_fence_unref(&fence);
869 
870 	if (do_reloc) {
871 		struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
872 			u64_to_user_ptr(req->buffers);
873 
874 		for (i = 0; i < req->nr_buffers; i++) {
875 			if (bo[i].presumed.valid)
876 				continue;
877 
878 			if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed,
879 					 sizeof(bo[i].presumed))) {
880 				ret = -EFAULT;
881 				break;
882 			}
883 		}
884 		u_free(reloc);
885 	}
886 out_prevalid:
887 	u_free(bo);
888 	u_free(push);
889 
890 out_next:
891 	if (chan->dma.ib_max) {
892 		req->suffix0 = 0x00000000;
893 		req->suffix1 = 0x00000000;
894 	} else
895 	if (drm->client.device.info.chipset >= 0x25) {
896 		req->suffix0 = 0x00020000;
897 		req->suffix1 = 0x00000000;
898 	} else {
899 		req->suffix0 = 0x20000000 |
900 			      (chan->push.addr + ((chan->dma.cur + 2) << 2));
901 		req->suffix1 = 0x00000000;
902 	}
903 
904 	return nouveau_abi16_put(abi16, ret);
905 }
906 
907 int
908 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
909 			   struct drm_file *file_priv)
910 {
911 	struct drm_nouveau_gem_cpu_prep *req = data;
912 	struct drm_gem_object *gem;
913 	struct nouveau_bo *nvbo;
914 	bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
915 	bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
916 	long lret;
917 	int ret;
918 
919 	gem = drm_gem_object_lookup(file_priv, req->handle);
920 	if (!gem)
921 		return -ENOENT;
922 	nvbo = nouveau_gem_object(gem);
923 
924 	lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true,
925 						   no_wait ? 0 : 30 * HZ);
926 	if (!lret)
927 		ret = -EBUSY;
928 	else if (lret > 0)
929 		ret = 0;
930 	else
931 		ret = lret;
932 
933 	nouveau_bo_sync_for_cpu(nvbo);
934 	drm_gem_object_put_unlocked(gem);
935 
936 	return ret;
937 }
938 
939 int
940 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
941 			   struct drm_file *file_priv)
942 {
943 	struct drm_nouveau_gem_cpu_fini *req = data;
944 	struct drm_gem_object *gem;
945 	struct nouveau_bo *nvbo;
946 
947 	gem = drm_gem_object_lookup(file_priv, req->handle);
948 	if (!gem)
949 		return -ENOENT;
950 	nvbo = nouveau_gem_object(gem);
951 
952 	nouveau_bo_sync_for_device(nvbo);
953 	drm_gem_object_put_unlocked(gem);
954 	return 0;
955 }
956 
957 int
958 nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
959 		       struct drm_file *file_priv)
960 {
961 	struct drm_nouveau_gem_info *req = data;
962 	struct drm_gem_object *gem;
963 	int ret;
964 
965 	gem = drm_gem_object_lookup(file_priv, req->handle);
966 	if (!gem)
967 		return -ENOENT;
968 
969 	ret = nouveau_gem_info(file_priv, gem, req);
970 	drm_gem_object_put_unlocked(gem);
971 	return ret;
972 }
973 
974