1 /*
2  * Copyright 2007 Dave Airlied
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  */
24 /*
25  * Authors: Dave Airlied <airlied@linux.ie>
26  *	    Ben Skeggs   <darktama@iinet.net.au>
27  *	    Jeremy Kolb  <jkolb@brandeis.edu>
28  */
29 
30 #include <linux/dma-mapping.h>
31 #include <drm/ttm/ttm_tt.h>
32 
33 #include "nouveau_drv.h"
34 #include "nouveau_chan.h"
35 #include "nouveau_fence.h"
36 
37 #include "nouveau_bo.h"
38 #include "nouveau_ttm.h"
39 #include "nouveau_gem.h"
40 #include "nouveau_mem.h"
41 #include "nouveau_vmm.h"
42 
43 #include <nvif/class.h>
44 #include <nvif/if500b.h>
45 #include <nvif/if900b.h>
46 
47 static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm,
48 			       struct ttm_resource *reg);
49 static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm);
50 
51 /*
52  * NV10-NV40 tiling helpers
53  */
54 
55 static void
56 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
57 			   u32 addr, u32 size, u32 pitch, u32 flags)
58 {
59 	struct nouveau_drm *drm = nouveau_drm(dev);
60 	int i = reg - drm->tile.reg;
61 	struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
62 	struct nvkm_fb_tile *tile = &fb->tile.region[i];
63 
64 	nouveau_fence_unref(&reg->fence);
65 
66 	if (tile->pitch)
67 		nvkm_fb_tile_fini(fb, i, tile);
68 
69 	if (pitch)
70 		nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
71 
72 	nvkm_fb_tile_prog(fb, i, tile);
73 }
74 
75 static struct nouveau_drm_tile *
76 nv10_bo_get_tile_region(struct drm_device *dev, int i)
77 {
78 	struct nouveau_drm *drm = nouveau_drm(dev);
79 	struct nouveau_drm_tile *tile = &drm->tile.reg[i];
80 
81 	spin_lock(&drm->tile.lock);
82 
83 	if (!tile->used &&
84 	    (!tile->fence || nouveau_fence_done(tile->fence)))
85 		tile->used = true;
86 	else
87 		tile = NULL;
88 
89 	spin_unlock(&drm->tile.lock);
90 	return tile;
91 }
92 
93 static void
94 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
95 			struct dma_fence *fence)
96 {
97 	struct nouveau_drm *drm = nouveau_drm(dev);
98 
99 	if (tile) {
100 		spin_lock(&drm->tile.lock);
101 		tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
102 		tile->used = false;
103 		spin_unlock(&drm->tile.lock);
104 	}
105 }
106 
107 static struct nouveau_drm_tile *
108 nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
109 		   u32 size, u32 pitch, u32 zeta)
110 {
111 	struct nouveau_drm *drm = nouveau_drm(dev);
112 	struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
113 	struct nouveau_drm_tile *tile, *found = NULL;
114 	int i;
115 
116 	for (i = 0; i < fb->tile.regions; i++) {
117 		tile = nv10_bo_get_tile_region(dev, i);
118 
119 		if (pitch && !found) {
120 			found = tile;
121 			continue;
122 
123 		} else if (tile && fb->tile.region[i].pitch) {
124 			/* Kill an unused tile region. */
125 			nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
126 		}
127 
128 		nv10_bo_put_tile_region(dev, tile, NULL);
129 	}
130 
131 	if (found)
132 		nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta);
133 	return found;
134 }
135 
136 static void
137 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
138 {
139 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
140 	struct drm_device *dev = drm->dev;
141 	struct nouveau_bo *nvbo = nouveau_bo(bo);
142 
143 	WARN_ON(nvbo->bo.pin_count > 0);
144 	nouveau_bo_del_io_reserve_lru(bo);
145 	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
146 
147 	/*
148 	 * If nouveau_bo_new() allocated this buffer, the GEM object was never
149 	 * initialized, so don't attempt to release it.
150 	 */
151 	if (bo->base.dev)
152 		drm_gem_object_release(&bo->base);
153 	else
154 		dma_resv_fini(&bo->base._resv);
155 
156 	kfree(nvbo);
157 }
158 
159 static inline u64
160 roundup_64(u64 x, u32 y)
161 {
162 	x += y - 1;
163 	do_div(x, y);
164 	return x * y;
165 }
166 
167 static void
168 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size)
169 {
170 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
171 	struct nvif_device *device = &drm->client.device;
172 
173 	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
174 		if (nvbo->mode) {
175 			if (device->info.chipset >= 0x40) {
176 				*align = 65536;
177 				*size = roundup_64(*size, 64 * nvbo->mode);
178 
179 			} else if (device->info.chipset >= 0x30) {
180 				*align = 32768;
181 				*size = roundup_64(*size, 64 * nvbo->mode);
182 
183 			} else if (device->info.chipset >= 0x20) {
184 				*align = 16384;
185 				*size = roundup_64(*size, 64 * nvbo->mode);
186 
187 			} else if (device->info.chipset >= 0x10) {
188 				*align = 16384;
189 				*size = roundup_64(*size, 32 * nvbo->mode);
190 			}
191 		}
192 	} else {
193 		*size = roundup_64(*size, (1 << nvbo->page));
194 		*align = max((1 <<  nvbo->page), *align);
195 	}
196 
197 	*size = roundup_64(*size, PAGE_SIZE);
198 }
199 
200 struct nouveau_bo *
201 nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain,
202 		 u32 tile_mode, u32 tile_flags, bool internal)
203 {
204 	struct nouveau_drm *drm = cli->drm;
205 	struct nouveau_bo *nvbo;
206 	struct nvif_mmu *mmu = &cli->mmu;
207 	struct nvif_vmm *vmm = &nouveau_cli_vmm(cli)->vmm;
208 	int i, pi = -1;
209 
210 	if (!*size) {
211 		NV_WARN(drm, "skipped size %016llx\n", *size);
212 		return ERR_PTR(-EINVAL);
213 	}
214 
215 	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
216 	if (!nvbo)
217 		return ERR_PTR(-ENOMEM);
218 
219 	INIT_LIST_HEAD(&nvbo->head);
220 	INIT_LIST_HEAD(&nvbo->entry);
221 	INIT_LIST_HEAD(&nvbo->vma_list);
222 	nvbo->bo.bdev = &drm->ttm.bdev;
223 
224 	/* This is confusing, and doesn't actually mean we want an uncached
225 	 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated
226 	 * into in nouveau_gem_new().
227 	 */
228 	if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) {
229 		/* Determine if we can get a cache-coherent map, forcing
230 		 * uncached mapping if we can't.
231 		 */
232 		if (!nouveau_drm_use_coherent_gpu_mapping(drm))
233 			nvbo->force_coherent = true;
234 	}
235 
236 	nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);
237 
238 	if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
239 		nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
240 		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
241 			kfree(nvbo);
242 			return ERR_PTR(-EINVAL);
243 		}
244 
245 		nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
246 	} else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
247 		nvbo->kind = (tile_flags & 0x00007f00) >> 8;
248 		nvbo->comp = (tile_flags & 0x00030000) >> 16;
249 		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
250 			kfree(nvbo);
251 			return ERR_PTR(-EINVAL);
252 		}
253 	} else {
254 		nvbo->zeta = (tile_flags & 0x00000007);
255 	}
256 	nvbo->mode = tile_mode;
257 
258 	if (!nouveau_cli_uvmm(cli) || internal) {
259 		/* Determine the desirable target GPU page size for the buffer. */
260 		for (i = 0; i < vmm->page_nr; i++) {
261 			/* Because we cannot currently allow VMM maps to fail
262 			 * during buffer migration, we need to determine page
263 			 * size for the buffer up-front, and pre-allocate its
264 			 * page tables.
265 			 *
266 			 * Skip page sizes that can't support needed domains.
267 			 */
268 			if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE &&
269 			    (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram)
270 				continue;
271 			if ((domain & NOUVEAU_GEM_DOMAIN_GART) &&
272 			    (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
273 				continue;
274 
275 			/* Select this page size if it's the first that supports
276 			 * the potential memory domains, or when it's compatible
277 			 * with the requested compression settings.
278 			 */
279 			if (pi < 0 || !nvbo->comp || vmm->page[i].comp)
280 				pi = i;
281 
282 			/* Stop once the buffer is larger than the current page size. */
283 			if (*size >= 1ULL << vmm->page[i].shift)
284 				break;
285 		}
286 
287 		if (WARN_ON(pi < 0)) {
288 			kfree(nvbo);
289 			return ERR_PTR(-EINVAL);
290 		}
291 
292 		/* Disable compression if suitable settings couldn't be found. */
293 		if (nvbo->comp && !vmm->page[pi].comp) {
294 			if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100)
295 				nvbo->kind = mmu->kind[nvbo->kind];
296 			nvbo->comp = 0;
297 		}
298 		nvbo->page = vmm->page[pi].shift;
299 	} else {
300 		/* Determine the desirable target GPU page size for the buffer. */
301 		for (i = 0; i < vmm->page_nr; i++) {
302 			/* Because we cannot currently allow VMM maps to fail
303 			 * during buffer migration, we need to determine page
304 			 * size for the buffer up-front, and pre-allocate its
305 			 * page tables.
306 			 *
307 			 * Skip page sizes that can't support needed domains.
308 			 */
309 			if ((domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram)
310 				continue;
311 			if ((domain & NOUVEAU_GEM_DOMAIN_GART) &&
312 			    (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
313 				continue;
314 
315 			/* pick the last one as it will be smallest. */
316 			pi = i;
317 
318 			/* Stop once the buffer is larger than the current page size. */
319 			if (*size >= 1ULL << vmm->page[i].shift)
320 				break;
321 		}
322 		if (WARN_ON(pi < 0)) {
323 			kfree(nvbo);
324 			return ERR_PTR(-EINVAL);
325 		}
326 		nvbo->page = vmm->page[pi].shift;
327 	}
328 
329 	nouveau_bo_fixup_align(nvbo, align, size);
330 
331 	return nvbo;
332 }
333 
334 int
335 nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain,
336 		struct sg_table *sg, struct dma_resv *robj)
337 {
338 	int type = sg ? ttm_bo_type_sg : ttm_bo_type_device;
339 	int ret;
340 	struct ttm_operation_ctx ctx = {
341 		.interruptible = false,
342 		.no_wait_gpu = false,
343 		.resv = robj,
344 	};
345 
346 	nouveau_bo_placement_set(nvbo, domain, 0);
347 	INIT_LIST_HEAD(&nvbo->io_reserve_lru);
348 
349 	ret = ttm_bo_init_reserved(nvbo->bo.bdev, &nvbo->bo, type,
350 				   &nvbo->placement, align >> PAGE_SHIFT, &ctx,
351 				   sg, robj, nouveau_bo_del_ttm);
352 	if (ret) {
353 		/* ttm will call nouveau_bo_del_ttm if it fails.. */
354 		return ret;
355 	}
356 
357 	if (!robj)
358 		ttm_bo_unreserve(&nvbo->bo);
359 
360 	return 0;
361 }
362 
363 int
364 nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align,
365 	       uint32_t domain, uint32_t tile_mode, uint32_t tile_flags,
366 	       struct sg_table *sg, struct dma_resv *robj,
367 	       struct nouveau_bo **pnvbo)
368 {
369 	struct nouveau_bo *nvbo;
370 	int ret;
371 
372 	nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode,
373 				tile_flags, true);
374 	if (IS_ERR(nvbo))
375 		return PTR_ERR(nvbo);
376 
377 	nvbo->bo.base.size = size;
378 	dma_resv_init(&nvbo->bo.base._resv);
379 	drm_vma_node_reset(&nvbo->bo.base.vma_node);
380 
381 	/* This must be called before ttm_bo_init_reserved(). Subsequent
382 	 * bo_move() callbacks might already iterate the GEMs GPUVA list.
383 	 */
384 	drm_gem_gpuva_init(&nvbo->bo.base);
385 
386 	ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj);
387 	if (ret)
388 		return ret;
389 
390 	*pnvbo = nvbo;
391 	return 0;
392 }
393 
394 static void
395 set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t domain)
396 {
397 	*n = 0;
398 
399 	if (domain & NOUVEAU_GEM_DOMAIN_VRAM) {
400 		pl[*n].mem_type = TTM_PL_VRAM;
401 		pl[*n].flags = 0;
402 		(*n)++;
403 	}
404 	if (domain & NOUVEAU_GEM_DOMAIN_GART) {
405 		pl[*n].mem_type = TTM_PL_TT;
406 		pl[*n].flags = 0;
407 		(*n)++;
408 	}
409 	if (domain & NOUVEAU_GEM_DOMAIN_CPU) {
410 		pl[*n].mem_type = TTM_PL_SYSTEM;
411 		pl[(*n)++].flags = 0;
412 	}
413 }
414 
415 static void
416 set_placement_range(struct nouveau_bo *nvbo, uint32_t domain)
417 {
418 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
419 	u64 vram_size = drm->client.device.info.ram_size;
420 	unsigned i, fpfn, lpfn;
421 
422 	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
423 	    nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) &&
424 	    nvbo->bo.base.size < vram_size / 4) {
425 		/*
426 		 * Make sure that the color and depth buffers are handled
427 		 * by independent memory controller units. Up to a 9x
428 		 * speed up when alpha-blending and depth-test are enabled
429 		 * at the same time.
430 		 */
431 		if (nvbo->zeta) {
432 			fpfn = (vram_size / 2) >> PAGE_SHIFT;
433 			lpfn = ~0;
434 		} else {
435 			fpfn = 0;
436 			lpfn = (vram_size / 2) >> PAGE_SHIFT;
437 		}
438 		for (i = 0; i < nvbo->placement.num_placement; ++i) {
439 			nvbo->placements[i].fpfn = fpfn;
440 			nvbo->placements[i].lpfn = lpfn;
441 		}
442 		for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
443 			nvbo->busy_placements[i].fpfn = fpfn;
444 			nvbo->busy_placements[i].lpfn = lpfn;
445 		}
446 	}
447 }
448 
449 void
450 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain,
451 			 uint32_t busy)
452 {
453 	struct ttm_placement *pl = &nvbo->placement;
454 
455 	pl->placement = nvbo->placements;
456 	set_placement_list(nvbo->placements, &pl->num_placement, domain);
457 
458 	pl->busy_placement = nvbo->busy_placements;
459 	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
460 			   domain | busy);
461 
462 	set_placement_range(nvbo, domain);
463 }
464 
465 int
466 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
467 {
468 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
469 	struct ttm_buffer_object *bo = &nvbo->bo;
470 	bool force = false, evict = false;
471 	int ret;
472 
473 	ret = ttm_bo_reserve(bo, false, false, NULL);
474 	if (ret)
475 		return ret;
476 
477 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
478 	    domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) {
479 		if (!nvbo->contig) {
480 			nvbo->contig = true;
481 			force = true;
482 			evict = true;
483 		}
484 	}
485 
486 	if (nvbo->bo.pin_count) {
487 		bool error = evict;
488 
489 		switch (bo->resource->mem_type) {
490 		case TTM_PL_VRAM:
491 			error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM);
492 			break;
493 		case TTM_PL_TT:
494 			error |= !(domain & NOUVEAU_GEM_DOMAIN_GART);
495 			break;
496 		default:
497 			break;
498 		}
499 
500 		if (error) {
501 			NV_ERROR(drm, "bo %p pinned elsewhere: "
502 				      "0x%08x vs 0x%08x\n", bo,
503 				 bo->resource->mem_type, domain);
504 			ret = -EBUSY;
505 		}
506 		ttm_bo_pin(&nvbo->bo);
507 		goto out;
508 	}
509 
510 	if (evict) {
511 		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0);
512 		ret = nouveau_bo_validate(nvbo, false, false);
513 		if (ret)
514 			goto out;
515 	}
516 
517 	nouveau_bo_placement_set(nvbo, domain, 0);
518 	ret = nouveau_bo_validate(nvbo, false, false);
519 	if (ret)
520 		goto out;
521 
522 	ttm_bo_pin(&nvbo->bo);
523 
524 	switch (bo->resource->mem_type) {
525 	case TTM_PL_VRAM:
526 		drm->gem.vram_available -= bo->base.size;
527 		break;
528 	case TTM_PL_TT:
529 		drm->gem.gart_available -= bo->base.size;
530 		break;
531 	default:
532 		break;
533 	}
534 
535 out:
536 	if (force && ret)
537 		nvbo->contig = false;
538 	ttm_bo_unreserve(bo);
539 	return ret;
540 }
541 
542 int
543 nouveau_bo_unpin(struct nouveau_bo *nvbo)
544 {
545 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
546 	struct ttm_buffer_object *bo = &nvbo->bo;
547 	int ret;
548 
549 	ret = ttm_bo_reserve(bo, false, false, NULL);
550 	if (ret)
551 		return ret;
552 
553 	ttm_bo_unpin(&nvbo->bo);
554 	if (!nvbo->bo.pin_count) {
555 		switch (bo->resource->mem_type) {
556 		case TTM_PL_VRAM:
557 			drm->gem.vram_available += bo->base.size;
558 			break;
559 		case TTM_PL_TT:
560 			drm->gem.gart_available += bo->base.size;
561 			break;
562 		default:
563 			break;
564 		}
565 	}
566 
567 	ttm_bo_unreserve(bo);
568 	return 0;
569 }
570 
571 int
572 nouveau_bo_map(struct nouveau_bo *nvbo)
573 {
574 	int ret;
575 
576 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
577 	if (ret)
578 		return ret;
579 
580 	ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap);
581 
582 	ttm_bo_unreserve(&nvbo->bo);
583 	return ret;
584 }
585 
586 void
587 nouveau_bo_unmap(struct nouveau_bo *nvbo)
588 {
589 	if (!nvbo)
590 		return;
591 
592 	ttm_bo_kunmap(&nvbo->kmap);
593 }
594 
595 void
596 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
597 {
598 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
599 	struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm;
600 	int i, j;
601 
602 	if (!ttm_dma || !ttm_dma->dma_address)
603 		return;
604 	if (!ttm_dma->pages) {
605 		NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma);
606 		return;
607 	}
608 
609 	/* Don't waste time looping if the object is coherent */
610 	if (nvbo->force_coherent)
611 		return;
612 
613 	i = 0;
614 	while (i < ttm_dma->num_pages) {
615 		struct page *p = ttm_dma->pages[i];
616 		size_t num_pages = 1;
617 
618 		for (j = i + 1; j < ttm_dma->num_pages; ++j) {
619 			if (++p != ttm_dma->pages[j])
620 				break;
621 
622 			++num_pages;
623 		}
624 		dma_sync_single_for_device(drm->dev->dev,
625 					   ttm_dma->dma_address[i],
626 					   num_pages * PAGE_SIZE, DMA_TO_DEVICE);
627 		i += num_pages;
628 	}
629 }
630 
631 void
632 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
633 {
634 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
635 	struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm;
636 	int i, j;
637 
638 	if (!ttm_dma || !ttm_dma->dma_address)
639 		return;
640 	if (!ttm_dma->pages) {
641 		NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma);
642 		return;
643 	}
644 
645 	/* Don't waste time looping if the object is coherent */
646 	if (nvbo->force_coherent)
647 		return;
648 
649 	i = 0;
650 	while (i < ttm_dma->num_pages) {
651 		struct page *p = ttm_dma->pages[i];
652 		size_t num_pages = 1;
653 
654 		for (j = i + 1; j < ttm_dma->num_pages; ++j) {
655 			if (++p != ttm_dma->pages[j])
656 				break;
657 
658 			++num_pages;
659 		}
660 
661 		dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i],
662 					num_pages * PAGE_SIZE, DMA_FROM_DEVICE);
663 		i += num_pages;
664 	}
665 }
666 
667 void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo)
668 {
669 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
670 	struct nouveau_bo *nvbo = nouveau_bo(bo);
671 
672 	mutex_lock(&drm->ttm.io_reserve_mutex);
673 	list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru);
674 	mutex_unlock(&drm->ttm.io_reserve_mutex);
675 }
676 
677 void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo)
678 {
679 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
680 	struct nouveau_bo *nvbo = nouveau_bo(bo);
681 
682 	mutex_lock(&drm->ttm.io_reserve_mutex);
683 	list_del_init(&nvbo->io_reserve_lru);
684 	mutex_unlock(&drm->ttm.io_reserve_mutex);
685 }
686 
687 int
688 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
689 		    bool no_wait_gpu)
690 {
691 	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
692 	int ret;
693 
694 	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx);
695 	if (ret)
696 		return ret;
697 
698 	nouveau_bo_sync_for_device(nvbo);
699 
700 	return 0;
701 }
702 
703 void
704 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
705 {
706 	bool is_iomem;
707 	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
708 
709 	mem += index;
710 
711 	if (is_iomem)
712 		iowrite16_native(val, (void __force __iomem *)mem);
713 	else
714 		*mem = val;
715 }
716 
717 u32
718 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
719 {
720 	bool is_iomem;
721 	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
722 
723 	mem += index;
724 
725 	if (is_iomem)
726 		return ioread32_native((void __force __iomem *)mem);
727 	else
728 		return *mem;
729 }
730 
731 void
732 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
733 {
734 	bool is_iomem;
735 	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
736 
737 	mem += index;
738 
739 	if (is_iomem)
740 		iowrite32_native(val, (void __force __iomem *)mem);
741 	else
742 		*mem = val;
743 }
744 
745 static struct ttm_tt *
746 nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags)
747 {
748 #if IS_ENABLED(CONFIG_AGP)
749 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
750 
751 	if (drm->agp.bridge) {
752 		return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags);
753 	}
754 #endif
755 
756 	return nouveau_sgdma_create_ttm(bo, page_flags);
757 }
758 
759 static int
760 nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm,
761 		    struct ttm_resource *reg)
762 {
763 #if IS_ENABLED(CONFIG_AGP)
764 	struct nouveau_drm *drm = nouveau_bdev(bdev);
765 #endif
766 	if (!reg)
767 		return -EINVAL;
768 #if IS_ENABLED(CONFIG_AGP)
769 	if (drm->agp.bridge)
770 		return ttm_agp_bind(ttm, reg);
771 #endif
772 	return nouveau_sgdma_bind(bdev, ttm, reg);
773 }
774 
775 static void
776 nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm)
777 {
778 #if IS_ENABLED(CONFIG_AGP)
779 	struct nouveau_drm *drm = nouveau_bdev(bdev);
780 
781 	if (drm->agp.bridge) {
782 		ttm_agp_unbind(ttm);
783 		return;
784 	}
785 #endif
786 	nouveau_sgdma_unbind(bdev, ttm);
787 }
788 
789 static void
790 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
791 {
792 	struct nouveau_bo *nvbo = nouveau_bo(bo);
793 
794 	switch (bo->resource->mem_type) {
795 	case TTM_PL_VRAM:
796 		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART,
797 					 NOUVEAU_GEM_DOMAIN_CPU);
798 		break;
799 	default:
800 		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0);
801 		break;
802 	}
803 
804 	*pl = nvbo->placement;
805 }
806 
807 static int
808 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
809 		     struct ttm_resource *reg)
810 {
811 	struct nouveau_mem *old_mem = nouveau_mem(bo->resource);
812 	struct nouveau_mem *new_mem = nouveau_mem(reg);
813 	struct nvif_vmm *vmm = &drm->client.vmm.vmm;
814 	int ret;
815 
816 	ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0,
817 			   old_mem->mem.size, &old_mem->vma[0]);
818 	if (ret)
819 		return ret;
820 
821 	ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0,
822 			   new_mem->mem.size, &old_mem->vma[1]);
823 	if (ret)
824 		goto done;
825 
826 	ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]);
827 	if (ret)
828 		goto done;
829 
830 	ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]);
831 done:
832 	if (ret) {
833 		nvif_vmm_put(vmm, &old_mem->vma[1]);
834 		nvif_vmm_put(vmm, &old_mem->vma[0]);
835 	}
836 	return 0;
837 }
838 
839 static int
840 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict,
841 		     struct ttm_operation_ctx *ctx,
842 		     struct ttm_resource *new_reg)
843 {
844 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
845 	struct nouveau_channel *chan = drm->ttm.chan;
846 	struct nouveau_cli *cli = (void *)chan->user.client;
847 	struct nouveau_fence *fence;
848 	int ret;
849 
850 	/* create temporary vmas for the transfer and attach them to the
851 	 * old nvkm_mem node, these will get cleaned up after ttm has
852 	 * destroyed the ttm_resource
853 	 */
854 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
855 		ret = nouveau_bo_move_prep(drm, bo, new_reg);
856 		if (ret)
857 			return ret;
858 	}
859 
860 	if (drm_drv_uses_atomic_modeset(drm->dev))
861 		mutex_lock(&cli->mutex);
862 	else
863 		mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
864 
865 	ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible);
866 	if (ret)
867 		goto out_unlock;
868 
869 	ret = drm->ttm.move(chan, bo, bo->resource, new_reg);
870 	if (ret)
871 		goto out_unlock;
872 
873 	ret = nouveau_fence_new(&fence, chan);
874 	if (ret)
875 		goto out_unlock;
876 
877 	/* TODO: figure out a better solution here
878 	 *
879 	 * wait on the fence here explicitly as going through
880 	 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it.
881 	 *
882 	 * Without this the operation can timeout and we'll fallback to a
883 	 * software copy, which might take several minutes to finish.
884 	 */
885 	nouveau_fence_wait(fence, false, false);
886 	ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false,
887 					new_reg);
888 	nouveau_fence_unref(&fence);
889 
890 out_unlock:
891 	mutex_unlock(&cli->mutex);
892 	return ret;
893 }
894 
895 void
896 nouveau_bo_move_init(struct nouveau_drm *drm)
897 {
898 	static const struct _method_table {
899 		const char *name;
900 		int engine;
901 		s32 oclass;
902 		int (*exec)(struct nouveau_channel *,
903 			    struct ttm_buffer_object *,
904 			    struct ttm_resource *, struct ttm_resource *);
905 		int (*init)(struct nouveau_channel *, u32 handle);
906 	} _methods[] = {
907 		{  "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init },
908 		{  "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init },
909 		{  "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init },
910 		{  "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init },
911 		{  "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init },
912 		{  "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init },
913 		{  "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init },
914 		{  "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init },
915 		{  "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
916 		{  "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
917 		{  "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
918 		{  "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
919 		{  "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
920 		{  "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
921 		{  "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
922 		{  "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
923 		{ "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
924 		{ "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
925 		{  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
926 		{ "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
927 		{  "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
928 		{  "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
929 		{  "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
930 		{},
931 	};
932 	const struct _method_table *mthd = _methods;
933 	const char *name = "CPU";
934 	int ret;
935 
936 	do {
937 		struct nouveau_channel *chan;
938 
939 		if (mthd->engine)
940 			chan = drm->cechan;
941 		else
942 			chan = drm->channel;
943 		if (chan == NULL)
944 			continue;
945 
946 		ret = nvif_object_ctor(&chan->user, "ttmBoMove",
947 				       mthd->oclass | (mthd->engine << 16),
948 				       mthd->oclass, NULL, 0,
949 				       &drm->ttm.copy);
950 		if (ret == 0) {
951 			ret = mthd->init(chan, drm->ttm.copy.handle);
952 			if (ret) {
953 				nvif_object_dtor(&drm->ttm.copy);
954 				continue;
955 			}
956 
957 			drm->ttm.move = mthd->exec;
958 			drm->ttm.chan = chan;
959 			name = mthd->name;
960 			break;
961 		}
962 	} while ((++mthd)->exec);
963 
964 	NV_INFO(drm, "MM: using %s for buffer copies\n", name);
965 }
966 
967 static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo,
968 				 struct ttm_resource *new_reg)
969 {
970 	struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL;
971 	struct nouveau_bo *nvbo = nouveau_bo(bo);
972 	struct nouveau_vma *vma;
973 	long ret;
974 
975 	/* ttm can now (stupidly) pass the driver bos it didn't create... */
976 	if (bo->destroy != nouveau_bo_del_ttm)
977 		return;
978 
979 	nouveau_bo_del_io_reserve_lru(bo);
980 
981 	if (mem && new_reg->mem_type != TTM_PL_SYSTEM &&
982 	    mem->mem.page == nvbo->page) {
983 		list_for_each_entry(vma, &nvbo->vma_list, head) {
984 			nouveau_vma_map(vma, mem);
985 		}
986 		nouveau_uvmm_bo_map_all(nvbo, mem);
987 	} else {
988 		list_for_each_entry(vma, &nvbo->vma_list, head) {
989 			ret = dma_resv_wait_timeout(bo->base.resv,
990 						    DMA_RESV_USAGE_BOOKKEEP,
991 						    false, 15 * HZ);
992 			WARN_ON(ret <= 0);
993 			nouveau_vma_unmap(vma);
994 		}
995 		nouveau_uvmm_bo_unmap_all(nvbo);
996 	}
997 
998 	if (new_reg)
999 		nvbo->offset = (new_reg->start << PAGE_SHIFT);
1000 
1001 }
1002 
1003 static int
1004 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg,
1005 		   struct nouveau_drm_tile **new_tile)
1006 {
1007 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1008 	struct drm_device *dev = drm->dev;
1009 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1010 	u64 offset = new_reg->start << PAGE_SHIFT;
1011 
1012 	*new_tile = NULL;
1013 	if (new_reg->mem_type != TTM_PL_VRAM)
1014 		return 0;
1015 
1016 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1017 		*new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size,
1018 					       nvbo->mode, nvbo->zeta);
1019 	}
1020 
1021 	return 0;
1022 }
1023 
1024 static void
1025 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1026 		      struct nouveau_drm_tile *new_tile,
1027 		      struct nouveau_drm_tile **old_tile)
1028 {
1029 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1030 	struct drm_device *dev = drm->dev;
1031 	struct dma_fence *fence;
1032 	int ret;
1033 
1034 	ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE,
1035 				     &fence);
1036 	if (ret)
1037 		dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE,
1038 				      false, MAX_SCHEDULE_TIMEOUT);
1039 
1040 	nv10_bo_put_tile_region(dev, *old_tile, fence);
1041 	*old_tile = new_tile;
1042 }
1043 
1044 static int
1045 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
1046 		struct ttm_operation_ctx *ctx,
1047 		struct ttm_resource *new_reg,
1048 		struct ttm_place *hop)
1049 {
1050 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1051 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1052 	struct ttm_resource *old_reg = bo->resource;
1053 	struct nouveau_drm_tile *new_tile = NULL;
1054 	int ret = 0;
1055 
1056 
1057 	if (new_reg->mem_type == TTM_PL_TT) {
1058 		ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg);
1059 		if (ret)
1060 			return ret;
1061 	}
1062 
1063 	nouveau_bo_move_ntfy(bo, new_reg);
1064 	ret = ttm_bo_wait_ctx(bo, ctx);
1065 	if (ret)
1066 		goto out_ntfy;
1067 
1068 	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1069 		ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
1070 		if (ret)
1071 			goto out_ntfy;
1072 	}
1073 
1074 	/* Fake bo copy. */
1075 	if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM &&
1076 			 !bo->ttm)) {
1077 		ttm_bo_move_null(bo, new_reg);
1078 		goto out;
1079 	}
1080 
1081 	if (old_reg->mem_type == TTM_PL_SYSTEM &&
1082 	    new_reg->mem_type == TTM_PL_TT) {
1083 		ttm_bo_move_null(bo, new_reg);
1084 		goto out;
1085 	}
1086 
1087 	if (old_reg->mem_type == TTM_PL_TT &&
1088 	    new_reg->mem_type == TTM_PL_SYSTEM) {
1089 		nouveau_ttm_tt_unbind(bo->bdev, bo->ttm);
1090 		ttm_resource_free(bo, &bo->resource);
1091 		ttm_bo_assign_mem(bo, new_reg);
1092 		goto out;
1093 	}
1094 
1095 	/* Hardware assisted copy. */
1096 	if (drm->ttm.move) {
1097 		if ((old_reg->mem_type == TTM_PL_SYSTEM &&
1098 		     new_reg->mem_type == TTM_PL_VRAM) ||
1099 		    (old_reg->mem_type == TTM_PL_VRAM &&
1100 		     new_reg->mem_type == TTM_PL_SYSTEM)) {
1101 			hop->fpfn = 0;
1102 			hop->lpfn = 0;
1103 			hop->mem_type = TTM_PL_TT;
1104 			hop->flags = 0;
1105 			return -EMULTIHOP;
1106 		}
1107 		ret = nouveau_bo_move_m2mf(bo, evict, ctx,
1108 					   new_reg);
1109 	} else
1110 		ret = -ENODEV;
1111 
1112 	if (ret) {
1113 		/* Fallback to software copy. */
1114 		ret = ttm_bo_move_memcpy(bo, ctx, new_reg);
1115 	}
1116 
1117 out:
1118 	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1119 		if (ret)
1120 			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1121 		else
1122 			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1123 	}
1124 out_ntfy:
1125 	if (ret) {
1126 		nouveau_bo_move_ntfy(bo, bo->resource);
1127 	}
1128 	return ret;
1129 }
1130 
1131 static void
1132 nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm,
1133 			       struct ttm_resource *reg)
1134 {
1135 	struct nouveau_mem *mem = nouveau_mem(reg);
1136 
1137 	if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
1138 		switch (reg->mem_type) {
1139 		case TTM_PL_TT:
1140 			if (mem->kind)
1141 				nvif_object_unmap_handle(&mem->mem.object);
1142 			break;
1143 		case TTM_PL_VRAM:
1144 			nvif_object_unmap_handle(&mem->mem.object);
1145 			break;
1146 		default:
1147 			break;
1148 		}
1149 	}
1150 }
1151 
1152 static int
1153 nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg)
1154 {
1155 	struct nouveau_drm *drm = nouveau_bdev(bdev);
1156 	struct nvkm_device *device = nvxx_device(&drm->client.device);
1157 	struct nouveau_mem *mem = nouveau_mem(reg);
1158 	struct nvif_mmu *mmu = &drm->client.mmu;
1159 	int ret;
1160 
1161 	mutex_lock(&drm->ttm.io_reserve_mutex);
1162 retry:
1163 	switch (reg->mem_type) {
1164 	case TTM_PL_SYSTEM:
1165 		/* System memory */
1166 		ret = 0;
1167 		goto out;
1168 	case TTM_PL_TT:
1169 #if IS_ENABLED(CONFIG_AGP)
1170 		if (drm->agp.bridge) {
1171 			reg->bus.offset = (reg->start << PAGE_SHIFT) +
1172 				drm->agp.base;
1173 			reg->bus.is_iomem = !drm->agp.cma;
1174 			reg->bus.caching = ttm_write_combined;
1175 		}
1176 #endif
1177 		if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 ||
1178 		    !mem->kind) {
1179 			/* untiled */
1180 			ret = 0;
1181 			break;
1182 		}
1183 		fallthrough;	/* tiled memory */
1184 	case TTM_PL_VRAM:
1185 		reg->bus.offset = (reg->start << PAGE_SHIFT) +
1186 			device->func->resource_addr(device, 1);
1187 		reg->bus.is_iomem = true;
1188 
1189 		/* Some BARs do not support being ioremapped WC */
1190 		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
1191 		    mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED)
1192 			reg->bus.caching = ttm_uncached;
1193 		else
1194 			reg->bus.caching = ttm_write_combined;
1195 
1196 		if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
1197 			union {
1198 				struct nv50_mem_map_v0 nv50;
1199 				struct gf100_mem_map_v0 gf100;
1200 			} args;
1201 			u64 handle, length;
1202 			u32 argc = 0;
1203 
1204 			switch (mem->mem.object.oclass) {
1205 			case NVIF_CLASS_MEM_NV50:
1206 				args.nv50.version = 0;
1207 				args.nv50.ro = 0;
1208 				args.nv50.kind = mem->kind;
1209 				args.nv50.comp = mem->comp;
1210 				argc = sizeof(args.nv50);
1211 				break;
1212 			case NVIF_CLASS_MEM_GF100:
1213 				args.gf100.version = 0;
1214 				args.gf100.ro = 0;
1215 				args.gf100.kind = mem->kind;
1216 				argc = sizeof(args.gf100);
1217 				break;
1218 			default:
1219 				WARN_ON(1);
1220 				break;
1221 			}
1222 
1223 			ret = nvif_object_map_handle(&mem->mem.object,
1224 						     &args, argc,
1225 						     &handle, &length);
1226 			if (ret != 1) {
1227 				if (WARN_ON(ret == 0))
1228 					ret = -EINVAL;
1229 				goto out;
1230 			}
1231 
1232 			reg->bus.offset = handle;
1233 		}
1234 		ret = 0;
1235 		break;
1236 	default:
1237 		ret = -EINVAL;
1238 	}
1239 
1240 out:
1241 	if (ret == -ENOSPC) {
1242 		struct nouveau_bo *nvbo;
1243 
1244 		nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru,
1245 						typeof(*nvbo),
1246 						io_reserve_lru);
1247 		if (nvbo) {
1248 			list_del_init(&nvbo->io_reserve_lru);
1249 			drm_vma_node_unmap(&nvbo->bo.base.vma_node,
1250 					   bdev->dev_mapping);
1251 			nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource);
1252 			nvbo->bo.resource->bus.offset = 0;
1253 			nvbo->bo.resource->bus.addr = NULL;
1254 			goto retry;
1255 		}
1256 
1257 	}
1258 	mutex_unlock(&drm->ttm.io_reserve_mutex);
1259 	return ret;
1260 }
1261 
1262 static void
1263 nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg)
1264 {
1265 	struct nouveau_drm *drm = nouveau_bdev(bdev);
1266 
1267 	mutex_lock(&drm->ttm.io_reserve_mutex);
1268 	nouveau_ttm_io_mem_free_locked(drm, reg);
1269 	mutex_unlock(&drm->ttm.io_reserve_mutex);
1270 }
1271 
1272 vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1273 {
1274 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1275 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1276 	struct nvkm_device *device = nvxx_device(&drm->client.device);
1277 	u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT;
1278 	int i, ret;
1279 
1280 	/* as long as the bo isn't in vram, and isn't tiled, we've got
1281 	 * nothing to do here.
1282 	 */
1283 	if (bo->resource->mem_type != TTM_PL_VRAM) {
1284 		if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1285 		    !nvbo->kind)
1286 			return 0;
1287 
1288 		if (bo->resource->mem_type != TTM_PL_SYSTEM)
1289 			return 0;
1290 
1291 		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0);
1292 
1293 	} else {
1294 		/* make sure bo is in mappable vram */
1295 		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1296 		    bo->resource->start + PFN_UP(bo->resource->size) < mappable)
1297 			return 0;
1298 
1299 		for (i = 0; i < nvbo->placement.num_placement; ++i) {
1300 			nvbo->placements[i].fpfn = 0;
1301 			nvbo->placements[i].lpfn = mappable;
1302 		}
1303 
1304 		for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
1305 			nvbo->busy_placements[i].fpfn = 0;
1306 			nvbo->busy_placements[i].lpfn = mappable;
1307 		}
1308 
1309 		nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0);
1310 	}
1311 
1312 	ret = nouveau_bo_validate(nvbo, false, false);
1313 	if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS))
1314 		return VM_FAULT_NOPAGE;
1315 	else if (unlikely(ret))
1316 		return VM_FAULT_SIGBUS;
1317 
1318 	ttm_bo_move_to_lru_tail_unlocked(bo);
1319 	return 0;
1320 }
1321 
1322 static int
1323 nouveau_ttm_tt_populate(struct ttm_device *bdev,
1324 			struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
1325 {
1326 	struct ttm_tt *ttm_dma = (void *)ttm;
1327 	struct nouveau_drm *drm;
1328 	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
1329 
1330 	if (ttm_tt_is_populated(ttm))
1331 		return 0;
1332 
1333 	if (slave && ttm->sg) {
1334 		drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address,
1335 					       ttm->num_pages);
1336 		return 0;
1337 	}
1338 
1339 	drm = nouveau_bdev(bdev);
1340 
1341 	return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx);
1342 }
1343 
1344 static void
1345 nouveau_ttm_tt_unpopulate(struct ttm_device *bdev,
1346 			  struct ttm_tt *ttm)
1347 {
1348 	struct nouveau_drm *drm;
1349 	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
1350 
1351 	if (slave)
1352 		return;
1353 
1354 	nouveau_ttm_tt_unbind(bdev, ttm);
1355 
1356 	drm = nouveau_bdev(bdev);
1357 
1358 	return ttm_pool_free(&drm->ttm.bdev.pool, ttm);
1359 }
1360 
1361 static void
1362 nouveau_ttm_tt_destroy(struct ttm_device *bdev,
1363 		       struct ttm_tt *ttm)
1364 {
1365 #if IS_ENABLED(CONFIG_AGP)
1366 	struct nouveau_drm *drm = nouveau_bdev(bdev);
1367 	if (drm->agp.bridge) {
1368 		ttm_agp_destroy(ttm);
1369 		return;
1370 	}
1371 #endif
1372 	nouveau_sgdma_destroy(bdev, ttm);
1373 }
1374 
1375 void
1376 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1377 {
1378 	struct dma_resv *resv = nvbo->bo.base.resv;
1379 
1380 	if (!fence)
1381 		return;
1382 
1383 	dma_resv_add_fence(resv, &fence->base, exclusive ?
1384 			   DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ);
1385 }
1386 
1387 static void
1388 nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo)
1389 {
1390 	nouveau_bo_move_ntfy(bo, NULL);
1391 }
1392 
1393 struct ttm_device_funcs nouveau_bo_driver = {
1394 	.ttm_tt_create = &nouveau_ttm_tt_create,
1395 	.ttm_tt_populate = &nouveau_ttm_tt_populate,
1396 	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1397 	.ttm_tt_destroy = &nouveau_ttm_tt_destroy,
1398 	.eviction_valuable = ttm_bo_eviction_valuable,
1399 	.evict_flags = nouveau_bo_evict_flags,
1400 	.delete_mem_notify = nouveau_bo_delete_mem_notify,
1401 	.move = nouveau_bo_move,
1402 	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1403 	.io_mem_free = &nouveau_ttm_io_mem_free,
1404 };
1405