1 /*
2  * Copyright 2007 Dave Airlied
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  */
24 /*
25  * Authors: Dave Airlied <airlied@linux.ie>
26  *	    Ben Skeggs   <darktama@iinet.net.au>
27  *	    Jeremy Kolb  <jkolb@brandeis.edu>
28  */
29 
30 #include <linux/dma-mapping.h>
31 #include <linux/swiotlb.h>
32 
33 #include "nouveau_drm.h"
34 #include "nouveau_dma.h"
35 #include "nouveau_fence.h"
36 
37 #include "nouveau_bo.h"
38 #include "nouveau_ttm.h"
39 #include "nouveau_gem.h"
40 
41 /*
42  * NV10-NV40 tiling helpers
43  */
44 
45 static void
46 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
47 			   u32 addr, u32 size, u32 pitch, u32 flags)
48 {
49 	struct nouveau_drm *drm = nouveau_drm(dev);
50 	int i = reg - drm->tile.reg;
51 	struct nouveau_fb *pfb = nvkm_fb(&drm->device);
52 	struct nouveau_fb_tile *tile = &pfb->tile.region[i];
53 	struct nouveau_engine *engine;
54 
55 	nouveau_fence_unref(&reg->fence);
56 
57 	if (tile->pitch)
58 		pfb->tile.fini(pfb, i, tile);
59 
60 	if (pitch)
61 		pfb->tile.init(pfb, i, addr, size, pitch, flags, tile);
62 
63 	pfb->tile.prog(pfb, i, tile);
64 
65 	if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_GR)))
66 		engine->tile_prog(engine, i);
67 	if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_MPEG)))
68 		engine->tile_prog(engine, i);
69 }
70 
71 static struct nouveau_drm_tile *
72 nv10_bo_get_tile_region(struct drm_device *dev, int i)
73 {
74 	struct nouveau_drm *drm = nouveau_drm(dev);
75 	struct nouveau_drm_tile *tile = &drm->tile.reg[i];
76 
77 	spin_lock(&drm->tile.lock);
78 
79 	if (!tile->used &&
80 	    (!tile->fence || nouveau_fence_done(tile->fence)))
81 		tile->used = true;
82 	else
83 		tile = NULL;
84 
85 	spin_unlock(&drm->tile.lock);
86 	return tile;
87 }
88 
89 static void
90 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
91 			struct fence *fence)
92 {
93 	struct nouveau_drm *drm = nouveau_drm(dev);
94 
95 	if (tile) {
96 		spin_lock(&drm->tile.lock);
97 		tile->fence = (struct nouveau_fence *)fence_get(fence);
98 		tile->used = false;
99 		spin_unlock(&drm->tile.lock);
100 	}
101 }
102 
103 static struct nouveau_drm_tile *
104 nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
105 		   u32 size, u32 pitch, u32 flags)
106 {
107 	struct nouveau_drm *drm = nouveau_drm(dev);
108 	struct nouveau_fb *pfb = nvkm_fb(&drm->device);
109 	struct nouveau_drm_tile *tile, *found = NULL;
110 	int i;
111 
112 	for (i = 0; i < pfb->tile.regions; i++) {
113 		tile = nv10_bo_get_tile_region(dev, i);
114 
115 		if (pitch && !found) {
116 			found = tile;
117 			continue;
118 
119 		} else if (tile && pfb->tile.region[i].pitch) {
120 			/* Kill an unused tile region. */
121 			nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
122 		}
123 
124 		nv10_bo_put_tile_region(dev, tile, NULL);
125 	}
126 
127 	if (found)
128 		nv10_bo_update_tile_region(dev, found, addr, size,
129 					    pitch, flags);
130 	return found;
131 }
132 
133 static void
134 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
135 {
136 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
137 	struct drm_device *dev = drm->dev;
138 	struct nouveau_bo *nvbo = nouveau_bo(bo);
139 
140 	if (unlikely(nvbo->gem.filp))
141 		DRM_ERROR("bo %p still attached to GEM object\n", bo);
142 	WARN_ON(nvbo->pin_refcnt > 0);
143 	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
144 	kfree(nvbo);
145 }
146 
147 static void
148 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
149 		       int *align, int *size)
150 {
151 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
152 	struct nvif_device *device = &drm->device;
153 
154 	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
155 		if (nvbo->tile_mode) {
156 			if (device->info.chipset >= 0x40) {
157 				*align = 65536;
158 				*size = roundup(*size, 64 * nvbo->tile_mode);
159 
160 			} else if (device->info.chipset >= 0x30) {
161 				*align = 32768;
162 				*size = roundup(*size, 64 * nvbo->tile_mode);
163 
164 			} else if (device->info.chipset >= 0x20) {
165 				*align = 16384;
166 				*size = roundup(*size, 64 * nvbo->tile_mode);
167 
168 			} else if (device->info.chipset >= 0x10) {
169 				*align = 16384;
170 				*size = roundup(*size, 32 * nvbo->tile_mode);
171 			}
172 		}
173 	} else {
174 		*size = roundup(*size, (1 << nvbo->page_shift));
175 		*align = max((1 <<  nvbo->page_shift), *align);
176 	}
177 
178 	*size = roundup(*size, PAGE_SIZE);
179 }
180 
181 int
182 nouveau_bo_new(struct drm_device *dev, int size, int align,
183 	       uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
184 	       struct sg_table *sg, struct reservation_object *robj,
185 	       struct nouveau_bo **pnvbo)
186 {
187 	struct nouveau_drm *drm = nouveau_drm(dev);
188 	struct nouveau_bo *nvbo;
189 	size_t acc_size;
190 	int ret;
191 	int type = ttm_bo_type_device;
192 	int lpg_shift = 12;
193 	int max_size;
194 
195 	if (drm->client.vm)
196 		lpg_shift = drm->client.vm->vmm->lpg_shift;
197 	max_size = INT_MAX & ~((1 << lpg_shift) - 1);
198 
199 	if (size <= 0 || size > max_size) {
200 		NV_WARN(drm, "skipped size %x\n", (u32)size);
201 		return -EINVAL;
202 	}
203 
204 	if (sg)
205 		type = ttm_bo_type_sg;
206 
207 	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
208 	if (!nvbo)
209 		return -ENOMEM;
210 	INIT_LIST_HEAD(&nvbo->head);
211 	INIT_LIST_HEAD(&nvbo->entry);
212 	INIT_LIST_HEAD(&nvbo->vma_list);
213 	nvbo->tile_mode = tile_mode;
214 	nvbo->tile_flags = tile_flags;
215 	nvbo->bo.bdev = &drm->ttm.bdev;
216 
217 	if (!nv_device_is_cpu_coherent(nvkm_device(&drm->device)))
218 		nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
219 
220 	nvbo->page_shift = 12;
221 	if (drm->client.vm) {
222 		if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
223 			nvbo->page_shift = drm->client.vm->vmm->lpg_shift;
224 	}
225 
226 	nouveau_bo_fixup_align(nvbo, flags, &align, &size);
227 	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
228 	nouveau_bo_placement_set(nvbo, flags, 0);
229 
230 	acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
231 				       sizeof(struct nouveau_bo));
232 
233 	ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
234 			  type, &nvbo->placement,
235 			  align >> PAGE_SHIFT, false, NULL, acc_size, sg,
236 			  robj, nouveau_bo_del_ttm);
237 	if (ret) {
238 		/* ttm will call nouveau_bo_del_ttm if it fails.. */
239 		return ret;
240 	}
241 
242 	*pnvbo = nvbo;
243 	return 0;
244 }
245 
246 static void
247 set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags)
248 {
249 	*n = 0;
250 
251 	if (type & TTM_PL_FLAG_VRAM)
252 		pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags;
253 	if (type & TTM_PL_FLAG_TT)
254 		pl[(*n)++].flags = TTM_PL_FLAG_TT | flags;
255 	if (type & TTM_PL_FLAG_SYSTEM)
256 		pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags;
257 }
258 
259 static void
260 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
261 {
262 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
263 	u32 vram_pages = drm->device.info.ram_size >> PAGE_SHIFT;
264 	unsigned i, fpfn, lpfn;
265 
266 	if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
267 	    nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
268 	    nvbo->bo.mem.num_pages < vram_pages / 4) {
269 		/*
270 		 * Make sure that the color and depth buffers are handled
271 		 * by independent memory controller units. Up to a 9x
272 		 * speed up when alpha-blending and depth-test are enabled
273 		 * at the same time.
274 		 */
275 		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
276 			fpfn = vram_pages / 2;
277 			lpfn = ~0;
278 		} else {
279 			fpfn = 0;
280 			lpfn = vram_pages / 2;
281 		}
282 		for (i = 0; i < nvbo->placement.num_placement; ++i) {
283 			nvbo->placements[i].fpfn = fpfn;
284 			nvbo->placements[i].lpfn = lpfn;
285 		}
286 		for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
287 			nvbo->busy_placements[i].fpfn = fpfn;
288 			nvbo->busy_placements[i].lpfn = lpfn;
289 		}
290 	}
291 }
292 
293 void
294 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
295 {
296 	struct ttm_placement *pl = &nvbo->placement;
297 	uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
298 						 TTM_PL_MASK_CACHING) |
299 			 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
300 
301 	pl->placement = nvbo->placements;
302 	set_placement_list(nvbo->placements, &pl->num_placement,
303 			   type, flags);
304 
305 	pl->busy_placement = nvbo->busy_placements;
306 	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
307 			   type | busy, flags);
308 
309 	set_placement_range(nvbo, type);
310 }
311 
312 int
313 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)
314 {
315 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
316 	struct ttm_buffer_object *bo = &nvbo->bo;
317 	bool force = false, evict = false;
318 	int ret;
319 
320 	ret = ttm_bo_reserve(bo, false, false, false, NULL);
321 	if (ret)
322 		return ret;
323 
324 	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
325 	    memtype == TTM_PL_FLAG_VRAM && contig) {
326 		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
327 			if (bo->mem.mem_type == TTM_PL_VRAM) {
328 				struct nouveau_mem *mem = bo->mem.mm_node;
329 				if (!list_is_singular(&mem->regions))
330 					evict = true;
331 			}
332 			nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG;
333 			force = true;
334 		}
335 	}
336 
337 	if (nvbo->pin_refcnt) {
338 		if (!(memtype & (1 << bo->mem.mem_type)) || evict) {
339 			NV_ERROR(drm, "bo %p pinned elsewhere: "
340 				      "0x%08x vs 0x%08x\n", bo,
341 				 1 << bo->mem.mem_type, memtype);
342 			ret = -EBUSY;
343 		}
344 		nvbo->pin_refcnt++;
345 		goto out;
346 	}
347 
348 	if (evict) {
349 		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);
350 		ret = nouveau_bo_validate(nvbo, false, false);
351 		if (ret)
352 			goto out;
353 	}
354 
355 	nvbo->pin_refcnt++;
356 	nouveau_bo_placement_set(nvbo, memtype, 0);
357 
358 	/* drop pin_refcnt temporarily, so we don't trip the assertion
359 	 * in nouveau_bo_move() that makes sure we're not trying to
360 	 * move a pinned buffer
361 	 */
362 	nvbo->pin_refcnt--;
363 	ret = nouveau_bo_validate(nvbo, false, false);
364 	if (ret)
365 		goto out;
366 	nvbo->pin_refcnt++;
367 
368 	switch (bo->mem.mem_type) {
369 	case TTM_PL_VRAM:
370 		drm->gem.vram_available -= bo->mem.size;
371 		break;
372 	case TTM_PL_TT:
373 		drm->gem.gart_available -= bo->mem.size;
374 		break;
375 	default:
376 		break;
377 	}
378 
379 out:
380 	if (force && ret)
381 		nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG;
382 	ttm_bo_unreserve(bo);
383 	return ret;
384 }
385 
386 int
387 nouveau_bo_unpin(struct nouveau_bo *nvbo)
388 {
389 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
390 	struct ttm_buffer_object *bo = &nvbo->bo;
391 	int ret, ref;
392 
393 	ret = ttm_bo_reserve(bo, false, false, false, NULL);
394 	if (ret)
395 		return ret;
396 
397 	ref = --nvbo->pin_refcnt;
398 	WARN_ON_ONCE(ref < 0);
399 	if (ref)
400 		goto out;
401 
402 	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
403 
404 	ret = nouveau_bo_validate(nvbo, false, false);
405 	if (ret == 0) {
406 		switch (bo->mem.mem_type) {
407 		case TTM_PL_VRAM:
408 			drm->gem.vram_available += bo->mem.size;
409 			break;
410 		case TTM_PL_TT:
411 			drm->gem.gart_available += bo->mem.size;
412 			break;
413 		default:
414 			break;
415 		}
416 	}
417 
418 out:
419 	ttm_bo_unreserve(bo);
420 	return ret;
421 }
422 
423 int
424 nouveau_bo_map(struct nouveau_bo *nvbo)
425 {
426 	int ret;
427 
428 	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
429 	if (ret)
430 		return ret;
431 
432 	/*
433 	 * TTM buffers allocated using the DMA API already have a mapping, let's
434 	 * use it instead.
435 	 */
436 	if (!nvbo->force_coherent)
437 		ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
438 				  &nvbo->kmap);
439 
440 	ttm_bo_unreserve(&nvbo->bo);
441 	return ret;
442 }
443 
444 void
445 nouveau_bo_unmap(struct nouveau_bo *nvbo)
446 {
447 	if (!nvbo)
448 		return;
449 
450 	/*
451 	 * TTM buffers allocated using the DMA API already had a coherent
452 	 * mapping which we used, no need to unmap.
453 	 */
454 	if (!nvbo->force_coherent)
455 		ttm_bo_kunmap(&nvbo->kmap);
456 }
457 
458 void
459 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
460 {
461 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
462 	struct nouveau_device *device = nvkm_device(&drm->device);
463 	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
464 	int i;
465 
466 	if (!ttm_dma)
467 		return;
468 
469 	/* Don't waste time looping if the object is coherent */
470 	if (nvbo->force_coherent)
471 		return;
472 
473 	for (i = 0; i < ttm_dma->ttm.num_pages; i++)
474 		dma_sync_single_for_device(nv_device_base(device),
475 			ttm_dma->dma_address[i], PAGE_SIZE, DMA_TO_DEVICE);
476 }
477 
478 void
479 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
480 {
481 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
482 	struct nouveau_device *device = nvkm_device(&drm->device);
483 	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
484 	int i;
485 
486 	if (!ttm_dma)
487 		return;
488 
489 	/* Don't waste time looping if the object is coherent */
490 	if (nvbo->force_coherent)
491 		return;
492 
493 	for (i = 0; i < ttm_dma->ttm.num_pages; i++)
494 		dma_sync_single_for_cpu(nv_device_base(device),
495 			ttm_dma->dma_address[i], PAGE_SIZE, DMA_FROM_DEVICE);
496 }
497 
498 int
499 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
500 		    bool no_wait_gpu)
501 {
502 	int ret;
503 
504 	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
505 			      interruptible, no_wait_gpu);
506 	if (ret)
507 		return ret;
508 
509 	nouveau_bo_sync_for_device(nvbo);
510 
511 	return 0;
512 }
513 
514 static inline void *
515 _nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz)
516 {
517 	struct ttm_dma_tt *dma_tt;
518 	u8 *m = mem;
519 
520 	index *= sz;
521 
522 	if (m) {
523 		/* kmap'd address, return the corresponding offset */
524 		m += index;
525 	} else {
526 		/* DMA-API mapping, lookup the right address */
527 		dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm;
528 		m = dma_tt->cpu_address[index / PAGE_SIZE];
529 		m += index % PAGE_SIZE;
530 	}
531 
532 	return m;
533 }
534 #define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m))
535 
536 u16
537 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
538 {
539 	bool is_iomem;
540 	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
541 
542 	mem = nouveau_bo_mem_index(nvbo, index, mem);
543 
544 	if (is_iomem)
545 		return ioread16_native((void __force __iomem *)mem);
546 	else
547 		return *mem;
548 }
549 
550 void
551 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
552 {
553 	bool is_iomem;
554 	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
555 
556 	mem = nouveau_bo_mem_index(nvbo, index, mem);
557 
558 	if (is_iomem)
559 		iowrite16_native(val, (void __force __iomem *)mem);
560 	else
561 		*mem = val;
562 }
563 
564 u32
565 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
566 {
567 	bool is_iomem;
568 	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
569 
570 	mem = nouveau_bo_mem_index(nvbo, index, mem);
571 
572 	if (is_iomem)
573 		return ioread32_native((void __force __iomem *)mem);
574 	else
575 		return *mem;
576 }
577 
578 void
579 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
580 {
581 	bool is_iomem;
582 	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
583 
584 	mem = nouveau_bo_mem_index(nvbo, index, mem);
585 
586 	if (is_iomem)
587 		iowrite32_native(val, (void __force __iomem *)mem);
588 	else
589 		*mem = val;
590 }
591 
592 static struct ttm_tt *
593 nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
594 		      uint32_t page_flags, struct page *dummy_read)
595 {
596 #if __OS_HAS_AGP
597 	struct nouveau_drm *drm = nouveau_bdev(bdev);
598 	struct drm_device *dev = drm->dev;
599 
600 	if (drm->agp.stat == ENABLED) {
601 		return ttm_agp_tt_create(bdev, dev->agp->bridge, size,
602 					 page_flags, dummy_read);
603 	}
604 #endif
605 
606 	return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
607 }
608 
609 static int
610 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
611 {
612 	/* We'll do this from user space. */
613 	return 0;
614 }
615 
616 static int
617 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
618 			 struct ttm_mem_type_manager *man)
619 {
620 	struct nouveau_drm *drm = nouveau_bdev(bdev);
621 
622 	switch (type) {
623 	case TTM_PL_SYSTEM:
624 		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
625 		man->available_caching = TTM_PL_MASK_CACHING;
626 		man->default_caching = TTM_PL_FLAG_CACHED;
627 		break;
628 	case TTM_PL_VRAM:
629 		man->flags = TTM_MEMTYPE_FLAG_FIXED |
630 			     TTM_MEMTYPE_FLAG_MAPPABLE;
631 		man->available_caching = TTM_PL_FLAG_UNCACHED |
632 					 TTM_PL_FLAG_WC;
633 		man->default_caching = TTM_PL_FLAG_WC;
634 
635 		if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
636 			/* Some BARs do not support being ioremapped WC */
637 			if (nvkm_bar(&drm->device)->iomap_uncached) {
638 				man->available_caching = TTM_PL_FLAG_UNCACHED;
639 				man->default_caching = TTM_PL_FLAG_UNCACHED;
640 			}
641 
642 			man->func = &nouveau_vram_manager;
643 			man->io_reserve_fastpath = false;
644 			man->use_io_reserve_lru = true;
645 		} else {
646 			man->func = &ttm_bo_manager_func;
647 		}
648 		break;
649 	case TTM_PL_TT:
650 		if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
651 			man->func = &nouveau_gart_manager;
652 		else
653 		if (drm->agp.stat != ENABLED)
654 			man->func = &nv04_gart_manager;
655 		else
656 			man->func = &ttm_bo_manager_func;
657 
658 		if (drm->agp.stat == ENABLED) {
659 			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
660 			man->available_caching = TTM_PL_FLAG_UNCACHED |
661 				TTM_PL_FLAG_WC;
662 			man->default_caching = TTM_PL_FLAG_WC;
663 		} else {
664 			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
665 				     TTM_MEMTYPE_FLAG_CMA;
666 			man->available_caching = TTM_PL_MASK_CACHING;
667 			man->default_caching = TTM_PL_FLAG_CACHED;
668 		}
669 
670 		break;
671 	default:
672 		return -EINVAL;
673 	}
674 	return 0;
675 }
676 
677 static void
678 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
679 {
680 	struct nouveau_bo *nvbo = nouveau_bo(bo);
681 
682 	switch (bo->mem.mem_type) {
683 	case TTM_PL_VRAM:
684 		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
685 					 TTM_PL_FLAG_SYSTEM);
686 		break;
687 	default:
688 		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
689 		break;
690 	}
691 
692 	*pl = nvbo->placement;
693 }
694 
695 
696 static int
697 nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
698 {
699 	int ret = RING_SPACE(chan, 2);
700 	if (ret == 0) {
701 		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
702 		OUT_RING  (chan, handle & 0x0000ffff);
703 		FIRE_RING (chan);
704 	}
705 	return ret;
706 }
707 
708 static int
709 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
710 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
711 {
712 	struct nouveau_mem *node = old_mem->mm_node;
713 	int ret = RING_SPACE(chan, 10);
714 	if (ret == 0) {
715 		BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
716 		OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
717 		OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
718 		OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
719 		OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
720 		OUT_RING  (chan, PAGE_SIZE);
721 		OUT_RING  (chan, PAGE_SIZE);
722 		OUT_RING  (chan, PAGE_SIZE);
723 		OUT_RING  (chan, new_mem->num_pages);
724 		BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
725 	}
726 	return ret;
727 }
728 
729 static int
730 nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
731 {
732 	int ret = RING_SPACE(chan, 2);
733 	if (ret == 0) {
734 		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
735 		OUT_RING  (chan, handle);
736 	}
737 	return ret;
738 }
739 
740 static int
741 nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
742 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
743 {
744 	struct nouveau_mem *node = old_mem->mm_node;
745 	u64 src_offset = node->vma[0].offset;
746 	u64 dst_offset = node->vma[1].offset;
747 	u32 page_count = new_mem->num_pages;
748 	int ret;
749 
750 	page_count = new_mem->num_pages;
751 	while (page_count) {
752 		int line_count = (page_count > 8191) ? 8191 : page_count;
753 
754 		ret = RING_SPACE(chan, 11);
755 		if (ret)
756 			return ret;
757 
758 		BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
759 		OUT_RING  (chan, upper_32_bits(src_offset));
760 		OUT_RING  (chan, lower_32_bits(src_offset));
761 		OUT_RING  (chan, upper_32_bits(dst_offset));
762 		OUT_RING  (chan, lower_32_bits(dst_offset));
763 		OUT_RING  (chan, PAGE_SIZE);
764 		OUT_RING  (chan, PAGE_SIZE);
765 		OUT_RING  (chan, PAGE_SIZE);
766 		OUT_RING  (chan, line_count);
767 		BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
768 		OUT_RING  (chan, 0x00000110);
769 
770 		page_count -= line_count;
771 		src_offset += (PAGE_SIZE * line_count);
772 		dst_offset += (PAGE_SIZE * line_count);
773 	}
774 
775 	return 0;
776 }
777 
778 static int
779 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
780 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
781 {
782 	struct nouveau_mem *node = old_mem->mm_node;
783 	u64 src_offset = node->vma[0].offset;
784 	u64 dst_offset = node->vma[1].offset;
785 	u32 page_count = new_mem->num_pages;
786 	int ret;
787 
788 	page_count = new_mem->num_pages;
789 	while (page_count) {
790 		int line_count = (page_count > 2047) ? 2047 : page_count;
791 
792 		ret = RING_SPACE(chan, 12);
793 		if (ret)
794 			return ret;
795 
796 		BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
797 		OUT_RING  (chan, upper_32_bits(dst_offset));
798 		OUT_RING  (chan, lower_32_bits(dst_offset));
799 		BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
800 		OUT_RING  (chan, upper_32_bits(src_offset));
801 		OUT_RING  (chan, lower_32_bits(src_offset));
802 		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
803 		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
804 		OUT_RING  (chan, PAGE_SIZE); /* line_length */
805 		OUT_RING  (chan, line_count);
806 		BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
807 		OUT_RING  (chan, 0x00100110);
808 
809 		page_count -= line_count;
810 		src_offset += (PAGE_SIZE * line_count);
811 		dst_offset += (PAGE_SIZE * line_count);
812 	}
813 
814 	return 0;
815 }
816 
817 static int
818 nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
819 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
820 {
821 	struct nouveau_mem *node = old_mem->mm_node;
822 	u64 src_offset = node->vma[0].offset;
823 	u64 dst_offset = node->vma[1].offset;
824 	u32 page_count = new_mem->num_pages;
825 	int ret;
826 
827 	page_count = new_mem->num_pages;
828 	while (page_count) {
829 		int line_count = (page_count > 8191) ? 8191 : page_count;
830 
831 		ret = RING_SPACE(chan, 11);
832 		if (ret)
833 			return ret;
834 
835 		BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
836 		OUT_RING  (chan, upper_32_bits(src_offset));
837 		OUT_RING  (chan, lower_32_bits(src_offset));
838 		OUT_RING  (chan, upper_32_bits(dst_offset));
839 		OUT_RING  (chan, lower_32_bits(dst_offset));
840 		OUT_RING  (chan, PAGE_SIZE);
841 		OUT_RING  (chan, PAGE_SIZE);
842 		OUT_RING  (chan, PAGE_SIZE);
843 		OUT_RING  (chan, line_count);
844 		BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
845 		OUT_RING  (chan, 0x00000110);
846 
847 		page_count -= line_count;
848 		src_offset += (PAGE_SIZE * line_count);
849 		dst_offset += (PAGE_SIZE * line_count);
850 	}
851 
852 	return 0;
853 }
854 
855 static int
856 nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
857 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
858 {
859 	struct nouveau_mem *node = old_mem->mm_node;
860 	int ret = RING_SPACE(chan, 7);
861 	if (ret == 0) {
862 		BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
863 		OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
864 		OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
865 		OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
866 		OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
867 		OUT_RING  (chan, 0x00000000 /* COPY */);
868 		OUT_RING  (chan, new_mem->num_pages << PAGE_SHIFT);
869 	}
870 	return ret;
871 }
872 
873 static int
874 nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
875 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
876 {
877 	struct nouveau_mem *node = old_mem->mm_node;
878 	int ret = RING_SPACE(chan, 7);
879 	if (ret == 0) {
880 		BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
881 		OUT_RING  (chan, new_mem->num_pages << PAGE_SHIFT);
882 		OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
883 		OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
884 		OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
885 		OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
886 		OUT_RING  (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
887 	}
888 	return ret;
889 }
890 
891 static int
892 nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
893 {
894 	int ret = RING_SPACE(chan, 6);
895 	if (ret == 0) {
896 		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
897 		OUT_RING  (chan, handle);
898 		BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
899 		OUT_RING  (chan, chan->drm->ntfy.handle);
900 		OUT_RING  (chan, chan->vram.handle);
901 		OUT_RING  (chan, chan->vram.handle);
902 	}
903 
904 	return ret;
905 }
906 
907 static int
908 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
909 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
910 {
911 	struct nouveau_mem *node = old_mem->mm_node;
912 	u64 length = (new_mem->num_pages << PAGE_SHIFT);
913 	u64 src_offset = node->vma[0].offset;
914 	u64 dst_offset = node->vma[1].offset;
915 	int src_tiled = !!node->memtype;
916 	int dst_tiled = !!((struct nouveau_mem *)new_mem->mm_node)->memtype;
917 	int ret;
918 
919 	while (length) {
920 		u32 amount, stride, height;
921 
922 		ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
923 		if (ret)
924 			return ret;
925 
926 		amount  = min(length, (u64)(4 * 1024 * 1024));
927 		stride  = 16 * 4;
928 		height  = amount / stride;
929 
930 		if (src_tiled) {
931 			BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
932 			OUT_RING  (chan, 0);
933 			OUT_RING  (chan, 0);
934 			OUT_RING  (chan, stride);
935 			OUT_RING  (chan, height);
936 			OUT_RING  (chan, 1);
937 			OUT_RING  (chan, 0);
938 			OUT_RING  (chan, 0);
939 		} else {
940 			BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
941 			OUT_RING  (chan, 1);
942 		}
943 		if (dst_tiled) {
944 			BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
945 			OUT_RING  (chan, 0);
946 			OUT_RING  (chan, 0);
947 			OUT_RING  (chan, stride);
948 			OUT_RING  (chan, height);
949 			OUT_RING  (chan, 1);
950 			OUT_RING  (chan, 0);
951 			OUT_RING  (chan, 0);
952 		} else {
953 			BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
954 			OUT_RING  (chan, 1);
955 		}
956 
957 		BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
958 		OUT_RING  (chan, upper_32_bits(src_offset));
959 		OUT_RING  (chan, upper_32_bits(dst_offset));
960 		BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
961 		OUT_RING  (chan, lower_32_bits(src_offset));
962 		OUT_RING  (chan, lower_32_bits(dst_offset));
963 		OUT_RING  (chan, stride);
964 		OUT_RING  (chan, stride);
965 		OUT_RING  (chan, stride);
966 		OUT_RING  (chan, height);
967 		OUT_RING  (chan, 0x00000101);
968 		OUT_RING  (chan, 0x00000000);
969 		BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
970 		OUT_RING  (chan, 0);
971 
972 		length -= amount;
973 		src_offset += amount;
974 		dst_offset += amount;
975 	}
976 
977 	return 0;
978 }
979 
980 static int
981 nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
982 {
983 	int ret = RING_SPACE(chan, 4);
984 	if (ret == 0) {
985 		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
986 		OUT_RING  (chan, handle);
987 		BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
988 		OUT_RING  (chan, chan->drm->ntfy.handle);
989 	}
990 
991 	return ret;
992 }
993 
994 static inline uint32_t
995 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
996 		      struct nouveau_channel *chan, struct ttm_mem_reg *mem)
997 {
998 	if (mem->mem_type == TTM_PL_TT)
999 		return NvDmaTT;
1000 	return chan->vram.handle;
1001 }
1002 
1003 static int
1004 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
1005 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
1006 {
1007 	u32 src_offset = old_mem->start << PAGE_SHIFT;
1008 	u32 dst_offset = new_mem->start << PAGE_SHIFT;
1009 	u32 page_count = new_mem->num_pages;
1010 	int ret;
1011 
1012 	ret = RING_SPACE(chan, 3);
1013 	if (ret)
1014 		return ret;
1015 
1016 	BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
1017 	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
1018 	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
1019 
1020 	page_count = new_mem->num_pages;
1021 	while (page_count) {
1022 		int line_count = (page_count > 2047) ? 2047 : page_count;
1023 
1024 		ret = RING_SPACE(chan, 11);
1025 		if (ret)
1026 			return ret;
1027 
1028 		BEGIN_NV04(chan, NvSubCopy,
1029 				 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
1030 		OUT_RING  (chan, src_offset);
1031 		OUT_RING  (chan, dst_offset);
1032 		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
1033 		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
1034 		OUT_RING  (chan, PAGE_SIZE); /* line_length */
1035 		OUT_RING  (chan, line_count);
1036 		OUT_RING  (chan, 0x00000101);
1037 		OUT_RING  (chan, 0x00000000);
1038 		BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
1039 		OUT_RING  (chan, 0);
1040 
1041 		page_count -= line_count;
1042 		src_offset += (PAGE_SIZE * line_count);
1043 		dst_offset += (PAGE_SIZE * line_count);
1044 	}
1045 
1046 	return 0;
1047 }
1048 
1049 static int
1050 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
1051 		     struct ttm_mem_reg *mem)
1052 {
1053 	struct nouveau_mem *old_node = bo->mem.mm_node;
1054 	struct nouveau_mem *new_node = mem->mm_node;
1055 	u64 size = (u64)mem->num_pages << PAGE_SHIFT;
1056 	int ret;
1057 
1058 	ret = nouveau_vm_get(drm->client.vm, size, old_node->page_shift,
1059 			     NV_MEM_ACCESS_RW, &old_node->vma[0]);
1060 	if (ret)
1061 		return ret;
1062 
1063 	ret = nouveau_vm_get(drm->client.vm, size, new_node->page_shift,
1064 			     NV_MEM_ACCESS_RW, &old_node->vma[1]);
1065 	if (ret) {
1066 		nouveau_vm_put(&old_node->vma[0]);
1067 		return ret;
1068 	}
1069 
1070 	nouveau_vm_map(&old_node->vma[0], old_node);
1071 	nouveau_vm_map(&old_node->vma[1], new_node);
1072 	return 0;
1073 }
1074 
1075 static int
1076 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
1077 		     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1078 {
1079 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1080 	struct nouveau_channel *chan = drm->ttm.chan;
1081 	struct nouveau_cli *cli = (void *)nvif_client(&chan->device->base);
1082 	struct nouveau_fence *fence;
1083 	int ret;
1084 
1085 	/* create temporary vmas for the transfer and attach them to the
1086 	 * old nouveau_mem node, these will get cleaned up after ttm has
1087 	 * destroyed the ttm_mem_reg
1088 	 */
1089 	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1090 		ret = nouveau_bo_move_prep(drm, bo, new_mem);
1091 		if (ret)
1092 			return ret;
1093 	}
1094 
1095 	mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
1096 	ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
1097 	if (ret == 0) {
1098 		ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);
1099 		if (ret == 0) {
1100 			ret = nouveau_fence_new(chan, false, &fence);
1101 			if (ret == 0) {
1102 				ret = ttm_bo_move_accel_cleanup(bo,
1103 								&fence->base,
1104 								evict,
1105 								no_wait_gpu,
1106 								new_mem);
1107 				nouveau_fence_unref(&fence);
1108 			}
1109 		}
1110 	}
1111 	mutex_unlock(&cli->mutex);
1112 	return ret;
1113 }
1114 
1115 void
1116 nouveau_bo_move_init(struct nouveau_drm *drm)
1117 {
1118 	static const struct {
1119 		const char *name;
1120 		int engine;
1121 		u32 oclass;
1122 		int (*exec)(struct nouveau_channel *,
1123 			    struct ttm_buffer_object *,
1124 			    struct ttm_mem_reg *, struct ttm_mem_reg *);
1125 		int (*init)(struct nouveau_channel *, u32 handle);
1126 	} _methods[] = {
1127 		{  "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1128 		{  "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1129 		{ "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
1130 		{ "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
1131 		{  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
1132 		{ "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
1133 		{  "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
1134 		{  "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
1135 		{  "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1136 		{},
1137 		{ "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1138 	}, *mthd = _methods;
1139 	const char *name = "CPU";
1140 	int ret;
1141 
1142 	do {
1143 		struct nouveau_channel *chan;
1144 
1145 		if (mthd->engine)
1146 			chan = drm->cechan;
1147 		else
1148 			chan = drm->channel;
1149 		if (chan == NULL)
1150 			continue;
1151 
1152 		ret = nvif_object_init(chan->object, NULL,
1153 				       mthd->oclass | (mthd->engine << 16),
1154 				       mthd->oclass, NULL, 0,
1155 				       &drm->ttm.copy);
1156 		if (ret == 0) {
1157 			ret = mthd->init(chan, drm->ttm.copy.handle);
1158 			if (ret) {
1159 				nvif_object_fini(&drm->ttm.copy);
1160 				continue;
1161 			}
1162 
1163 			drm->ttm.move = mthd->exec;
1164 			drm->ttm.chan = chan;
1165 			name = mthd->name;
1166 			break;
1167 		}
1168 	} while ((++mthd)->exec);
1169 
1170 	NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1171 }
1172 
1173 static int
1174 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1175 		      bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1176 {
1177 	struct ttm_place placement_memtype = {
1178 		.fpfn = 0,
1179 		.lpfn = 0,
1180 		.flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1181 	};
1182 	struct ttm_placement placement;
1183 	struct ttm_mem_reg tmp_mem;
1184 	int ret;
1185 
1186 	placement.num_placement = placement.num_busy_placement = 1;
1187 	placement.placement = placement.busy_placement = &placement_memtype;
1188 
1189 	tmp_mem = *new_mem;
1190 	tmp_mem.mm_node = NULL;
1191 	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1192 	if (ret)
1193 		return ret;
1194 
1195 	ret = ttm_tt_bind(bo->ttm, &tmp_mem);
1196 	if (ret)
1197 		goto out;
1198 
1199 	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_mem);
1200 	if (ret)
1201 		goto out;
1202 
1203 	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
1204 out:
1205 	ttm_bo_mem_put(bo, &tmp_mem);
1206 	return ret;
1207 }
1208 
1209 static int
1210 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1211 		      bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1212 {
1213 	struct ttm_place placement_memtype = {
1214 		.fpfn = 0,
1215 		.lpfn = 0,
1216 		.flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
1217 	};
1218 	struct ttm_placement placement;
1219 	struct ttm_mem_reg tmp_mem;
1220 	int ret;
1221 
1222 	placement.num_placement = placement.num_busy_placement = 1;
1223 	placement.placement = placement.busy_placement = &placement_memtype;
1224 
1225 	tmp_mem = *new_mem;
1226 	tmp_mem.mm_node = NULL;
1227 	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1228 	if (ret)
1229 		return ret;
1230 
1231 	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
1232 	if (ret)
1233 		goto out;
1234 
1235 	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_mem);
1236 	if (ret)
1237 		goto out;
1238 
1239 out:
1240 	ttm_bo_mem_put(bo, &tmp_mem);
1241 	return ret;
1242 }
1243 
1244 static void
1245 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
1246 {
1247 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1248 	struct nouveau_vma *vma;
1249 
1250 	/* ttm can now (stupidly) pass the driver bos it didn't create... */
1251 	if (bo->destroy != nouveau_bo_del_ttm)
1252 		return;
1253 
1254 	list_for_each_entry(vma, &nvbo->vma_list, head) {
1255 		if (new_mem && new_mem->mem_type != TTM_PL_SYSTEM &&
1256 			      (new_mem->mem_type == TTM_PL_VRAM ||
1257 			       nvbo->page_shift != vma->vm->vmm->lpg_shift)) {
1258 			nouveau_vm_map(vma, new_mem->mm_node);
1259 		} else {
1260 			nouveau_vm_unmap(vma);
1261 		}
1262 	}
1263 }
1264 
1265 static int
1266 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
1267 		   struct nouveau_drm_tile **new_tile)
1268 {
1269 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1270 	struct drm_device *dev = drm->dev;
1271 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1272 	u64 offset = new_mem->start << PAGE_SHIFT;
1273 
1274 	*new_tile = NULL;
1275 	if (new_mem->mem_type != TTM_PL_VRAM)
1276 		return 0;
1277 
1278 	if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1279 		*new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size,
1280 						nvbo->tile_mode,
1281 						nvbo->tile_flags);
1282 	}
1283 
1284 	return 0;
1285 }
1286 
1287 static void
1288 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1289 		      struct nouveau_drm_tile *new_tile,
1290 		      struct nouveau_drm_tile **old_tile)
1291 {
1292 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1293 	struct drm_device *dev = drm->dev;
1294 	struct fence *fence = reservation_object_get_excl(bo->resv);
1295 
1296 	nv10_bo_put_tile_region(dev, *old_tile, fence);
1297 	*old_tile = new_tile;
1298 }
1299 
1300 static int
1301 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1302 		bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1303 {
1304 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1305 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1306 	struct ttm_mem_reg *old_mem = &bo->mem;
1307 	struct nouveau_drm_tile *new_tile = NULL;
1308 	int ret = 0;
1309 
1310 	if (nvbo->pin_refcnt)
1311 		NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
1312 
1313 	if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1314 		ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
1315 		if (ret)
1316 			return ret;
1317 	}
1318 
1319 	/* Fake bo copy. */
1320 	if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1321 		BUG_ON(bo->mem.mm_node != NULL);
1322 		bo->mem = *new_mem;
1323 		new_mem->mm_node = NULL;
1324 		goto out;
1325 	}
1326 
1327 	/* Hardware assisted copy. */
1328 	if (drm->ttm.move) {
1329 		if (new_mem->mem_type == TTM_PL_SYSTEM)
1330 			ret = nouveau_bo_move_flipd(bo, evict, intr,
1331 						    no_wait_gpu, new_mem);
1332 		else if (old_mem->mem_type == TTM_PL_SYSTEM)
1333 			ret = nouveau_bo_move_flips(bo, evict, intr,
1334 						    no_wait_gpu, new_mem);
1335 		else
1336 			ret = nouveau_bo_move_m2mf(bo, evict, intr,
1337 						   no_wait_gpu, new_mem);
1338 		if (!ret)
1339 			goto out;
1340 	}
1341 
1342 	/* Fallback to software copy. */
1343 	ret = ttm_bo_wait(bo, true, intr, no_wait_gpu);
1344 	if (ret == 0)
1345 		ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
1346 
1347 out:
1348 	if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1349 		if (ret)
1350 			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1351 		else
1352 			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1353 	}
1354 
1355 	return ret;
1356 }
1357 
1358 static int
1359 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
1360 {
1361 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1362 
1363 	return drm_vma_node_verify_access(&nvbo->gem.vma_node, filp);
1364 }
1365 
1366 static int
1367 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1368 {
1369 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1370 	struct nouveau_drm *drm = nouveau_bdev(bdev);
1371 	struct nouveau_mem *node = mem->mm_node;
1372 	int ret;
1373 
1374 	mem->bus.addr = NULL;
1375 	mem->bus.offset = 0;
1376 	mem->bus.size = mem->num_pages << PAGE_SHIFT;
1377 	mem->bus.base = 0;
1378 	mem->bus.is_iomem = false;
1379 	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1380 		return -EINVAL;
1381 	switch (mem->mem_type) {
1382 	case TTM_PL_SYSTEM:
1383 		/* System memory */
1384 		return 0;
1385 	case TTM_PL_TT:
1386 #if __OS_HAS_AGP
1387 		if (drm->agp.stat == ENABLED) {
1388 			mem->bus.offset = mem->start << PAGE_SHIFT;
1389 			mem->bus.base = drm->agp.base;
1390 			mem->bus.is_iomem = !drm->dev->agp->cant_use_aperture;
1391 		}
1392 #endif
1393 		if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !node->memtype)
1394 			/* untiled */
1395 			break;
1396 		/* fallthrough, tiled memory */
1397 	case TTM_PL_VRAM:
1398 		mem->bus.offset = mem->start << PAGE_SHIFT;
1399 		mem->bus.base = nv_device_resource_start(nvkm_device(&drm->device), 1);
1400 		mem->bus.is_iomem = true;
1401 		if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1402 			struct nouveau_bar *bar = nvkm_bar(&drm->device);
1403 
1404 			ret = bar->umap(bar, node, NV_MEM_ACCESS_RW,
1405 					&node->bar_vma);
1406 			if (ret)
1407 				return ret;
1408 
1409 			mem->bus.offset = node->bar_vma.offset;
1410 		}
1411 		break;
1412 	default:
1413 		return -EINVAL;
1414 	}
1415 	return 0;
1416 }
1417 
1418 static void
1419 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1420 {
1421 	struct nouveau_drm *drm = nouveau_bdev(bdev);
1422 	struct nouveau_bar *bar = nvkm_bar(&drm->device);
1423 	struct nouveau_mem *node = mem->mm_node;
1424 
1425 	if (!node->bar_vma.node)
1426 		return;
1427 
1428 	bar->unmap(bar, &node->bar_vma);
1429 }
1430 
1431 static int
1432 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1433 {
1434 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1435 	struct nouveau_bo *nvbo = nouveau_bo(bo);
1436 	struct nvif_device *device = &drm->device;
1437 	u32 mappable = nv_device_resource_len(nvkm_device(device), 1) >> PAGE_SHIFT;
1438 	int i, ret;
1439 
1440 	/* as long as the bo isn't in vram, and isn't tiled, we've got
1441 	 * nothing to do here.
1442 	 */
1443 	if (bo->mem.mem_type != TTM_PL_VRAM) {
1444 		if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1445 		    !nouveau_bo_tile_layout(nvbo))
1446 			return 0;
1447 
1448 		if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1449 			nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);
1450 
1451 			ret = nouveau_bo_validate(nvbo, false, false);
1452 			if (ret)
1453 				return ret;
1454 		}
1455 		return 0;
1456 	}
1457 
1458 	/* make sure bo is in mappable vram */
1459 	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1460 	    bo->mem.start + bo->mem.num_pages < mappable)
1461 		return 0;
1462 
1463 	for (i = 0; i < nvbo->placement.num_placement; ++i) {
1464 		nvbo->placements[i].fpfn = 0;
1465 		nvbo->placements[i].lpfn = mappable;
1466 	}
1467 
1468 	for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
1469 		nvbo->busy_placements[i].fpfn = 0;
1470 		nvbo->busy_placements[i].lpfn = mappable;
1471 	}
1472 
1473 	nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1474 	return nouveau_bo_validate(nvbo, false, false);
1475 }
1476 
1477 static int
1478 nouveau_ttm_tt_populate(struct ttm_tt *ttm)
1479 {
1480 	struct ttm_dma_tt *ttm_dma = (void *)ttm;
1481 	struct nouveau_drm *drm;
1482 	struct nouveau_device *device;
1483 	struct drm_device *dev;
1484 	struct device *pdev;
1485 	unsigned i;
1486 	int r;
1487 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1488 
1489 	if (ttm->state != tt_unpopulated)
1490 		return 0;
1491 
1492 	if (slave && ttm->sg) {
1493 		/* make userspace faulting work */
1494 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1495 						 ttm_dma->dma_address, ttm->num_pages);
1496 		ttm->state = tt_unbound;
1497 		return 0;
1498 	}
1499 
1500 	drm = nouveau_bdev(ttm->bdev);
1501 	device = nvkm_device(&drm->device);
1502 	dev = drm->dev;
1503 	pdev = nv_device_base(device);
1504 
1505 	/*
1506 	 * Objects matching this condition have been marked as force_coherent,
1507 	 * so use the DMA API for them.
1508 	 */
1509 	if (!nv_device_is_cpu_coherent(device) &&
1510 	    ttm->caching_state == tt_uncached)
1511 		return ttm_dma_populate(ttm_dma, dev->dev);
1512 
1513 #if __OS_HAS_AGP
1514 	if (drm->agp.stat == ENABLED) {
1515 		return ttm_agp_tt_populate(ttm);
1516 	}
1517 #endif
1518 
1519 #ifdef CONFIG_SWIOTLB
1520 	if (swiotlb_nr_tbl()) {
1521 		return ttm_dma_populate((void *)ttm, dev->dev);
1522 	}
1523 #endif
1524 
1525 	r = ttm_pool_populate(ttm);
1526 	if (r) {
1527 		return r;
1528 	}
1529 
1530 	for (i = 0; i < ttm->num_pages; i++) {
1531 		dma_addr_t addr;
1532 
1533 		addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE,
1534 				    DMA_BIDIRECTIONAL);
1535 
1536 		if (dma_mapping_error(pdev, addr)) {
1537 			while (--i) {
1538 				dma_unmap_page(pdev, ttm_dma->dma_address[i],
1539 					       PAGE_SIZE, DMA_BIDIRECTIONAL);
1540 				ttm_dma->dma_address[i] = 0;
1541 			}
1542 			ttm_pool_unpopulate(ttm);
1543 			return -EFAULT;
1544 		}
1545 
1546 		ttm_dma->dma_address[i] = addr;
1547 	}
1548 	return 0;
1549 }
1550 
1551 static void
1552 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
1553 {
1554 	struct ttm_dma_tt *ttm_dma = (void *)ttm;
1555 	struct nouveau_drm *drm;
1556 	struct nouveau_device *device;
1557 	struct drm_device *dev;
1558 	struct device *pdev;
1559 	unsigned i;
1560 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1561 
1562 	if (slave)
1563 		return;
1564 
1565 	drm = nouveau_bdev(ttm->bdev);
1566 	device = nvkm_device(&drm->device);
1567 	dev = drm->dev;
1568 	pdev = nv_device_base(device);
1569 
1570 	/*
1571 	 * Objects matching this condition have been marked as force_coherent,
1572 	 * so use the DMA API for them.
1573 	 */
1574 	if (!nv_device_is_cpu_coherent(device) &&
1575 	    ttm->caching_state == tt_uncached) {
1576 		ttm_dma_unpopulate(ttm_dma, dev->dev);
1577 		return;
1578 	}
1579 
1580 #if __OS_HAS_AGP
1581 	if (drm->agp.stat == ENABLED) {
1582 		ttm_agp_tt_unpopulate(ttm);
1583 		return;
1584 	}
1585 #endif
1586 
1587 #ifdef CONFIG_SWIOTLB
1588 	if (swiotlb_nr_tbl()) {
1589 		ttm_dma_unpopulate((void *)ttm, dev->dev);
1590 		return;
1591 	}
1592 #endif
1593 
1594 	for (i = 0; i < ttm->num_pages; i++) {
1595 		if (ttm_dma->dma_address[i]) {
1596 			dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE,
1597 				       DMA_BIDIRECTIONAL);
1598 		}
1599 	}
1600 
1601 	ttm_pool_unpopulate(ttm);
1602 }
1603 
1604 void
1605 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1606 {
1607 	struct reservation_object *resv = nvbo->bo.resv;
1608 
1609 	if (exclusive)
1610 		reservation_object_add_excl_fence(resv, &fence->base);
1611 	else if (fence)
1612 		reservation_object_add_shared_fence(resv, &fence->base);
1613 }
1614 
1615 struct ttm_bo_driver nouveau_bo_driver = {
1616 	.ttm_tt_create = &nouveau_ttm_tt_create,
1617 	.ttm_tt_populate = &nouveau_ttm_tt_populate,
1618 	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1619 	.invalidate_caches = nouveau_bo_invalidate_caches,
1620 	.init_mem_type = nouveau_bo_init_mem_type,
1621 	.evict_flags = nouveau_bo_evict_flags,
1622 	.move_notify = nouveau_bo_move_ntfy,
1623 	.move = nouveau_bo_move,
1624 	.verify_access = nouveau_bo_verify_access,
1625 	.fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1626 	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1627 	.io_mem_free = &nouveau_ttm_io_mem_free,
1628 };
1629 
1630 struct nouveau_vma *
1631 nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm)
1632 {
1633 	struct nouveau_vma *vma;
1634 	list_for_each_entry(vma, &nvbo->vma_list, head) {
1635 		if (vma->vm == vm)
1636 			return vma;
1637 	}
1638 
1639 	return NULL;
1640 }
1641 
1642 int
1643 nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm,
1644 		   struct nouveau_vma *vma)
1645 {
1646 	const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
1647 	int ret;
1648 
1649 	ret = nouveau_vm_get(vm, size, nvbo->page_shift,
1650 			     NV_MEM_ACCESS_RW, vma);
1651 	if (ret)
1652 		return ret;
1653 
1654 	if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&
1655 	    (nvbo->bo.mem.mem_type == TTM_PL_VRAM ||
1656 	     nvbo->page_shift != vma->vm->vmm->lpg_shift))
1657 		nouveau_vm_map(vma, nvbo->bo.mem.mm_node);
1658 
1659 	list_add_tail(&vma->head, &nvbo->vma_list);
1660 	vma->refcount = 1;
1661 	return 0;
1662 }
1663 
1664 void
1665 nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
1666 {
1667 	if (vma->node) {
1668 		if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM)
1669 			nouveau_vm_unmap(vma);
1670 		nouveau_vm_put(vma);
1671 		list_del(&vma->head);
1672 	}
1673 }
1674