1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <drm/drmP.h>
29 #include <drm/amdgpu_drm.h>
30 #include "amdgpu.h"
31 #include "amdgpu_trace.h"
32 
33 /*
34  * GPUVM
35  * GPUVM is similar to the legacy gart on older asics, however
36  * rather than there being a single global gart table
37  * for the entire GPU, there are multiple VM page tables active
38  * at any given time.  The VM page tables can contain a mix
39  * vram pages and system memory pages and system memory pages
40  * can be mapped as snooped (cached system pages) or unsnooped
41  * (uncached system pages).
42  * Each VM has an ID associated with it and there is a page table
43  * associated with each VMID.  When execting a command buffer,
44  * the kernel tells the the ring what VMID to use for that command
45  * buffer.  VMIDs are allocated dynamically as commands are submitted.
46  * The userspace drivers maintain their own address space and the kernel
47  * sets up their pages tables accordingly when they submit their
48  * command buffers and a VMID is assigned.
49  * Cayman/Trinity support up to 8 active VMs at any given time;
50  * SI supports 16.
51  */
52 
53 /**
54  * amdgpu_vm_num_pde - return the number of page directory entries
55  *
56  * @adev: amdgpu_device pointer
57  *
58  * Calculate the number of page directory entries (cayman+).
59  */
60 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
61 {
62 	return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
63 }
64 
65 /**
66  * amdgpu_vm_directory_size - returns the size of the page directory in bytes
67  *
68  * @adev: amdgpu_device pointer
69  *
70  * Calculate the size of the page directory in bytes (cayman+).
71  */
72 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
73 {
74 	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
75 }
76 
77 /**
78  * amdgpu_vm_get_bos - add the vm BOs to a validation list
79  *
80  * @vm: vm providing the BOs
81  * @head: head of validation list
82  *
83  * Add the page directory to the list of BOs to
84  * validate for command submission (cayman+).
85  */
86 struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev,
87 					  struct amdgpu_vm *vm,
88 					  struct list_head *head)
89 {
90 	struct amdgpu_bo_list_entry *list;
91 	unsigned i, idx;
92 
93 	mutex_lock(&vm->mutex);
94 	list = drm_malloc_ab(vm->max_pde_used + 2,
95 			     sizeof(struct amdgpu_bo_list_entry));
96 	if (!list) {
97 		mutex_unlock(&vm->mutex);
98 		return NULL;
99 	}
100 
101 	/* add the vm page table to the list */
102 	list[0].robj = vm->page_directory;
103 	list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM;
104 	list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM;
105 	list[0].priority = 0;
106 	list[0].tv.bo = &vm->page_directory->tbo;
107 	list[0].tv.shared = true;
108 	list_add(&list[0].tv.head, head);
109 
110 	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {
111 		if (!vm->page_tables[i].bo)
112 			continue;
113 
114 		list[idx].robj = vm->page_tables[i].bo;
115 		list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM;
116 		list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM;
117 		list[idx].priority = 0;
118 		list[idx].tv.bo = &list[idx].robj->tbo;
119 		list[idx].tv.shared = true;
120 		list_add(&list[idx++].tv.head, head);
121 	}
122 	mutex_unlock(&vm->mutex);
123 
124 	return list;
125 }
126 
127 /**
128  * amdgpu_vm_grab_id - allocate the next free VMID
129  *
130  * @vm: vm to allocate id for
131  * @ring: ring we want to submit job to
132  * @sync: sync object where we add dependencies
133  *
134  * Allocate an id for the vm, adding fences to the sync obj as necessary.
135  *
136  * Global mutex must be locked!
137  */
138 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
139 		      struct amdgpu_sync *sync)
140 {
141 	struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {};
142 	struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx];
143 	struct amdgpu_device *adev = ring->adev;
144 
145 	unsigned choices[2] = {};
146 	unsigned i;
147 
148 	/* check if the id is still valid */
149 	if (vm_id->id && vm_id->last_id_use &&
150 	    vm_id->last_id_use == adev->vm_manager.active[vm_id->id])
151 		return 0;
152 
153 	/* we definately need to flush */
154 	vm_id->pd_gpu_addr = ~0ll;
155 
156 	/* skip over VMID 0, since it is the system VM */
157 	for (i = 1; i < adev->vm_manager.nvm; ++i) {
158 		struct amdgpu_fence *fence = adev->vm_manager.active[i];
159 
160 		if (fence == NULL) {
161 			/* found a free one */
162 			vm_id->id = i;
163 			trace_amdgpu_vm_grab_id(i, ring->idx);
164 			return 0;
165 		}
166 
167 		if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) {
168 			best[fence->ring->idx] = fence;
169 			choices[fence->ring == ring ? 0 : 1] = i;
170 		}
171 	}
172 
173 	for (i = 0; i < 2; ++i) {
174 		if (choices[i]) {
175 			struct amdgpu_fence *fence;
176 
177 			fence  = adev->vm_manager.active[choices[i]];
178 			vm_id->id = choices[i];
179 
180 			trace_amdgpu_vm_grab_id(choices[i], ring->idx);
181 			return amdgpu_sync_fence(ring->adev, sync, &fence->base);
182 		}
183 	}
184 
185 	/* should never happen */
186 	BUG();
187 	return -EINVAL;
188 }
189 
190 /**
191  * amdgpu_vm_flush - hardware flush the vm
192  *
193  * @ring: ring to use for flush
194  * @vm: vm we want to flush
195  * @updates: last vm update that we waited for
196  *
197  * Flush the vm (cayman+).
198  *
199  * Global and local mutex must be locked!
200  */
201 void amdgpu_vm_flush(struct amdgpu_ring *ring,
202 		     struct amdgpu_vm *vm,
203 		     struct fence *updates)
204 {
205 	uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
206 	struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx];
207 	struct fence *flushed_updates = vm_id->flushed_updates;
208 	bool is_earlier = false;
209 
210 	if (flushed_updates && updates) {
211 		BUG_ON(flushed_updates->context != updates->context);
212 		is_earlier = (updates->seqno - flushed_updates->seqno <=
213 			      INT_MAX) ? true : false;
214 	}
215 
216 	if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates ||
217 	    is_earlier) {
218 
219 		trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id);
220 		if (is_earlier) {
221 			vm_id->flushed_updates = fence_get(updates);
222 			fence_put(flushed_updates);
223 		}
224 		if (!flushed_updates)
225 			vm_id->flushed_updates = fence_get(updates);
226 		vm_id->pd_gpu_addr = pd_addr;
227 		amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr);
228 	}
229 }
230 
231 /**
232  * amdgpu_vm_fence - remember fence for vm
233  *
234  * @adev: amdgpu_device pointer
235  * @vm: vm we want to fence
236  * @fence: fence to remember
237  *
238  * Fence the vm (cayman+).
239  * Set the fence used to protect page table and id.
240  *
241  * Global and local mutex must be locked!
242  */
243 void amdgpu_vm_fence(struct amdgpu_device *adev,
244 		     struct amdgpu_vm *vm,
245 		     struct amdgpu_fence *fence)
246 {
247 	unsigned ridx = fence->ring->idx;
248 	unsigned vm_id = vm->ids[ridx].id;
249 
250 	amdgpu_fence_unref(&adev->vm_manager.active[vm_id]);
251 	adev->vm_manager.active[vm_id] = amdgpu_fence_ref(fence);
252 
253 	amdgpu_fence_unref(&vm->ids[ridx].last_id_use);
254 	vm->ids[ridx].last_id_use = amdgpu_fence_ref(fence);
255 }
256 
257 /**
258  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
259  *
260  * @vm: requested vm
261  * @bo: requested buffer object
262  *
263  * Find @bo inside the requested vm (cayman+).
264  * Search inside the @bos vm list for the requested vm
265  * Returns the found bo_va or NULL if none is found
266  *
267  * Object has to be reserved!
268  */
269 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
270 				       struct amdgpu_bo *bo)
271 {
272 	struct amdgpu_bo_va *bo_va;
273 
274 	list_for_each_entry(bo_va, &bo->va, bo_list) {
275 		if (bo_va->vm == vm) {
276 			return bo_va;
277 		}
278 	}
279 	return NULL;
280 }
281 
282 /**
283  * amdgpu_vm_update_pages - helper to call the right asic function
284  *
285  * @adev: amdgpu_device pointer
286  * @ib: indirect buffer to fill with commands
287  * @pe: addr of the page entry
288  * @addr: dst addr to write into pe
289  * @count: number of page entries to update
290  * @incr: increase next addr by incr bytes
291  * @flags: hw access flags
292  * @gtt_flags: GTT hw access flags
293  *
294  * Traces the parameters and calls the right asic functions
295  * to setup the page table using the DMA.
296  */
297 static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
298 				   struct amdgpu_ib *ib,
299 				   uint64_t pe, uint64_t addr,
300 				   unsigned count, uint32_t incr,
301 				   uint32_t flags, uint32_t gtt_flags)
302 {
303 	trace_amdgpu_vm_set_page(pe, addr, count, incr, flags);
304 
305 	if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) {
306 		uint64_t src = adev->gart.table_addr + (addr >> 12) * 8;
307 		amdgpu_vm_copy_pte(adev, ib, pe, src, count);
308 
309 	} else if ((flags & AMDGPU_PTE_SYSTEM) || (count < 3)) {
310 		amdgpu_vm_write_pte(adev, ib, pe, addr,
311 				      count, incr, flags);
312 
313 	} else {
314 		amdgpu_vm_set_pte_pde(adev, ib, pe, addr,
315 				      count, incr, flags);
316 	}
317 }
318 
319 int amdgpu_vm_free_job(struct amdgpu_job *sched_job)
320 {
321 	int i;
322 	for (i = 0; i < sched_job->num_ibs; i++)
323 		amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]);
324 	kfree(sched_job->ibs);
325 	return 0;
326 }
327 
328 /**
329  * amdgpu_vm_clear_bo - initially clear the page dir/table
330  *
331  * @adev: amdgpu_device pointer
332  * @bo: bo to clear
333  */
334 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
335 			      struct amdgpu_bo *bo)
336 {
337 	struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring;
338 	struct fence *fence = NULL;
339 	struct amdgpu_ib *ib;
340 	unsigned entries;
341 	uint64_t addr;
342 	int r;
343 
344 	r = amdgpu_bo_reserve(bo, false);
345 	if (r)
346 		return r;
347 
348 	r = reservation_object_reserve_shared(bo->tbo.resv);
349 	if (r)
350 		return r;
351 
352 	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
353 	if (r)
354 		goto error_unreserve;
355 
356 	addr = amdgpu_bo_gpu_offset(bo);
357 	entries = amdgpu_bo_size(bo) / 8;
358 
359 	ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
360 	if (!ib)
361 		goto error_unreserve;
362 
363 	r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib);
364 	if (r)
365 		goto error_free;
366 
367 	ib->length_dw = 0;
368 
369 	amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0);
370 	amdgpu_vm_pad_ib(adev, ib);
371 	WARN_ON(ib->length_dw > 64);
372 	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
373 						 &amdgpu_vm_free_job,
374 						 AMDGPU_FENCE_OWNER_VM,
375 						 &fence);
376 	if (!r)
377 		amdgpu_bo_fence(bo, fence, true);
378 	fence_put(fence);
379 	if (amdgpu_enable_scheduler) {
380 		amdgpu_bo_unreserve(bo);
381 		return 0;
382 	}
383 error_free:
384 	amdgpu_ib_free(adev, ib);
385 	kfree(ib);
386 
387 error_unreserve:
388 	amdgpu_bo_unreserve(bo);
389 	return r;
390 }
391 
392 /**
393  * amdgpu_vm_map_gart - get the physical address of a gart page
394  *
395  * @adev: amdgpu_device pointer
396  * @addr: the unmapped addr
397  *
398  * Look up the physical address of the page that the pte resolves
399  * to (cayman+).
400  * Returns the physical address of the page.
401  */
402 uint64_t amdgpu_vm_map_gart(struct amdgpu_device *adev, uint64_t addr)
403 {
404 	uint64_t result;
405 
406 	/* page table offset */
407 	result = adev->gart.pages_addr[addr >> PAGE_SHIFT];
408 
409 	/* in case cpu page size != gpu page size*/
410 	result |= addr & (~PAGE_MASK);
411 
412 	return result;
413 }
414 
415 /**
416  * amdgpu_vm_update_pdes - make sure that page directory is valid
417  *
418  * @adev: amdgpu_device pointer
419  * @vm: requested vm
420  * @start: start of GPU address range
421  * @end: end of GPU address range
422  *
423  * Allocates new page tables if necessary
424  * and updates the page directory (cayman+).
425  * Returns 0 for success, error for failure.
426  *
427  * Global and local mutex must be locked!
428  */
429 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
430 				    struct amdgpu_vm *vm)
431 {
432 	struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring;
433 	struct amdgpu_bo *pd = vm->page_directory;
434 	uint64_t pd_addr = amdgpu_bo_gpu_offset(pd);
435 	uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
436 	uint64_t last_pde = ~0, last_pt = ~0;
437 	unsigned count = 0, pt_idx, ndw;
438 	struct amdgpu_ib *ib;
439 	struct fence *fence = NULL;
440 
441 	int r;
442 
443 	/* padding, etc. */
444 	ndw = 64;
445 
446 	/* assume the worst case */
447 	ndw += vm->max_pde_used * 6;
448 
449 	/* update too big for an IB */
450 	if (ndw > 0xfffff)
451 		return -ENOMEM;
452 
453 	ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
454 	if (!ib)
455 		return -ENOMEM;
456 
457 	r = amdgpu_ib_get(ring, NULL, ndw * 4, ib);
458 	if (r)
459 		return r;
460 	ib->length_dw = 0;
461 
462 	/* walk over the address space and update the page directory */
463 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
464 		struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo;
465 		uint64_t pde, pt;
466 
467 		if (bo == NULL)
468 			continue;
469 
470 		pt = amdgpu_bo_gpu_offset(bo);
471 		if (vm->page_tables[pt_idx].addr == pt)
472 			continue;
473 		vm->page_tables[pt_idx].addr = pt;
474 
475 		pde = pd_addr + pt_idx * 8;
476 		if (((last_pde + 8 * count) != pde) ||
477 		    ((last_pt + incr * count) != pt)) {
478 
479 			if (count) {
480 				amdgpu_vm_update_pages(adev, ib, last_pde,
481 						       last_pt, count, incr,
482 						       AMDGPU_PTE_VALID, 0);
483 			}
484 
485 			count = 1;
486 			last_pde = pde;
487 			last_pt = pt;
488 		} else {
489 			++count;
490 		}
491 	}
492 
493 	if (count)
494 		amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count,
495 				       incr, AMDGPU_PTE_VALID, 0);
496 
497 	if (ib->length_dw != 0) {
498 		amdgpu_vm_pad_ib(adev, ib);
499 		amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM);
500 		WARN_ON(ib->length_dw > ndw);
501 		r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
502 							 &amdgpu_vm_free_job,
503 							 AMDGPU_FENCE_OWNER_VM,
504 							 &fence);
505 		if (r)
506 			goto error_free;
507 
508 		amdgpu_bo_fence(pd, fence, true);
509 		fence_put(vm->page_directory_fence);
510 		vm->page_directory_fence = fence_get(fence);
511 		fence_put(fence);
512 	}
513 
514 	if (!amdgpu_enable_scheduler || ib->length_dw == 0) {
515 		amdgpu_ib_free(adev, ib);
516 		kfree(ib);
517 	}
518 
519 	return 0;
520 
521 error_free:
522 	amdgpu_ib_free(adev, ib);
523 	kfree(ib);
524 	return r;
525 }
526 
527 /**
528  * amdgpu_vm_frag_ptes - add fragment information to PTEs
529  *
530  * @adev: amdgpu_device pointer
531  * @ib: IB for the update
532  * @pe_start: first PTE to handle
533  * @pe_end: last PTE to handle
534  * @addr: addr those PTEs should point to
535  * @flags: hw mapping flags
536  * @gtt_flags: GTT hw mapping flags
537  *
538  * Global and local mutex must be locked!
539  */
540 static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
541 				struct amdgpu_ib *ib,
542 				uint64_t pe_start, uint64_t pe_end,
543 				uint64_t addr, uint32_t flags,
544 				uint32_t gtt_flags)
545 {
546 	/**
547 	 * The MC L1 TLB supports variable sized pages, based on a fragment
548 	 * field in the PTE. When this field is set to a non-zero value, page
549 	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
550 	 * flags are considered valid for all PTEs within the fragment range
551 	 * and corresponding mappings are assumed to be physically contiguous.
552 	 *
553 	 * The L1 TLB can store a single PTE for the whole fragment,
554 	 * significantly increasing the space available for translation
555 	 * caching. This leads to large improvements in throughput when the
556 	 * TLB is under pressure.
557 	 *
558 	 * The L2 TLB distributes small and large fragments into two
559 	 * asymmetric partitions. The large fragment cache is significantly
560 	 * larger. Thus, we try to use large fragments wherever possible.
561 	 * Userspace can support this by aligning virtual base address and
562 	 * allocation size to the fragment size.
563 	 */
564 
565 	/* SI and newer are optimized for 64KB */
566 	uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB;
567 	uint64_t frag_align = 0x80;
568 
569 	uint64_t frag_start = ALIGN(pe_start, frag_align);
570 	uint64_t frag_end = pe_end & ~(frag_align - 1);
571 
572 	unsigned count;
573 
574 	/* system pages are non continuously */
575 	if ((flags & AMDGPU_PTE_SYSTEM) || !(flags & AMDGPU_PTE_VALID) ||
576 	    (frag_start >= frag_end)) {
577 
578 		count = (pe_end - pe_start) / 8;
579 		amdgpu_vm_update_pages(adev, ib, pe_start, addr, count,
580 				       AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags);
581 		return;
582 	}
583 
584 	/* handle the 4K area at the beginning */
585 	if (pe_start != frag_start) {
586 		count = (frag_start - pe_start) / 8;
587 		amdgpu_vm_update_pages(adev, ib, pe_start, addr, count,
588 				       AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags);
589 		addr += AMDGPU_GPU_PAGE_SIZE * count;
590 	}
591 
592 	/* handle the area in the middle */
593 	count = (frag_end - frag_start) / 8;
594 	amdgpu_vm_update_pages(adev, ib, frag_start, addr, count,
595 			       AMDGPU_GPU_PAGE_SIZE, flags | frag_flags,
596 			       gtt_flags);
597 
598 	/* handle the 4K area at the end */
599 	if (frag_end != pe_end) {
600 		addr += AMDGPU_GPU_PAGE_SIZE * count;
601 		count = (pe_end - frag_end) / 8;
602 		amdgpu_vm_update_pages(adev, ib, frag_end, addr, count,
603 				       AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags);
604 	}
605 }
606 
607 /**
608  * amdgpu_vm_update_ptes - make sure that page tables are valid
609  *
610  * @adev: amdgpu_device pointer
611  * @vm: requested vm
612  * @start: start of GPU address range
613  * @end: end of GPU address range
614  * @dst: destination address to map to
615  * @flags: mapping flags
616  *
617  * Update the page tables in the range @start - @end (cayman+).
618  *
619  * Global and local mutex must be locked!
620  */
621 static int amdgpu_vm_update_ptes(struct amdgpu_device *adev,
622 				 struct amdgpu_vm *vm,
623 				 struct amdgpu_ib *ib,
624 				 uint64_t start, uint64_t end,
625 				 uint64_t dst, uint32_t flags,
626 				 uint32_t gtt_flags)
627 {
628 	uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
629 	uint64_t last_pte = ~0, last_dst = ~0;
630 	unsigned count = 0;
631 	uint64_t addr;
632 
633 	/* walk over the address space and update the page tables */
634 	for (addr = start; addr < end; ) {
635 		uint64_t pt_idx = addr >> amdgpu_vm_block_size;
636 		struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo;
637 		unsigned nptes;
638 		uint64_t pte;
639 		int r;
640 
641 		amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv,
642 				 AMDGPU_FENCE_OWNER_VM);
643 		r = reservation_object_reserve_shared(pt->tbo.resv);
644 		if (r)
645 			return r;
646 
647 		if ((addr & ~mask) == (end & ~mask))
648 			nptes = end - addr;
649 		else
650 			nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
651 
652 		pte = amdgpu_bo_gpu_offset(pt);
653 		pte += (addr & mask) * 8;
654 
655 		if ((last_pte + 8 * count) != pte) {
656 
657 			if (count) {
658 				amdgpu_vm_frag_ptes(adev, ib, last_pte,
659 						    last_pte + 8 * count,
660 						    last_dst, flags,
661 						    gtt_flags);
662 			}
663 
664 			count = nptes;
665 			last_pte = pte;
666 			last_dst = dst;
667 		} else {
668 			count += nptes;
669 		}
670 
671 		addr += nptes;
672 		dst += nptes * AMDGPU_GPU_PAGE_SIZE;
673 	}
674 
675 	if (count) {
676 		amdgpu_vm_frag_ptes(adev, ib, last_pte,
677 				    last_pte + 8 * count,
678 				    last_dst, flags, gtt_flags);
679 	}
680 
681 	return 0;
682 }
683 
684 /**
685  * amdgpu_vm_fence_pts - fence page tables after an update
686  *
687  * @vm: requested vm
688  * @start: start of GPU address range
689  * @end: end of GPU address range
690  * @fence: fence to use
691  *
692  * Fence the page tables in the range @start - @end (cayman+).
693  *
694  * Global and local mutex must be locked!
695  */
696 static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm,
697 				uint64_t start, uint64_t end,
698 				struct fence *fence)
699 {
700 	unsigned i;
701 
702 	start >>= amdgpu_vm_block_size;
703 	end >>= amdgpu_vm_block_size;
704 
705 	for (i = start; i <= end; ++i)
706 		amdgpu_bo_fence(vm->page_tables[i].bo, fence, true);
707 }
708 
709 /**
710  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
711  *
712  * @adev: amdgpu_device pointer
713  * @vm: requested vm
714  * @mapping: mapped range and flags to use for the update
715  * @addr: addr to set the area to
716  * @gtt_flags: flags as they are used for GTT
717  * @fence: optional resulting fence
718  *
719  * Fill in the page table entries for @mapping.
720  * Returns 0 for success, -EINVAL for failure.
721  *
722  * Object have to be reserved and mutex must be locked!
723  */
724 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
725 				       struct amdgpu_vm *vm,
726 				       struct amdgpu_bo_va_mapping *mapping,
727 				       uint64_t addr, uint32_t gtt_flags,
728 				       struct fence **fence)
729 {
730 	struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring;
731 	unsigned nptes, ncmds, ndw;
732 	uint32_t flags = gtt_flags;
733 	struct amdgpu_ib *ib;
734 	struct fence *f = NULL;
735 	int r;
736 
737 	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
738 	 * but in case of something, we filter the flags in first place
739 	 */
740 	if (!(mapping->flags & AMDGPU_PTE_READABLE))
741 		flags &= ~AMDGPU_PTE_READABLE;
742 	if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
743 		flags &= ~AMDGPU_PTE_WRITEABLE;
744 
745 	trace_amdgpu_vm_bo_update(mapping);
746 
747 	nptes = mapping->it.last - mapping->it.start + 1;
748 
749 	/*
750 	 * reserve space for one command every (1 << BLOCK_SIZE)
751 	 *  entries or 2k dwords (whatever is smaller)
752 	 */
753 	ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;
754 
755 	/* padding, etc. */
756 	ndw = 64;
757 
758 	if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) {
759 		/* only copy commands needed */
760 		ndw += ncmds * 7;
761 
762 	} else if (flags & AMDGPU_PTE_SYSTEM) {
763 		/* header for write data commands */
764 		ndw += ncmds * 4;
765 
766 		/* body of write data command */
767 		ndw += nptes * 2;
768 
769 	} else {
770 		/* set page commands needed */
771 		ndw += ncmds * 10;
772 
773 		/* two extra commands for begin/end of fragment */
774 		ndw += 2 * 10;
775 	}
776 
777 	/* update too big for an IB */
778 	if (ndw > 0xfffff)
779 		return -ENOMEM;
780 
781 	ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
782 	if (!ib)
783 		return -ENOMEM;
784 
785 	r = amdgpu_ib_get(ring, NULL, ndw * 4, ib);
786 	if (r) {
787 		kfree(ib);
788 		return r;
789 	}
790 
791 	ib->length_dw = 0;
792 
793 	if (!(flags & AMDGPU_PTE_VALID)) {
794 		unsigned i;
795 
796 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
797 			struct amdgpu_fence *f = vm->ids[i].last_id_use;
798 			r = amdgpu_sync_fence(adev, &ib->sync, &f->base);
799 			if (r)
800 				return r;
801 		}
802 	}
803 
804 	r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start,
805 				  mapping->it.last + 1, addr + mapping->offset,
806 				  flags, gtt_flags);
807 
808 	if (r) {
809 		amdgpu_ib_free(adev, ib);
810 		kfree(ib);
811 		return r;
812 	}
813 
814 	amdgpu_vm_pad_ib(adev, ib);
815 	WARN_ON(ib->length_dw > ndw);
816 	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
817 						 &amdgpu_vm_free_job,
818 						 AMDGPU_FENCE_OWNER_VM,
819 						 &f);
820 	if (r)
821 		goto error_free;
822 
823 	amdgpu_vm_fence_pts(vm, mapping->it.start,
824 			    mapping->it.last + 1, f);
825 	if (fence) {
826 		fence_put(*fence);
827 		*fence = fence_get(f);
828 	}
829 	fence_put(f);
830 	if (!amdgpu_enable_scheduler) {
831 		amdgpu_ib_free(adev, ib);
832 		kfree(ib);
833 	}
834 	return 0;
835 
836 error_free:
837 	amdgpu_ib_free(adev, ib);
838 	kfree(ib);
839 	return r;
840 }
841 
842 /**
843  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
844  *
845  * @adev: amdgpu_device pointer
846  * @bo_va: requested BO and VM object
847  * @mem: ttm mem
848  *
849  * Fill in the page table entries for @bo_va.
850  * Returns 0 for success, -EINVAL for failure.
851  *
852  * Object have to be reserved and mutex must be locked!
853  */
854 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
855 			struct amdgpu_bo_va *bo_va,
856 			struct ttm_mem_reg *mem)
857 {
858 	struct amdgpu_vm *vm = bo_va->vm;
859 	struct amdgpu_bo_va_mapping *mapping;
860 	uint32_t flags;
861 	uint64_t addr;
862 	int r;
863 
864 	if (mem) {
865 		addr = mem->start << PAGE_SHIFT;
866 		if (mem->mem_type != TTM_PL_TT)
867 			addr += adev->vm_manager.vram_base_offset;
868 	} else {
869 		addr = 0;
870 	}
871 
872 	flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
873 
874 	spin_lock(&vm->status_lock);
875 	if (!list_empty(&bo_va->vm_status))
876 		list_splice_init(&bo_va->valids, &bo_va->invalids);
877 	spin_unlock(&vm->status_lock);
878 
879 	list_for_each_entry(mapping, &bo_va->invalids, list) {
880 		r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr,
881 						flags, &bo_va->last_pt_update);
882 		if (r)
883 			return r;
884 	}
885 
886 	spin_lock(&vm->status_lock);
887 	list_splice_init(&bo_va->invalids, &bo_va->valids);
888 	list_del_init(&bo_va->vm_status);
889 	if (!mem)
890 		list_add(&bo_va->vm_status, &vm->cleared);
891 	spin_unlock(&vm->status_lock);
892 
893 	return 0;
894 }
895 
896 /**
897  * amdgpu_vm_clear_freed - clear freed BOs in the PT
898  *
899  * @adev: amdgpu_device pointer
900  * @vm: requested vm
901  *
902  * Make sure all freed BOs are cleared in the PT.
903  * Returns 0 for success.
904  *
905  * PTs have to be reserved and mutex must be locked!
906  */
907 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
908 			  struct amdgpu_vm *vm)
909 {
910 	struct amdgpu_bo_va_mapping *mapping;
911 	int r;
912 
913 	while (!list_empty(&vm->freed)) {
914 		mapping = list_first_entry(&vm->freed,
915 			struct amdgpu_bo_va_mapping, list);
916 		list_del(&mapping->list);
917 
918 		r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, 0, 0, NULL);
919 		kfree(mapping);
920 		if (r)
921 			return r;
922 
923 	}
924 	return 0;
925 
926 }
927 
928 /**
929  * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
930  *
931  * @adev: amdgpu_device pointer
932  * @vm: requested vm
933  *
934  * Make sure all invalidated BOs are cleared in the PT.
935  * Returns 0 for success.
936  *
937  * PTs have to be reserved and mutex must be locked!
938  */
939 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
940 			     struct amdgpu_vm *vm, struct amdgpu_sync *sync)
941 {
942 	struct amdgpu_bo_va *bo_va = NULL;
943 	int r = 0;
944 
945 	spin_lock(&vm->status_lock);
946 	while (!list_empty(&vm->invalidated)) {
947 		bo_va = list_first_entry(&vm->invalidated,
948 			struct amdgpu_bo_va, vm_status);
949 		spin_unlock(&vm->status_lock);
950 
951 		r = amdgpu_vm_bo_update(adev, bo_va, NULL);
952 		if (r)
953 			return r;
954 
955 		spin_lock(&vm->status_lock);
956 	}
957 	spin_unlock(&vm->status_lock);
958 
959 	if (bo_va)
960 		r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
961 
962 	return r;
963 }
964 
965 /**
966  * amdgpu_vm_bo_add - add a bo to a specific vm
967  *
968  * @adev: amdgpu_device pointer
969  * @vm: requested vm
970  * @bo: amdgpu buffer object
971  *
972  * Add @bo into the requested vm (cayman+).
973  * Add @bo to the list of bos associated with the vm
974  * Returns newly added bo_va or NULL for failure
975  *
976  * Object has to be reserved!
977  */
978 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
979 				      struct amdgpu_vm *vm,
980 				      struct amdgpu_bo *bo)
981 {
982 	struct amdgpu_bo_va *bo_va;
983 
984 	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
985 	if (bo_va == NULL) {
986 		return NULL;
987 	}
988 	bo_va->vm = vm;
989 	bo_va->bo = bo;
990 	bo_va->ref_count = 1;
991 	INIT_LIST_HEAD(&bo_va->bo_list);
992 	INIT_LIST_HEAD(&bo_va->valids);
993 	INIT_LIST_HEAD(&bo_va->invalids);
994 	INIT_LIST_HEAD(&bo_va->vm_status);
995 
996 	mutex_lock(&vm->mutex);
997 	list_add_tail(&bo_va->bo_list, &bo->va);
998 	mutex_unlock(&vm->mutex);
999 
1000 	return bo_va;
1001 }
1002 
1003 /**
1004  * amdgpu_vm_bo_map - map bo inside a vm
1005  *
1006  * @adev: amdgpu_device pointer
1007  * @bo_va: bo_va to store the address
1008  * @saddr: where to map the BO
1009  * @offset: requested offset in the BO
1010  * @flags: attributes of pages (read/write/valid/etc.)
1011  *
1012  * Add a mapping of the BO at the specefied addr into the VM.
1013  * Returns 0 for success, error for failure.
1014  *
1015  * Object has to be reserved and gets unreserved by this function!
1016  */
1017 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1018 		     struct amdgpu_bo_va *bo_va,
1019 		     uint64_t saddr, uint64_t offset,
1020 		     uint64_t size, uint32_t flags)
1021 {
1022 	struct amdgpu_bo_va_mapping *mapping;
1023 	struct amdgpu_vm *vm = bo_va->vm;
1024 	struct interval_tree_node *it;
1025 	unsigned last_pfn, pt_idx;
1026 	uint64_t eaddr;
1027 	int r;
1028 
1029 	/* validate the parameters */
1030 	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1031 	    size == 0 || size & AMDGPU_GPU_PAGE_MASK) {
1032 		amdgpu_bo_unreserve(bo_va->bo);
1033 		return -EINVAL;
1034 	}
1035 
1036 	/* make sure object fit at this offset */
1037 	eaddr = saddr + size;
1038 	if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo))) {
1039 		amdgpu_bo_unreserve(bo_va->bo);
1040 		return -EINVAL;
1041 	}
1042 
1043 	last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
1044 	if (last_pfn > adev->vm_manager.max_pfn) {
1045 		dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n",
1046 			last_pfn, adev->vm_manager.max_pfn);
1047 		amdgpu_bo_unreserve(bo_va->bo);
1048 		return -EINVAL;
1049 	}
1050 
1051 	mutex_lock(&vm->mutex);
1052 
1053 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1054 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
1055 
1056 	it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1);
1057 	if (it) {
1058 		struct amdgpu_bo_va_mapping *tmp;
1059 		tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1060 		/* bo and tmp overlap, invalid addr */
1061 		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1062 			"0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
1063 			tmp->it.start, tmp->it.last + 1);
1064 		amdgpu_bo_unreserve(bo_va->bo);
1065 		r = -EINVAL;
1066 		goto error_unlock;
1067 	}
1068 
1069 	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1070 	if (!mapping) {
1071 		amdgpu_bo_unreserve(bo_va->bo);
1072 		r = -ENOMEM;
1073 		goto error_unlock;
1074 	}
1075 
1076 	INIT_LIST_HEAD(&mapping->list);
1077 	mapping->it.start = saddr;
1078 	mapping->it.last = eaddr - 1;
1079 	mapping->offset = offset;
1080 	mapping->flags = flags;
1081 
1082 	list_add(&mapping->list, &bo_va->invalids);
1083 	interval_tree_insert(&mapping->it, &vm->va);
1084 	trace_amdgpu_vm_bo_map(bo_va, mapping);
1085 
1086 	/* Make sure the page tables are allocated */
1087 	saddr >>= amdgpu_vm_block_size;
1088 	eaddr >>= amdgpu_vm_block_size;
1089 
1090 	BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));
1091 
1092 	if (eaddr > vm->max_pde_used)
1093 		vm->max_pde_used = eaddr;
1094 
1095 	amdgpu_bo_unreserve(bo_va->bo);
1096 
1097 	/* walk over the address space and allocate the page tables */
1098 	for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
1099 		struct amdgpu_bo *pt;
1100 
1101 		if (vm->page_tables[pt_idx].bo)
1102 			continue;
1103 
1104 		/* drop mutex to allocate and clear page table */
1105 		mutex_unlock(&vm->mutex);
1106 
1107 		r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
1108 				     AMDGPU_GPU_PAGE_SIZE, true,
1109 				     AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &pt);
1110 		if (r)
1111 			goto error_free;
1112 
1113 		r = amdgpu_vm_clear_bo(adev, pt);
1114 		if (r) {
1115 			amdgpu_bo_unref(&pt);
1116 			goto error_free;
1117 		}
1118 
1119 		/* aquire mutex again */
1120 		mutex_lock(&vm->mutex);
1121 		if (vm->page_tables[pt_idx].bo) {
1122 			/* someone else allocated the pt in the meantime */
1123 			mutex_unlock(&vm->mutex);
1124 			amdgpu_bo_unref(&pt);
1125 			mutex_lock(&vm->mutex);
1126 			continue;
1127 		}
1128 
1129 		vm->page_tables[pt_idx].addr = 0;
1130 		vm->page_tables[pt_idx].bo = pt;
1131 	}
1132 
1133 	mutex_unlock(&vm->mutex);
1134 	return 0;
1135 
1136 error_free:
1137 	mutex_lock(&vm->mutex);
1138 	list_del(&mapping->list);
1139 	interval_tree_remove(&mapping->it, &vm->va);
1140 	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1141 	kfree(mapping);
1142 
1143 error_unlock:
1144 	mutex_unlock(&vm->mutex);
1145 	return r;
1146 }
1147 
1148 /**
1149  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1150  *
1151  * @adev: amdgpu_device pointer
1152  * @bo_va: bo_va to remove the address from
1153  * @saddr: where to the BO is mapped
1154  *
1155  * Remove a mapping of the BO at the specefied addr from the VM.
1156  * Returns 0 for success, error for failure.
1157  *
1158  * Object has to be reserved and gets unreserved by this function!
1159  */
1160 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1161 		       struct amdgpu_bo_va *bo_va,
1162 		       uint64_t saddr)
1163 {
1164 	struct amdgpu_bo_va_mapping *mapping;
1165 	struct amdgpu_vm *vm = bo_va->vm;
1166 	bool valid = true;
1167 
1168 	saddr /= AMDGPU_GPU_PAGE_SIZE;
1169 
1170 	list_for_each_entry(mapping, &bo_va->valids, list) {
1171 		if (mapping->it.start == saddr)
1172 			break;
1173 	}
1174 
1175 	if (&mapping->list == &bo_va->valids) {
1176 		valid = false;
1177 
1178 		list_for_each_entry(mapping, &bo_va->invalids, list) {
1179 			if (mapping->it.start == saddr)
1180 				break;
1181 		}
1182 
1183 		if (&mapping->list == &bo_va->invalids) {
1184 			amdgpu_bo_unreserve(bo_va->bo);
1185 			return -ENOENT;
1186 		}
1187 	}
1188 
1189 	mutex_lock(&vm->mutex);
1190 	list_del(&mapping->list);
1191 	interval_tree_remove(&mapping->it, &vm->va);
1192 	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1193 
1194 	if (valid)
1195 		list_add(&mapping->list, &vm->freed);
1196 	else
1197 		kfree(mapping);
1198 	mutex_unlock(&vm->mutex);
1199 	amdgpu_bo_unreserve(bo_va->bo);
1200 
1201 	return 0;
1202 }
1203 
1204 /**
1205  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1206  *
1207  * @adev: amdgpu_device pointer
1208  * @bo_va: requested bo_va
1209  *
1210  * Remove @bo_va->bo from the requested vm (cayman+).
1211  *
1212  * Object have to be reserved!
1213  */
1214 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1215 		      struct amdgpu_bo_va *bo_va)
1216 {
1217 	struct amdgpu_bo_va_mapping *mapping, *next;
1218 	struct amdgpu_vm *vm = bo_va->vm;
1219 
1220 	list_del(&bo_va->bo_list);
1221 
1222 	mutex_lock(&vm->mutex);
1223 
1224 	spin_lock(&vm->status_lock);
1225 	list_del(&bo_va->vm_status);
1226 	spin_unlock(&vm->status_lock);
1227 
1228 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1229 		list_del(&mapping->list);
1230 		interval_tree_remove(&mapping->it, &vm->va);
1231 		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1232 		list_add(&mapping->list, &vm->freed);
1233 	}
1234 	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1235 		list_del(&mapping->list);
1236 		interval_tree_remove(&mapping->it, &vm->va);
1237 		kfree(mapping);
1238 	}
1239 
1240 	fence_put(bo_va->last_pt_update);
1241 	kfree(bo_va);
1242 
1243 	mutex_unlock(&vm->mutex);
1244 }
1245 
1246 /**
1247  * amdgpu_vm_bo_invalidate - mark the bo as invalid
1248  *
1249  * @adev: amdgpu_device pointer
1250  * @vm: requested vm
1251  * @bo: amdgpu buffer object
1252  *
1253  * Mark @bo as invalid (cayman+).
1254  */
1255 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1256 			     struct amdgpu_bo *bo)
1257 {
1258 	struct amdgpu_bo_va *bo_va;
1259 
1260 	list_for_each_entry(bo_va, &bo->va, bo_list) {
1261 		spin_lock(&bo_va->vm->status_lock);
1262 		if (list_empty(&bo_va->vm_status))
1263 			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1264 		spin_unlock(&bo_va->vm->status_lock);
1265 	}
1266 }
1267 
1268 /**
1269  * amdgpu_vm_init - initialize a vm instance
1270  *
1271  * @adev: amdgpu_device pointer
1272  * @vm: requested vm
1273  *
1274  * Init @vm fields (cayman+).
1275  */
1276 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1277 {
1278 	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
1279 		AMDGPU_VM_PTE_COUNT * 8);
1280 	unsigned pd_size, pd_entries, pts_size;
1281 	int i, r;
1282 
1283 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1284 		vm->ids[i].id = 0;
1285 		vm->ids[i].flushed_updates = NULL;
1286 		vm->ids[i].last_id_use = NULL;
1287 	}
1288 	mutex_init(&vm->mutex);
1289 	vm->va = RB_ROOT;
1290 	spin_lock_init(&vm->status_lock);
1291 	INIT_LIST_HEAD(&vm->invalidated);
1292 	INIT_LIST_HEAD(&vm->cleared);
1293 	INIT_LIST_HEAD(&vm->freed);
1294 
1295 	pd_size = amdgpu_vm_directory_size(adev);
1296 	pd_entries = amdgpu_vm_num_pdes(adev);
1297 
1298 	/* allocate page table array */
1299 	pts_size = pd_entries * sizeof(struct amdgpu_vm_pt);
1300 	vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
1301 	if (vm->page_tables == NULL) {
1302 		DRM_ERROR("Cannot allocate memory for page table array\n");
1303 		return -ENOMEM;
1304 	}
1305 
1306 	vm->page_directory_fence = NULL;
1307 
1308 	r = amdgpu_bo_create(adev, pd_size, align, true,
1309 			     AMDGPU_GEM_DOMAIN_VRAM, 0,
1310 			     NULL, &vm->page_directory);
1311 	if (r)
1312 		return r;
1313 
1314 	r = amdgpu_vm_clear_bo(adev, vm->page_directory);
1315 	if (r) {
1316 		amdgpu_bo_unref(&vm->page_directory);
1317 		vm->page_directory = NULL;
1318 		return r;
1319 	}
1320 
1321 	return 0;
1322 }
1323 
1324 /**
1325  * amdgpu_vm_fini - tear down a vm instance
1326  *
1327  * @adev: amdgpu_device pointer
1328  * @vm: requested vm
1329  *
1330  * Tear down @vm (cayman+).
1331  * Unbind the VM and remove all bos from the vm bo list
1332  */
1333 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1334 {
1335 	struct amdgpu_bo_va_mapping *mapping, *tmp;
1336 	int i;
1337 
1338 	if (!RB_EMPTY_ROOT(&vm->va)) {
1339 		dev_err(adev->dev, "still active bo inside vm\n");
1340 	}
1341 	rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
1342 		list_del(&mapping->list);
1343 		interval_tree_remove(&mapping->it, &vm->va);
1344 		kfree(mapping);
1345 	}
1346 	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1347 		list_del(&mapping->list);
1348 		kfree(mapping);
1349 	}
1350 
1351 	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++)
1352 		amdgpu_bo_unref(&vm->page_tables[i].bo);
1353 	kfree(vm->page_tables);
1354 
1355 	amdgpu_bo_unref(&vm->page_directory);
1356 	fence_put(vm->page_directory_fence);
1357 
1358 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1359 		fence_put(vm->ids[i].flushed_updates);
1360 		amdgpu_fence_unref(&vm->ids[i].last_id_use);
1361 	}
1362 
1363 	mutex_destroy(&vm->mutex);
1364 }
1365