1 /*
2  * Copyright 2016-2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/dma-fence.h>
24 #include <linux/spinlock.h>
25 #include <linux/atomic.h>
26 #include <linux/stacktrace.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/sched/mm.h>
30 #include "amdgpu_amdkfd.h"
31 
32 static const struct dma_fence_ops amdkfd_fence_ops;
33 static atomic_t fence_seq = ATOMIC_INIT(0);
34 
35 /* Eviction Fence
36  * Fence helper functions to deal with KFD memory eviction.
37  * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
38  *  evicted unless all the user queues for that process are evicted.
39  *
40  * All the BOs in a process share an eviction fence. When process X wants
41  * to map VRAM memory but TTM can't find enough space, TTM will attempt to
42  * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
43  * by calling ttm_bo_driver->eviction_valuable().
44  *
45  * ttm_bo_driver->eviction_valuable() - will return false if the BO belongs
46  *  to process X. Otherwise, it will return true to indicate BO can be
47  *  evicted by TTM.
48  *
49  * If ttm_bo_driver->eviction_valuable returns true, then TTM will continue
50  * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
51  * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
52  *
53  * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
54  *  nofity when the BO is free to move. fence_add_callback --> enable_signaling
55  *  --> amdgpu_amdkfd_fence.enable_signaling
56  *
57  * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
58  * user queues and signal fence. The work item will also start another delayed
59  * work item to restore BOs
60  */
61 
62 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
63 						       struct mm_struct *mm)
64 {
65 	struct amdgpu_amdkfd_fence *fence;
66 
67 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
68 	if (fence == NULL)
69 		return NULL;
70 
71 	/* This reference gets released in amdkfd_fence_release */
72 	mmgrab(mm);
73 	fence->mm = mm;
74 	get_task_comm(fence->timeline_name, current);
75 	spin_lock_init(&fence->lock);
76 
77 	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
78 		   context, atomic_inc_return(&fence_seq));
79 
80 	return fence;
81 }
82 
83 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
84 {
85 	struct amdgpu_amdkfd_fence *fence;
86 
87 	if (!f)
88 		return NULL;
89 
90 	fence = container_of(f, struct amdgpu_amdkfd_fence, base);
91 	if (fence && f->ops == &amdkfd_fence_ops)
92 		return fence;
93 
94 	return NULL;
95 }
96 
97 static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
98 {
99 	return "amdgpu_amdkfd_fence";
100 }
101 
102 static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
103 {
104 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
105 
106 	return fence->timeline_name;
107 }
108 
109 /**
110  * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
111  *  a KFD BO and schedules a job to move the BO.
112  *  If fence is already signaled return true.
113  *  If fence is not signaled schedule a evict KFD process work item.
114  */
115 static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
116 {
117 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
118 
119 	if (!fence)
120 		return false;
121 
122 	if (dma_fence_is_signaled(f))
123 		return true;
124 
125 	if (!kgd2kfd->schedule_evict_and_restore_process(fence->mm, f))
126 		return true;
127 
128 	return false;
129 }
130 
131 /**
132  * amdkfd_fence_release - callback that fence can be freed
133  *
134  * @fence: fence
135  *
136  * This function is called when the reference count becomes zero.
137  * Drops the mm_struct reference and RCU schedules freeing up the fence.
138  */
139 static void amdkfd_fence_release(struct dma_fence *f)
140 {
141 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
142 
143 	/* Unconditionally signal the fence. The process is getting
144 	 * terminated.
145 	 */
146 	if (WARN_ON(!fence))
147 		return; /* Not an amdgpu_amdkfd_fence */
148 
149 	mmdrop(fence->mm);
150 	kfree_rcu(f, rcu);
151 }
152 
153 /**
154  * amdkfd_fence_check_mm - Check if @mm is same as that of the fence @f
155  *  if same return TRUE else return FALSE.
156  *
157  * @f: [IN] fence
158  * @mm: [IN] mm that needs to be verified
159  */
160 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
161 {
162 	struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
163 
164 	if (!fence)
165 		return false;
166 	else if (fence->mm == mm)
167 		return true;
168 
169 	return false;
170 }
171 
172 static const struct dma_fence_ops amdkfd_fence_ops = {
173 	.get_driver_name = amdkfd_fence_get_driver_name,
174 	.get_timeline_name = amdkfd_fence_get_timeline_name,
175 	.enable_signaling = amdkfd_fence_enable_signaling,
176 	.signaled = NULL,
177 	.wait = dma_fence_default_wait,
178 	.release = amdkfd_fence_release,
179 };
180