1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Christian König <christian.koenig@amd.com>
29  */
30 
31 #include <linux/dma-fence-chain.h>
32 
33 #include "amdgpu.h"
34 #include "amdgpu_trace.h"
35 #include "amdgpu_amdkfd.h"
36 
37 struct amdgpu_sync_entry {
38 	struct hlist_node	node;
39 	struct dma_fence	*fence;
40 };
41 
42 static struct kmem_cache *amdgpu_sync_slab;
43 
44 /**
45  * amdgpu_sync_create - zero init sync object
46  *
47  * @sync: sync object to initialize
48  *
49  * Just clear the sync object for now.
50  */
51 void amdgpu_sync_create(struct amdgpu_sync *sync)
52 {
53 	hash_init(sync->fences);
54 }
55 
56 /**
57  * amdgpu_sync_same_dev - test if fence belong to us
58  *
59  * @adev: amdgpu device to use for the test
60  * @f: fence to test
61  *
62  * Test if the fence was issued by us.
63  */
64 static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,
65 				 struct dma_fence *f)
66 {
67 	struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
68 
69 	if (s_fence) {
70 		struct amdgpu_ring *ring;
71 
72 		ring = container_of(s_fence->sched, struct amdgpu_ring, sched);
73 		return ring->adev == adev;
74 	}
75 
76 	return false;
77 }
78 
79 /**
80  * amdgpu_sync_get_owner - extract the owner of a fence
81  *
82  * @f: fence get the owner from
83  *
84  * Extract who originally created the fence.
85  */
86 static void *amdgpu_sync_get_owner(struct dma_fence *f)
87 {
88 	struct drm_sched_fence *s_fence;
89 	struct amdgpu_amdkfd_fence *kfd_fence;
90 
91 	if (!f)
92 		return AMDGPU_FENCE_OWNER_UNDEFINED;
93 
94 	s_fence = to_drm_sched_fence(f);
95 	if (s_fence)
96 		return s_fence->owner;
97 
98 	kfd_fence = to_amdgpu_amdkfd_fence(f);
99 	if (kfd_fence)
100 		return AMDGPU_FENCE_OWNER_KFD;
101 
102 	return AMDGPU_FENCE_OWNER_UNDEFINED;
103 }
104 
105 /**
106  * amdgpu_sync_keep_later - Keep the later fence
107  *
108  * @keep: existing fence to test
109  * @fence: new fence
110  *
111  * Either keep the existing fence or the new one, depending which one is later.
112  */
113 static void amdgpu_sync_keep_later(struct dma_fence **keep,
114 				   struct dma_fence *fence)
115 {
116 	if (*keep && dma_fence_is_later(*keep, fence))
117 		return;
118 
119 	dma_fence_put(*keep);
120 	*keep = dma_fence_get(fence);
121 }
122 
123 /**
124  * amdgpu_sync_add_later - add the fence to the hash
125  *
126  * @sync: sync object to add the fence to
127  * @f: fence to add
128  *
129  * Tries to add the fence to an existing hash entry. Returns true when an entry
130  * was found, false otherwise.
131  */
132 static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
133 {
134 	struct amdgpu_sync_entry *e;
135 
136 	hash_for_each_possible(sync->fences, e, node, f->context) {
137 		if (unlikely(e->fence->context != f->context))
138 			continue;
139 
140 		amdgpu_sync_keep_later(&e->fence, f);
141 		return true;
142 	}
143 	return false;
144 }
145 
146 /**
147  * amdgpu_sync_fence - remember to sync to this fence
148  *
149  * @sync: sync object to add fence to
150  * @f: fence to sync to
151  *
152  * Add the fence to the sync object.
153  */
154 int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
155 {
156 	struct amdgpu_sync_entry *e;
157 
158 	if (!f)
159 		return 0;
160 
161 	if (amdgpu_sync_add_later(sync, f))
162 		return 0;
163 
164 	e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
165 	if (!e)
166 		return -ENOMEM;
167 
168 	hash_add(sync->fences, &e->node, f->context);
169 	e->fence = dma_fence_get(f);
170 	return 0;
171 }
172 
173 /* Determine based on the owner and mode if we should sync to a fence or not */
174 static bool amdgpu_sync_test_fence(struct amdgpu_device *adev,
175 				   enum amdgpu_sync_mode mode,
176 				   void *owner, struct dma_fence *f)
177 {
178 	void *fence_owner = amdgpu_sync_get_owner(f);
179 
180 	/* Always sync to moves, no matter what */
181 	if (fence_owner == AMDGPU_FENCE_OWNER_UNDEFINED)
182 		return true;
183 
184 	/* We only want to trigger KFD eviction fences on
185 	 * evict or move jobs. Skip KFD fences otherwise.
186 	 */
187 	if (fence_owner == AMDGPU_FENCE_OWNER_KFD &&
188 	    owner != AMDGPU_FENCE_OWNER_UNDEFINED)
189 		return false;
190 
191 	/* Never sync to VM updates either. */
192 	if (fence_owner == AMDGPU_FENCE_OWNER_VM &&
193 	    owner != AMDGPU_FENCE_OWNER_UNDEFINED)
194 		return false;
195 
196 	/* Ignore fences depending on the sync mode */
197 	switch (mode) {
198 	case AMDGPU_SYNC_ALWAYS:
199 		return true;
200 
201 	case AMDGPU_SYNC_NE_OWNER:
202 		if (amdgpu_sync_same_dev(adev, f) &&
203 		    fence_owner == owner)
204 			return false;
205 		break;
206 
207 	case AMDGPU_SYNC_EQ_OWNER:
208 		if (amdgpu_sync_same_dev(adev, f) &&
209 		    fence_owner != owner)
210 			return false;
211 		break;
212 
213 	case AMDGPU_SYNC_EXPLICIT:
214 		return false;
215 	}
216 
217 	WARN(debug_evictions && fence_owner == AMDGPU_FENCE_OWNER_KFD,
218 	     "Adding eviction fence to sync obj");
219 	return true;
220 }
221 
222 /**
223  * amdgpu_sync_resv - sync to a reservation object
224  *
225  * @adev: amdgpu device
226  * @sync: sync object to add fences from reservation object to
227  * @resv: reservation object with embedded fence
228  * @mode: how owner affects which fences we sync to
229  * @owner: owner of the planned job submission
230  *
231  * Sync to the fence
232  */
233 int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
234 		     struct dma_resv *resv, enum amdgpu_sync_mode mode,
235 		     void *owner)
236 {
237 	struct dma_resv_iter cursor;
238 	struct dma_fence *f;
239 	int r;
240 
241 	if (resv == NULL)
242 		return -EINVAL;
243 
244 	dma_resv_for_each_fence(&cursor, resv, true, f) {
245 		dma_fence_chain_for_each(f, f) {
246 			struct dma_fence *tmp = dma_fence_chain_contained(f);
247 
248 			if (amdgpu_sync_test_fence(adev, mode, owner, tmp)) {
249 				r = amdgpu_sync_fence(sync, f);
250 				dma_fence_put(f);
251 				if (r)
252 					return r;
253 				break;
254 			}
255 		}
256 	}
257 	return 0;
258 }
259 
260 /**
261  * amdgpu_sync_peek_fence - get the next fence not signaled yet
262  *
263  * @sync: the sync object
264  * @ring: optional ring to use for test
265  *
266  * Returns the next fence not signaled yet without removing it from the sync
267  * object.
268  */
269 struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
270 					 struct amdgpu_ring *ring)
271 {
272 	struct amdgpu_sync_entry *e;
273 	struct hlist_node *tmp;
274 	int i;
275 
276 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
277 		struct dma_fence *f = e->fence;
278 		struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
279 
280 		if (dma_fence_is_signaled(f)) {
281 			hash_del(&e->node);
282 			dma_fence_put(f);
283 			kmem_cache_free(amdgpu_sync_slab, e);
284 			continue;
285 		}
286 		if (ring && s_fence) {
287 			/* For fences from the same ring it is sufficient
288 			 * when they are scheduled.
289 			 */
290 			if (s_fence->sched == &ring->sched) {
291 				if (dma_fence_is_signaled(&s_fence->scheduled))
292 					continue;
293 
294 				return &s_fence->scheduled;
295 			}
296 		}
297 
298 		return f;
299 	}
300 
301 	return NULL;
302 }
303 
304 /**
305  * amdgpu_sync_get_fence - get the next fence from the sync object
306  *
307  * @sync: sync object to use
308  *
309  * Get and removes the next fence from the sync object not signaled yet.
310  */
311 struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
312 {
313 	struct amdgpu_sync_entry *e;
314 	struct hlist_node *tmp;
315 	struct dma_fence *f;
316 	int i;
317 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
318 
319 		f = e->fence;
320 
321 		hash_del(&e->node);
322 		kmem_cache_free(amdgpu_sync_slab, e);
323 
324 		if (!dma_fence_is_signaled(f))
325 			return f;
326 
327 		dma_fence_put(f);
328 	}
329 	return NULL;
330 }
331 
332 /**
333  * amdgpu_sync_clone - clone a sync object
334  *
335  * @source: sync object to clone
336  * @clone: pointer to destination sync object
337  *
338  * Adds references to all unsignaled fences in @source to @clone. Also
339  * removes signaled fences from @source while at it.
340  */
341 int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone)
342 {
343 	struct amdgpu_sync_entry *e;
344 	struct hlist_node *tmp;
345 	struct dma_fence *f;
346 	int i, r;
347 
348 	hash_for_each_safe(source->fences, i, tmp, e, node) {
349 		f = e->fence;
350 		if (!dma_fence_is_signaled(f)) {
351 			r = amdgpu_sync_fence(clone, f);
352 			if (r)
353 				return r;
354 		} else {
355 			hash_del(&e->node);
356 			dma_fence_put(f);
357 			kmem_cache_free(amdgpu_sync_slab, e);
358 		}
359 	}
360 
361 	return 0;
362 }
363 
364 int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr)
365 {
366 	struct amdgpu_sync_entry *e;
367 	struct hlist_node *tmp;
368 	int i, r;
369 
370 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
371 		r = dma_fence_wait(e->fence, intr);
372 		if (r)
373 			return r;
374 
375 		hash_del(&e->node);
376 		dma_fence_put(e->fence);
377 		kmem_cache_free(amdgpu_sync_slab, e);
378 	}
379 
380 	return 0;
381 }
382 
383 /**
384  * amdgpu_sync_free - free the sync object
385  *
386  * @sync: sync object to use
387  *
388  * Free the sync object.
389  */
390 void amdgpu_sync_free(struct amdgpu_sync *sync)
391 {
392 	struct amdgpu_sync_entry *e;
393 	struct hlist_node *tmp;
394 	unsigned i;
395 
396 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
397 		hash_del(&e->node);
398 		dma_fence_put(e->fence);
399 		kmem_cache_free(amdgpu_sync_slab, e);
400 	}
401 }
402 
403 /**
404  * amdgpu_sync_init - init sync object subsystem
405  *
406  * Allocate the slab allocator.
407  */
408 int amdgpu_sync_init(void)
409 {
410 	amdgpu_sync_slab = kmem_cache_create(
411 		"amdgpu_sync", sizeof(struct amdgpu_sync_entry), 0,
412 		SLAB_HWCACHE_ALIGN, NULL);
413 	if (!amdgpu_sync_slab)
414 		return -ENOMEM;
415 
416 	return 0;
417 }
418 
419 /**
420  * amdgpu_sync_fini - fini sync object subsystem
421  *
422  * Free the slab allocator.
423  */
424 void amdgpu_sync_fini(void)
425 {
426 	kmem_cache_destroy(amdgpu_sync_slab);
427 }
428