1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Christian König <christian.koenig@amd.com>
29  */
30 
31 #include <linux/dma-fence-chain.h>
32 
33 #include "amdgpu.h"
34 #include "amdgpu_trace.h"
35 #include "amdgpu_amdkfd.h"
36 
37 struct amdgpu_sync_entry {
38 	struct hlist_node	node;
39 	struct dma_fence	*fence;
40 };
41 
42 static struct kmem_cache *amdgpu_sync_slab;
43 
44 /**
45  * amdgpu_sync_create - zero init sync object
46  *
47  * @sync: sync object to initialize
48  *
49  * Just clear the sync object for now.
50  */
51 void amdgpu_sync_create(struct amdgpu_sync *sync)
52 {
53 	hash_init(sync->fences);
54 }
55 
56 /**
57  * amdgpu_sync_same_dev - test if fence belong to us
58  *
59  * @adev: amdgpu device to use for the test
60  * @f: fence to test
61  *
62  * Test if the fence was issued by us.
63  */
64 static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,
65 				 struct dma_fence *f)
66 {
67 	struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
68 
69 	if (s_fence) {
70 		struct amdgpu_ring *ring;
71 
72 		ring = container_of(s_fence->sched, struct amdgpu_ring, sched);
73 		return ring->adev == adev;
74 	}
75 
76 	return false;
77 }
78 
79 /**
80  * amdgpu_sync_get_owner - extract the owner of a fence
81  *
82  * @f: fence get the owner from
83  *
84  * Extract who originally created the fence.
85  */
86 static void *amdgpu_sync_get_owner(struct dma_fence *f)
87 {
88 	struct drm_sched_fence *s_fence;
89 	struct amdgpu_amdkfd_fence *kfd_fence;
90 
91 	if (!f)
92 		return AMDGPU_FENCE_OWNER_UNDEFINED;
93 
94 	s_fence = to_drm_sched_fence(f);
95 	if (s_fence)
96 		return s_fence->owner;
97 
98 	kfd_fence = to_amdgpu_amdkfd_fence(f);
99 	if (kfd_fence)
100 		return AMDGPU_FENCE_OWNER_KFD;
101 
102 	return AMDGPU_FENCE_OWNER_UNDEFINED;
103 }
104 
105 /**
106  * amdgpu_sync_keep_later - Keep the later fence
107  *
108  * @keep: existing fence to test
109  * @fence: new fence
110  *
111  * Either keep the existing fence or the new one, depending which one is later.
112  */
113 static void amdgpu_sync_keep_later(struct dma_fence **keep,
114 				   struct dma_fence *fence)
115 {
116 	if (*keep && dma_fence_is_later(*keep, fence))
117 		return;
118 
119 	dma_fence_put(*keep);
120 	*keep = dma_fence_get(fence);
121 }
122 
123 /**
124  * amdgpu_sync_add_later - add the fence to the hash
125  *
126  * @sync: sync object to add the fence to
127  * @f: fence to add
128  *
129  * Tries to add the fence to an existing hash entry. Returns true when an entry
130  * was found, false otherwise.
131  */
132 static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
133 {
134 	struct amdgpu_sync_entry *e;
135 
136 	hash_for_each_possible(sync->fences, e, node, f->context) {
137 		if (unlikely(e->fence->context != f->context))
138 			continue;
139 
140 		amdgpu_sync_keep_later(&e->fence, f);
141 		return true;
142 	}
143 	return false;
144 }
145 
146 /**
147  * amdgpu_sync_fence - remember to sync to this fence
148  *
149  * @sync: sync object to add fence to
150  * @f: fence to sync to
151  *
152  * Add the fence to the sync object.
153  */
154 int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
155 {
156 	struct amdgpu_sync_entry *e;
157 
158 	if (!f)
159 		return 0;
160 
161 	if (amdgpu_sync_add_later(sync, f))
162 		return 0;
163 
164 	e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
165 	if (!e)
166 		return -ENOMEM;
167 
168 	hash_add(sync->fences, &e->node, f->context);
169 	e->fence = dma_fence_get(f);
170 	return 0;
171 }
172 
173 /* Determine based on the owner and mode if we should sync to a fence or not */
174 static bool amdgpu_sync_test_fence(struct amdgpu_device *adev,
175 				   enum amdgpu_sync_mode mode,
176 				   void *owner, struct dma_fence *f)
177 {
178 	void *fence_owner = amdgpu_sync_get_owner(f);
179 
180 	/* Always sync to moves, no matter what */
181 	if (fence_owner == AMDGPU_FENCE_OWNER_UNDEFINED)
182 		return true;
183 
184 	/* We only want to trigger KFD eviction fences on
185 	 * evict or move jobs. Skip KFD fences otherwise.
186 	 */
187 	if (fence_owner == AMDGPU_FENCE_OWNER_KFD &&
188 	    owner != AMDGPU_FENCE_OWNER_UNDEFINED)
189 		return false;
190 
191 	/* Never sync to VM updates either. */
192 	if (fence_owner == AMDGPU_FENCE_OWNER_VM &&
193 	    owner != AMDGPU_FENCE_OWNER_UNDEFINED)
194 		return false;
195 
196 	/* Ignore fences depending on the sync mode */
197 	switch (mode) {
198 	case AMDGPU_SYNC_ALWAYS:
199 		return true;
200 
201 	case AMDGPU_SYNC_NE_OWNER:
202 		if (amdgpu_sync_same_dev(adev, f) &&
203 		    fence_owner == owner)
204 			return false;
205 		break;
206 
207 	case AMDGPU_SYNC_EQ_OWNER:
208 		if (amdgpu_sync_same_dev(adev, f) &&
209 		    fence_owner != owner)
210 			return false;
211 		break;
212 
213 	case AMDGPU_SYNC_EXPLICIT:
214 		return false;
215 	}
216 
217 	WARN(debug_evictions && fence_owner == AMDGPU_FENCE_OWNER_KFD,
218 	     "Adding eviction fence to sync obj");
219 	return true;
220 }
221 
222 /**
223  * amdgpu_sync_resv - sync to a reservation object
224  *
225  * @adev: amdgpu device
226  * @sync: sync object to add fences from reservation object to
227  * @resv: reservation object with embedded fence
228  * @mode: how owner affects which fences we sync to
229  * @owner: owner of the planned job submission
230  *
231  * Sync to the fence
232  */
233 int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
234 		     struct dma_resv *resv, enum amdgpu_sync_mode mode,
235 		     void *owner)
236 {
237 	struct dma_resv_iter cursor;
238 	struct dma_fence *f;
239 	int r;
240 
241 	if (resv == NULL)
242 		return -EINVAL;
243 
244 	/* TODO: Use DMA_RESV_USAGE_READ here */
245 	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, f) {
246 		dma_fence_chain_for_each(f, f) {
247 			struct dma_fence *tmp = dma_fence_chain_contained(f);
248 
249 			if (amdgpu_sync_test_fence(adev, mode, owner, tmp)) {
250 				r = amdgpu_sync_fence(sync, f);
251 				dma_fence_put(f);
252 				if (r)
253 					return r;
254 				break;
255 			}
256 		}
257 	}
258 	return 0;
259 }
260 
261 /**
262  * amdgpu_sync_peek_fence - get the next fence not signaled yet
263  *
264  * @sync: the sync object
265  * @ring: optional ring to use for test
266  *
267  * Returns the next fence not signaled yet without removing it from the sync
268  * object.
269  */
270 struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
271 					 struct amdgpu_ring *ring)
272 {
273 	struct amdgpu_sync_entry *e;
274 	struct hlist_node *tmp;
275 	int i;
276 
277 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
278 		struct dma_fence *f = e->fence;
279 		struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
280 
281 		if (dma_fence_is_signaled(f)) {
282 			hash_del(&e->node);
283 			dma_fence_put(f);
284 			kmem_cache_free(amdgpu_sync_slab, e);
285 			continue;
286 		}
287 		if (ring && s_fence) {
288 			/* For fences from the same ring it is sufficient
289 			 * when they are scheduled.
290 			 */
291 			if (s_fence->sched == &ring->sched) {
292 				if (dma_fence_is_signaled(&s_fence->scheduled))
293 					continue;
294 
295 				return &s_fence->scheduled;
296 			}
297 		}
298 
299 		return f;
300 	}
301 
302 	return NULL;
303 }
304 
305 /**
306  * amdgpu_sync_get_fence - get the next fence from the sync object
307  *
308  * @sync: sync object to use
309  *
310  * Get and removes the next fence from the sync object not signaled yet.
311  */
312 struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
313 {
314 	struct amdgpu_sync_entry *e;
315 	struct hlist_node *tmp;
316 	struct dma_fence *f;
317 	int i;
318 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
319 
320 		f = e->fence;
321 
322 		hash_del(&e->node);
323 		kmem_cache_free(amdgpu_sync_slab, e);
324 
325 		if (!dma_fence_is_signaled(f))
326 			return f;
327 
328 		dma_fence_put(f);
329 	}
330 	return NULL;
331 }
332 
333 /**
334  * amdgpu_sync_clone - clone a sync object
335  *
336  * @source: sync object to clone
337  * @clone: pointer to destination sync object
338  *
339  * Adds references to all unsignaled fences in @source to @clone. Also
340  * removes signaled fences from @source while at it.
341  */
342 int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone)
343 {
344 	struct amdgpu_sync_entry *e;
345 	struct hlist_node *tmp;
346 	struct dma_fence *f;
347 	int i, r;
348 
349 	hash_for_each_safe(source->fences, i, tmp, e, node) {
350 		f = e->fence;
351 		if (!dma_fence_is_signaled(f)) {
352 			r = amdgpu_sync_fence(clone, f);
353 			if (r)
354 				return r;
355 		} else {
356 			hash_del(&e->node);
357 			dma_fence_put(f);
358 			kmem_cache_free(amdgpu_sync_slab, e);
359 		}
360 	}
361 
362 	return 0;
363 }
364 
365 int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr)
366 {
367 	struct amdgpu_sync_entry *e;
368 	struct hlist_node *tmp;
369 	int i, r;
370 
371 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
372 		r = dma_fence_wait(e->fence, intr);
373 		if (r)
374 			return r;
375 
376 		hash_del(&e->node);
377 		dma_fence_put(e->fence);
378 		kmem_cache_free(amdgpu_sync_slab, e);
379 	}
380 
381 	return 0;
382 }
383 
384 /**
385  * amdgpu_sync_free - free the sync object
386  *
387  * @sync: sync object to use
388  *
389  * Free the sync object.
390  */
391 void amdgpu_sync_free(struct amdgpu_sync *sync)
392 {
393 	struct amdgpu_sync_entry *e;
394 	struct hlist_node *tmp;
395 	unsigned i;
396 
397 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
398 		hash_del(&e->node);
399 		dma_fence_put(e->fence);
400 		kmem_cache_free(amdgpu_sync_slab, e);
401 	}
402 }
403 
404 /**
405  * amdgpu_sync_init - init sync object subsystem
406  *
407  * Allocate the slab allocator.
408  */
409 int amdgpu_sync_init(void)
410 {
411 	amdgpu_sync_slab = kmem_cache_create(
412 		"amdgpu_sync", sizeof(struct amdgpu_sync_entry), 0,
413 		SLAB_HWCACHE_ALIGN, NULL);
414 	if (!amdgpu_sync_slab)
415 		return -ENOMEM;
416 
417 	return 0;
418 }
419 
420 /**
421  * amdgpu_sync_fini - fini sync object subsystem
422  *
423  * Free the slab allocator.
424  */
425 void amdgpu_sync_fini(void)
426 {
427 	kmem_cache_destroy(amdgpu_sync_slab);
428 }
429