1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3  *
4  * Copyright 2011-2023 VMware, Inc., Palo Alto, CA., USA
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 
28 #include <linux/sched/signal.h>
29 
30 #include "vmwgfx_drv.h"
31 
32 #define VMW_FENCE_WRAP (1 << 31)
33 
34 struct vmw_fence_manager {
35 	int num_fence_objects;
36 	struct vmw_private *dev_priv;
37 	spinlock_t lock;
38 	struct list_head fence_list;
39 	struct work_struct work;
40 	bool fifo_down;
41 	struct list_head cleanup_list;
42 	uint32_t pending_actions[VMW_ACTION_MAX];
43 	struct mutex goal_irq_mutex;
44 	bool goal_irq_on; /* Protected by @goal_irq_mutex */
45 	bool seqno_valid; /* Protected by @lock, and may not be set to true
46 			     without the @goal_irq_mutex held. */
47 	u64 ctx;
48 };
49 
50 struct vmw_user_fence {
51 	struct ttm_base_object base;
52 	struct vmw_fence_obj fence;
53 };
54 
55 /**
56  * struct vmw_event_fence_action - fence action that delivers a drm event.
57  *
58  * @action: A struct vmw_fence_action to hook up to a fence.
59  * @event: A pointer to the pending event.
60  * @fence: A referenced pointer to the fence to keep it alive while @action
61  * hangs on it.
62  * @dev: Pointer to a struct drm_device so we can access the event stuff.
63  * @tv_sec: If non-null, the variable pointed to will be assigned
64  * current time tv_sec val when the fence signals.
65  * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will
66  * be assigned the current time tv_usec val when the fence signals.
67  */
68 struct vmw_event_fence_action {
69 	struct vmw_fence_action action;
70 
71 	struct drm_pending_event *event;
72 	struct vmw_fence_obj *fence;
73 	struct drm_device *dev;
74 
75 	uint32_t *tv_sec;
76 	uint32_t *tv_usec;
77 };
78 
79 static struct vmw_fence_manager *
80 fman_from_fence(struct vmw_fence_obj *fence)
81 {
82 	return container_of(fence->base.lock, struct vmw_fence_manager, lock);
83 }
84 
85 static u32 vmw_fence_goal_read(struct vmw_private *vmw)
86 {
87 	if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
88 		return vmw_read(vmw, SVGA_REG_FENCE_GOAL);
89 	else
90 		return vmw_fifo_mem_read(vmw, SVGA_FIFO_FENCE_GOAL);
91 }
92 
93 static void vmw_fence_goal_write(struct vmw_private *vmw, u32 value)
94 {
95 	if ((vmw->capabilities2 & SVGA_CAP2_EXTRA_REGS) != 0)
96 		vmw_write(vmw, SVGA_REG_FENCE_GOAL, value);
97 	else
98 		vmw_fifo_mem_write(vmw, SVGA_FIFO_FENCE_GOAL, value);
99 }
100 
101 /*
102  * Note on fencing subsystem usage of irqs:
103  * Typically the vmw_fences_update function is called
104  *
105  * a) When a new fence seqno has been submitted by the fifo code.
106  * b) On-demand when we have waiters. Sleeping waiters will switch on the
107  * ANY_FENCE irq and call vmw_fences_update function each time an ANY_FENCE
108  * irq is received. When the last fence waiter is gone, that IRQ is masked
109  * away.
110  *
111  * In situations where there are no waiters and we don't submit any new fences,
112  * fence objects may not be signaled. This is perfectly OK, since there are
113  * no consumers of the signaled data, but that is NOT ok when there are fence
114  * actions attached to a fence. The fencing subsystem then makes use of the
115  * FENCE_GOAL irq and sets the fence goal seqno to that of the next fence
116  * which has an action attached, and each time vmw_fences_update is called,
117  * the subsystem makes sure the fence goal seqno is updated.
118  *
119  * The fence goal seqno irq is on as long as there are unsignaled fence
120  * objects with actions attached to them.
121  */
122 
123 static void vmw_fence_obj_destroy(struct dma_fence *f)
124 {
125 	struct vmw_fence_obj *fence =
126 		container_of(f, struct vmw_fence_obj, base);
127 
128 	struct vmw_fence_manager *fman = fman_from_fence(fence);
129 
130 	spin_lock(&fman->lock);
131 	list_del_init(&fence->head);
132 	--fman->num_fence_objects;
133 	spin_unlock(&fman->lock);
134 	fence->destroy(fence);
135 }
136 
137 static const char *vmw_fence_get_driver_name(struct dma_fence *f)
138 {
139 	return "vmwgfx";
140 }
141 
142 static const char *vmw_fence_get_timeline_name(struct dma_fence *f)
143 {
144 	return "svga";
145 }
146 
147 static bool vmw_fence_enable_signaling(struct dma_fence *f)
148 {
149 	struct vmw_fence_obj *fence =
150 		container_of(f, struct vmw_fence_obj, base);
151 
152 	struct vmw_fence_manager *fman = fman_from_fence(fence);
153 	struct vmw_private *dev_priv = fman->dev_priv;
154 
155 	u32 seqno = vmw_fence_read(dev_priv);
156 	if (seqno - fence->base.seqno < VMW_FENCE_WRAP)
157 		return false;
158 
159 	return true;
160 }
161 
162 struct vmwgfx_wait_cb {
163 	struct dma_fence_cb base;
164 	struct task_struct *task;
165 };
166 
167 static void
168 vmwgfx_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
169 {
170 	struct vmwgfx_wait_cb *wait =
171 		container_of(cb, struct vmwgfx_wait_cb, base);
172 
173 	wake_up_process(wait->task);
174 }
175 
176 static void __vmw_fences_update(struct vmw_fence_manager *fman);
177 
178 static long vmw_fence_wait(struct dma_fence *f, bool intr, signed long timeout)
179 {
180 	struct vmw_fence_obj *fence =
181 		container_of(f, struct vmw_fence_obj, base);
182 
183 	struct vmw_fence_manager *fman = fman_from_fence(fence);
184 	struct vmw_private *dev_priv = fman->dev_priv;
185 	struct vmwgfx_wait_cb cb;
186 	long ret = timeout;
187 
188 	if (likely(vmw_fence_obj_signaled(fence)))
189 		return timeout;
190 
191 	vmw_seqno_waiter_add(dev_priv);
192 
193 	spin_lock(f->lock);
194 
195 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
196 		goto out;
197 
198 	if (intr && signal_pending(current)) {
199 		ret = -ERESTARTSYS;
200 		goto out;
201 	}
202 
203 	cb.base.func = vmwgfx_wait_cb;
204 	cb.task = current;
205 	list_add(&cb.base.node, &f->cb_list);
206 
207 	for (;;) {
208 		__vmw_fences_update(fman);
209 
210 		/*
211 		 * We can use the barrier free __set_current_state() since
212 		 * DMA_FENCE_FLAG_SIGNALED_BIT + wakeup is protected by the
213 		 * fence spinlock.
214 		 */
215 		if (intr)
216 			__set_current_state(TASK_INTERRUPTIBLE);
217 		else
218 			__set_current_state(TASK_UNINTERRUPTIBLE);
219 
220 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags)) {
221 			if (ret == 0 && timeout > 0)
222 				ret = 1;
223 			break;
224 		}
225 
226 		if (intr && signal_pending(current)) {
227 			ret = -ERESTARTSYS;
228 			break;
229 		}
230 
231 		if (ret == 0)
232 			break;
233 
234 		spin_unlock(f->lock);
235 
236 		ret = schedule_timeout(ret);
237 
238 		spin_lock(f->lock);
239 	}
240 	__set_current_state(TASK_RUNNING);
241 	if (!list_empty(&cb.base.node))
242 		list_del(&cb.base.node);
243 
244 out:
245 	spin_unlock(f->lock);
246 
247 	vmw_seqno_waiter_remove(dev_priv);
248 
249 	return ret;
250 }
251 
252 static const struct dma_fence_ops vmw_fence_ops = {
253 	.get_driver_name = vmw_fence_get_driver_name,
254 	.get_timeline_name = vmw_fence_get_timeline_name,
255 	.enable_signaling = vmw_fence_enable_signaling,
256 	.wait = vmw_fence_wait,
257 	.release = vmw_fence_obj_destroy,
258 };
259 
260 
261 /*
262  * Execute signal actions on fences recently signaled.
263  * This is done from a workqueue so we don't have to execute
264  * signal actions from atomic context.
265  */
266 
267 static void vmw_fence_work_func(struct work_struct *work)
268 {
269 	struct vmw_fence_manager *fman =
270 		container_of(work, struct vmw_fence_manager, work);
271 	struct list_head list;
272 	struct vmw_fence_action *action, *next_action;
273 	bool seqno_valid;
274 
275 	do {
276 		INIT_LIST_HEAD(&list);
277 		mutex_lock(&fman->goal_irq_mutex);
278 
279 		spin_lock(&fman->lock);
280 		list_splice_init(&fman->cleanup_list, &list);
281 		seqno_valid = fman->seqno_valid;
282 		spin_unlock(&fman->lock);
283 
284 		if (!seqno_valid && fman->goal_irq_on) {
285 			fman->goal_irq_on = false;
286 			vmw_goal_waiter_remove(fman->dev_priv);
287 		}
288 		mutex_unlock(&fman->goal_irq_mutex);
289 
290 		if (list_empty(&list))
291 			return;
292 
293 		/*
294 		 * At this point, only we should be able to manipulate the
295 		 * list heads of the actions we have on the private list.
296 		 * hence fman::lock not held.
297 		 */
298 
299 		list_for_each_entry_safe(action, next_action, &list, head) {
300 			list_del_init(&action->head);
301 			if (action->cleanup)
302 				action->cleanup(action);
303 		}
304 	} while (1);
305 }
306 
307 struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv)
308 {
309 	struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);
310 
311 	if (unlikely(!fman))
312 		return NULL;
313 
314 	fman->dev_priv = dev_priv;
315 	spin_lock_init(&fman->lock);
316 	INIT_LIST_HEAD(&fman->fence_list);
317 	INIT_LIST_HEAD(&fman->cleanup_list);
318 	INIT_WORK(&fman->work, &vmw_fence_work_func);
319 	fman->fifo_down = true;
320 	mutex_init(&fman->goal_irq_mutex);
321 	fman->ctx = dma_fence_context_alloc(1);
322 
323 	return fman;
324 }
325 
326 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)
327 {
328 	bool lists_empty;
329 
330 	(void) cancel_work_sync(&fman->work);
331 
332 	spin_lock(&fman->lock);
333 	lists_empty = list_empty(&fman->fence_list) &&
334 		list_empty(&fman->cleanup_list);
335 	spin_unlock(&fman->lock);
336 
337 	BUG_ON(!lists_empty);
338 	kfree(fman);
339 }
340 
341 static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
342 			      struct vmw_fence_obj *fence, u32 seqno,
343 			      void (*destroy) (struct vmw_fence_obj *fence))
344 {
345 	int ret = 0;
346 
347 	dma_fence_init(&fence->base, &vmw_fence_ops, &fman->lock,
348 		       fman->ctx, seqno);
349 	INIT_LIST_HEAD(&fence->seq_passed_actions);
350 	fence->destroy = destroy;
351 
352 	spin_lock(&fman->lock);
353 	if (unlikely(fman->fifo_down)) {
354 		ret = -EBUSY;
355 		goto out_unlock;
356 	}
357 	list_add_tail(&fence->head, &fman->fence_list);
358 	++fman->num_fence_objects;
359 
360 out_unlock:
361 	spin_unlock(&fman->lock);
362 	return ret;
363 
364 }
365 
366 static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,
367 				struct list_head *list)
368 {
369 	struct vmw_fence_action *action, *next_action;
370 
371 	list_for_each_entry_safe(action, next_action, list, head) {
372 		list_del_init(&action->head);
373 		fman->pending_actions[action->type]--;
374 		if (action->seq_passed != NULL)
375 			action->seq_passed(action);
376 
377 		/*
378 		 * Add the cleanup action to the cleanup list so that
379 		 * it will be performed by a worker task.
380 		 */
381 
382 		list_add_tail(&action->head, &fman->cleanup_list);
383 	}
384 }
385 
386 /**
387  * vmw_fence_goal_new_locked - Figure out a new device fence goal
388  * seqno if needed.
389  *
390  * @fman: Pointer to a fence manager.
391  * @passed_seqno: The seqno the device currently signals as passed.
392  *
393  * This function should be called with the fence manager lock held.
394  * It is typically called when we have a new passed_seqno, and
395  * we might need to update the fence goal. It checks to see whether
396  * the current fence goal has already passed, and, in that case,
397  * scans through all unsignaled fences to get the next fence object with an
398  * action attached, and sets the seqno of that fence as a new fence goal.
399  *
400  * returns true if the device goal seqno was updated. False otherwise.
401  */
402 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,
403 				      u32 passed_seqno)
404 {
405 	u32 goal_seqno;
406 	struct vmw_fence_obj *fence;
407 
408 	if (likely(!fman->seqno_valid))
409 		return false;
410 
411 	goal_seqno = vmw_fence_goal_read(fman->dev_priv);
412 	if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP))
413 		return false;
414 
415 	fman->seqno_valid = false;
416 	list_for_each_entry(fence, &fman->fence_list, head) {
417 		if (!list_empty(&fence->seq_passed_actions)) {
418 			fman->seqno_valid = true;
419 			vmw_fence_goal_write(fman->dev_priv,
420 					     fence->base.seqno);
421 			break;
422 		}
423 	}
424 
425 	return true;
426 }
427 
428 
429 /**
430  * vmw_fence_goal_check_locked - Replace the device fence goal seqno if
431  * needed.
432  *
433  * @fence: Pointer to a struct vmw_fence_obj the seqno of which should be
434  * considered as a device fence goal.
435  *
436  * This function should be called with the fence manager lock held.
437  * It is typically called when an action has been attached to a fence to
438  * check whether the seqno of that fence should be used for a fence
439  * goal interrupt. This is typically needed if the current fence goal is
440  * invalid, or has a higher seqno than that of the current fence object.
441  *
442  * returns true if the device goal seqno was updated. False otherwise.
443  */
444 static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence)
445 {
446 	struct vmw_fence_manager *fman = fman_from_fence(fence);
447 	u32 goal_seqno;
448 
449 	if (dma_fence_is_signaled_locked(&fence->base))
450 		return false;
451 
452 	goal_seqno = vmw_fence_goal_read(fman->dev_priv);
453 	if (likely(fman->seqno_valid &&
454 		   goal_seqno - fence->base.seqno < VMW_FENCE_WRAP))
455 		return false;
456 
457 	vmw_fence_goal_write(fman->dev_priv, fence->base.seqno);
458 	fman->seqno_valid = true;
459 
460 	return true;
461 }
462 
463 static void __vmw_fences_update(struct vmw_fence_manager *fman)
464 {
465 	struct vmw_fence_obj *fence, *next_fence;
466 	struct list_head action_list;
467 	bool needs_rerun;
468 	uint32_t seqno, new_seqno;
469 
470 	seqno = vmw_fence_read(fman->dev_priv);
471 rerun:
472 	list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {
473 		if (seqno - fence->base.seqno < VMW_FENCE_WRAP) {
474 			list_del_init(&fence->head);
475 			dma_fence_signal_locked(&fence->base);
476 			INIT_LIST_HEAD(&action_list);
477 			list_splice_init(&fence->seq_passed_actions,
478 					 &action_list);
479 			vmw_fences_perform_actions(fman, &action_list);
480 		} else
481 			break;
482 	}
483 
484 	/*
485 	 * Rerun if the fence goal seqno was updated, and the
486 	 * hardware might have raced with that update, so that
487 	 * we missed a fence_goal irq.
488 	 */
489 
490 	needs_rerun = vmw_fence_goal_new_locked(fman, seqno);
491 	if (unlikely(needs_rerun)) {
492 		new_seqno = vmw_fence_read(fman->dev_priv);
493 		if (new_seqno != seqno) {
494 			seqno = new_seqno;
495 			goto rerun;
496 		}
497 	}
498 
499 	if (!list_empty(&fman->cleanup_list))
500 		(void) schedule_work(&fman->work);
501 }
502 
503 void vmw_fences_update(struct vmw_fence_manager *fman)
504 {
505 	spin_lock(&fman->lock);
506 	__vmw_fences_update(fman);
507 	spin_unlock(&fman->lock);
508 }
509 
510 bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence)
511 {
512 	struct vmw_fence_manager *fman = fman_from_fence(fence);
513 
514 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
515 		return true;
516 
517 	vmw_fences_update(fman);
518 
519 	return dma_fence_is_signaled(&fence->base);
520 }
521 
522 int vmw_fence_obj_wait(struct vmw_fence_obj *fence, bool lazy,
523 		       bool interruptible, unsigned long timeout)
524 {
525 	long ret = dma_fence_wait_timeout(&fence->base, interruptible, timeout);
526 
527 	if (likely(ret > 0))
528 		return 0;
529 	else if (ret == 0)
530 		return -EBUSY;
531 	else
532 		return ret;
533 }
534 
535 static void vmw_fence_destroy(struct vmw_fence_obj *fence)
536 {
537 	dma_fence_free(&fence->base);
538 }
539 
540 int vmw_fence_create(struct vmw_fence_manager *fman,
541 		     uint32_t seqno,
542 		     struct vmw_fence_obj **p_fence)
543 {
544 	struct vmw_fence_obj *fence;
545  	int ret;
546 
547 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
548 	if (unlikely(!fence))
549 		return -ENOMEM;
550 
551 	ret = vmw_fence_obj_init(fman, fence, seqno,
552 				 vmw_fence_destroy);
553 	if (unlikely(ret != 0))
554 		goto out_err_init;
555 
556 	*p_fence = fence;
557 	return 0;
558 
559 out_err_init:
560 	kfree(fence);
561 	return ret;
562 }
563 
564 
565 static void vmw_user_fence_destroy(struct vmw_fence_obj *fence)
566 {
567 	struct vmw_user_fence *ufence =
568 		container_of(fence, struct vmw_user_fence, fence);
569 
570 	ttm_base_object_kfree(ufence, base);
571 }
572 
573 static void vmw_user_fence_base_release(struct ttm_base_object **p_base)
574 {
575 	struct ttm_base_object *base = *p_base;
576 	struct vmw_user_fence *ufence =
577 		container_of(base, struct vmw_user_fence, base);
578 	struct vmw_fence_obj *fence = &ufence->fence;
579 
580 	*p_base = NULL;
581 	vmw_fence_obj_unreference(&fence);
582 }
583 
584 int vmw_user_fence_create(struct drm_file *file_priv,
585 			  struct vmw_fence_manager *fman,
586 			  uint32_t seqno,
587 			  struct vmw_fence_obj **p_fence,
588 			  uint32_t *p_handle)
589 {
590 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
591 	struct vmw_user_fence *ufence;
592 	struct vmw_fence_obj *tmp;
593 	int ret;
594 
595 	ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);
596 	if (unlikely(!ufence)) {
597 		ret = -ENOMEM;
598 		goto out_no_object;
599 	}
600 
601 	ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,
602 				 vmw_user_fence_destroy);
603 	if (unlikely(ret != 0)) {
604 		kfree(ufence);
605 		goto out_no_object;
606 	}
607 
608 	/*
609 	 * The base object holds a reference which is freed in
610 	 * vmw_user_fence_base_release.
611 	 */
612 	tmp = vmw_fence_obj_reference(&ufence->fence);
613 
614 	ret = ttm_base_object_init(tfile, &ufence->base, false,
615 				   VMW_RES_FENCE,
616 				   &vmw_user_fence_base_release);
617 
618 
619 	if (unlikely(ret != 0)) {
620 		/*
621 		 * Free the base object's reference
622 		 */
623 		vmw_fence_obj_unreference(&tmp);
624 		goto out_err;
625 	}
626 
627 	*p_fence = &ufence->fence;
628 	*p_handle = ufence->base.handle;
629 
630 	return 0;
631 out_err:
632 	tmp = &ufence->fence;
633 	vmw_fence_obj_unreference(&tmp);
634 out_no_object:
635 	return ret;
636 }
637 
638 /*
639  * vmw_fence_fifo_down - signal all unsignaled fence objects.
640  */
641 
642 void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
643 {
644 	struct list_head action_list;
645 	int ret;
646 
647 	/*
648 	 * The list may be altered while we traverse it, so always
649 	 * restart when we've released the fman->lock.
650 	 */
651 
652 	spin_lock(&fman->lock);
653 	fman->fifo_down = true;
654 	while (!list_empty(&fman->fence_list)) {
655 		struct vmw_fence_obj *fence =
656 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,
657 				   head);
658 		dma_fence_get(&fence->base);
659 		spin_unlock(&fman->lock);
660 
661 		ret = vmw_fence_obj_wait(fence, false, false,
662 					 VMW_FENCE_WAIT_TIMEOUT);
663 
664 		if (unlikely(ret != 0)) {
665 			list_del_init(&fence->head);
666 			dma_fence_signal(&fence->base);
667 			INIT_LIST_HEAD(&action_list);
668 			list_splice_init(&fence->seq_passed_actions,
669 					 &action_list);
670 			vmw_fences_perform_actions(fman, &action_list);
671 		}
672 
673 		BUG_ON(!list_empty(&fence->head));
674 		dma_fence_put(&fence->base);
675 		spin_lock(&fman->lock);
676 	}
677 	spin_unlock(&fman->lock);
678 }
679 
680 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)
681 {
682 	spin_lock(&fman->lock);
683 	fman->fifo_down = false;
684 	spin_unlock(&fman->lock);
685 }
686 
687 
688 /**
689  * vmw_fence_obj_lookup - Look up a user-space fence object
690  *
691  * @tfile: A struct ttm_object_file identifying the caller.
692  * @handle: A handle identifying the fence object.
693  * @return: A struct vmw_user_fence base ttm object on success or
694  * an error pointer on failure.
695  *
696  * The fence object is looked up and type-checked. The caller needs
697  * to have opened the fence object first, but since that happens on
698  * creation and fence objects aren't shareable, that's not an
699  * issue currently.
700  */
701 static struct ttm_base_object *
702 vmw_fence_obj_lookup(struct ttm_object_file *tfile, u32 handle)
703 {
704 	struct ttm_base_object *base = ttm_base_object_lookup(tfile, handle);
705 
706 	if (!base) {
707 		pr_err("Invalid fence object handle 0x%08lx.\n",
708 		       (unsigned long)handle);
709 		return ERR_PTR(-EINVAL);
710 	}
711 
712 	if (base->refcount_release != vmw_user_fence_base_release) {
713 		pr_err("Invalid fence object handle 0x%08lx.\n",
714 		       (unsigned long)handle);
715 		ttm_base_object_unref(&base);
716 		return ERR_PTR(-EINVAL);
717 	}
718 
719 	return base;
720 }
721 
722 
723 int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data,
724 			     struct drm_file *file_priv)
725 {
726 	struct drm_vmw_fence_wait_arg *arg =
727 	    (struct drm_vmw_fence_wait_arg *)data;
728 	unsigned long timeout;
729 	struct ttm_base_object *base;
730 	struct vmw_fence_obj *fence;
731 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
732 	int ret;
733 	uint64_t wait_timeout = ((uint64_t)arg->timeout_us * HZ);
734 
735 	/*
736 	 * 64-bit division not present on 32-bit systems, so do an
737 	 * approximation. (Divide by 1000000).
738 	 */
739 
740 	wait_timeout = (wait_timeout >> 20) + (wait_timeout >> 24) -
741 	  (wait_timeout >> 26);
742 
743 	if (!arg->cookie_valid) {
744 		arg->cookie_valid = 1;
745 		arg->kernel_cookie = jiffies + wait_timeout;
746 	}
747 
748 	base = vmw_fence_obj_lookup(tfile, arg->handle);
749 	if (IS_ERR(base))
750 		return PTR_ERR(base);
751 
752 	fence = &(container_of(base, struct vmw_user_fence, base)->fence);
753 
754 	timeout = jiffies;
755 	if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) {
756 		ret = ((vmw_fence_obj_signaled(fence)) ?
757 		       0 : -EBUSY);
758 		goto out;
759 	}
760 
761 	timeout = (unsigned long)arg->kernel_cookie - timeout;
762 
763 	ret = vmw_fence_obj_wait(fence, arg->lazy, true, timeout);
764 
765 out:
766 	ttm_base_object_unref(&base);
767 
768 	/*
769 	 * Optionally unref the fence object.
770 	 */
771 
772 	if (ret == 0 && (arg->wait_options & DRM_VMW_WAIT_OPTION_UNREF))
773 		return ttm_ref_object_base_unref(tfile, arg->handle);
774 	return ret;
775 }
776 
777 int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data,
778 				 struct drm_file *file_priv)
779 {
780 	struct drm_vmw_fence_signaled_arg *arg =
781 		(struct drm_vmw_fence_signaled_arg *) data;
782 	struct ttm_base_object *base;
783 	struct vmw_fence_obj *fence;
784 	struct vmw_fence_manager *fman;
785 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
786 	struct vmw_private *dev_priv = vmw_priv(dev);
787 
788 	base = vmw_fence_obj_lookup(tfile, arg->handle);
789 	if (IS_ERR(base))
790 		return PTR_ERR(base);
791 
792 	fence = &(container_of(base, struct vmw_user_fence, base)->fence);
793 	fman = fman_from_fence(fence);
794 
795 	arg->signaled = vmw_fence_obj_signaled(fence);
796 
797 	arg->signaled_flags = arg->flags;
798 	spin_lock(&fman->lock);
799 	arg->passed_seqno = dev_priv->last_read_seqno;
800 	spin_unlock(&fman->lock);
801 
802 	ttm_base_object_unref(&base);
803 
804 	return 0;
805 }
806 
807 
808 int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data,
809 			      struct drm_file *file_priv)
810 {
811 	struct drm_vmw_fence_arg *arg =
812 		(struct drm_vmw_fence_arg *) data;
813 
814 	return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
815 					 arg->handle);
816 }
817 
818 /**
819  * vmw_event_fence_action_seq_passed
820  *
821  * @action: The struct vmw_fence_action embedded in a struct
822  * vmw_event_fence_action.
823  *
824  * This function is called when the seqno of the fence where @action is
825  * attached has passed. It queues the event on the submitter's event list.
826  * This function is always called from atomic context.
827  */
828 static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
829 {
830 	struct vmw_event_fence_action *eaction =
831 		container_of(action, struct vmw_event_fence_action, action);
832 	struct drm_device *dev = eaction->dev;
833 	struct drm_pending_event *event = eaction->event;
834 
835 	if (unlikely(event == NULL))
836 		return;
837 
838 	spin_lock_irq(&dev->event_lock);
839 
840 	if (likely(eaction->tv_sec != NULL)) {
841 		struct timespec64 ts;
842 
843 		ktime_get_ts64(&ts);
844 		/* monotonic time, so no y2038 overflow */
845 		*eaction->tv_sec = ts.tv_sec;
846 		*eaction->tv_usec = ts.tv_nsec / NSEC_PER_USEC;
847 	}
848 
849 	drm_send_event_locked(dev, eaction->event);
850 	eaction->event = NULL;
851 	spin_unlock_irq(&dev->event_lock);
852 }
853 
854 /**
855  * vmw_event_fence_action_cleanup
856  *
857  * @action: The struct vmw_fence_action embedded in a struct
858  * vmw_event_fence_action.
859  *
860  * This function is the struct vmw_fence_action destructor. It's typically
861  * called from a workqueue.
862  */
863 static void vmw_event_fence_action_cleanup(struct vmw_fence_action *action)
864 {
865 	struct vmw_event_fence_action *eaction =
866 		container_of(action, struct vmw_event_fence_action, action);
867 
868 	vmw_fence_obj_unreference(&eaction->fence);
869 	kfree(eaction);
870 }
871 
872 
873 /**
874  * vmw_fence_obj_add_action - Add an action to a fence object.
875  *
876  * @fence: The fence object.
877  * @action: The action to add.
878  *
879  * Note that the action callbacks may be executed before this function
880  * returns.
881  */
882 static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
883 			      struct vmw_fence_action *action)
884 {
885 	struct vmw_fence_manager *fman = fman_from_fence(fence);
886 	bool run_update = false;
887 
888 	mutex_lock(&fman->goal_irq_mutex);
889 	spin_lock(&fman->lock);
890 
891 	fman->pending_actions[action->type]++;
892 	if (dma_fence_is_signaled_locked(&fence->base)) {
893 		struct list_head action_list;
894 
895 		INIT_LIST_HEAD(&action_list);
896 		list_add_tail(&action->head, &action_list);
897 		vmw_fences_perform_actions(fman, &action_list);
898 	} else {
899 		list_add_tail(&action->head, &fence->seq_passed_actions);
900 
901 		/*
902 		 * This function may set fman::seqno_valid, so it must
903 		 * be run with the goal_irq_mutex held.
904 		 */
905 		run_update = vmw_fence_goal_check_locked(fence);
906 	}
907 
908 	spin_unlock(&fman->lock);
909 
910 	if (run_update) {
911 		if (!fman->goal_irq_on) {
912 			fman->goal_irq_on = true;
913 			vmw_goal_waiter_add(fman->dev_priv);
914 		}
915 		vmw_fences_update(fman);
916 	}
917 	mutex_unlock(&fman->goal_irq_mutex);
918 
919 }
920 
921 /**
922  * vmw_event_fence_action_queue - Post an event for sending when a fence
923  * object seqno has passed.
924  *
925  * @file_priv: The file connection on which the event should be posted.
926  * @fence: The fence object on which to post the event.
927  * @event: Event to be posted. This event should've been alloced
928  * using k[mz]alloc, and should've been completely initialized.
929  * @tv_sec: If non-null, the variable pointed to will be assigned
930  * current time tv_sec val when the fence signals.
931  * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will
932  * be assigned the current time tv_usec val when the fence signals.
933  * @interruptible: Interruptible waits if possible.
934  *
935  * As a side effect, the object pointed to by @event may have been
936  * freed when this function returns. If this function returns with
937  * an error code, the caller needs to free that object.
938  */
939 
940 int vmw_event_fence_action_queue(struct drm_file *file_priv,
941 				 struct vmw_fence_obj *fence,
942 				 struct drm_pending_event *event,
943 				 uint32_t *tv_sec,
944 				 uint32_t *tv_usec,
945 				 bool interruptible)
946 {
947 	struct vmw_event_fence_action *eaction;
948 	struct vmw_fence_manager *fman = fman_from_fence(fence);
949 
950 	eaction = kzalloc(sizeof(*eaction), GFP_KERNEL);
951 	if (unlikely(!eaction))
952 		return -ENOMEM;
953 
954 	eaction->event = event;
955 
956 	eaction->action.seq_passed = vmw_event_fence_action_seq_passed;
957 	eaction->action.cleanup = vmw_event_fence_action_cleanup;
958 	eaction->action.type = VMW_ACTION_EVENT;
959 
960 	eaction->fence = vmw_fence_obj_reference(fence);
961 	eaction->dev = &fman->dev_priv->drm;
962 	eaction->tv_sec = tv_sec;
963 	eaction->tv_usec = tv_usec;
964 
965 	vmw_fence_obj_add_action(fence, &eaction->action);
966 
967 	return 0;
968 }
969 
970 struct vmw_event_fence_pending {
971 	struct drm_pending_event base;
972 	struct drm_vmw_event_fence event;
973 };
974 
975 static int vmw_event_fence_action_create(struct drm_file *file_priv,
976 				  struct vmw_fence_obj *fence,
977 				  uint32_t flags,
978 				  uint64_t user_data,
979 				  bool interruptible)
980 {
981 	struct vmw_event_fence_pending *event;
982 	struct vmw_fence_manager *fman = fman_from_fence(fence);
983 	struct drm_device *dev = &fman->dev_priv->drm;
984 	int ret;
985 
986 	event = kzalloc(sizeof(*event), GFP_KERNEL);
987 	if (unlikely(!event)) {
988 		DRM_ERROR("Failed to allocate an event.\n");
989 		ret = -ENOMEM;
990 		goto out_no_space;
991 	}
992 
993 	event->event.base.type = DRM_VMW_EVENT_FENCE_SIGNALED;
994 	event->event.base.length = sizeof(*event);
995 	event->event.user_data = user_data;
996 
997 	ret = drm_event_reserve_init(dev, file_priv, &event->base, &event->event.base);
998 
999 	if (unlikely(ret != 0)) {
1000 		DRM_ERROR("Failed to allocate event space for this file.\n");
1001 		kfree(event);
1002 		goto out_no_space;
1003 	}
1004 
1005 	if (flags & DRM_VMW_FE_FLAG_REQ_TIME)
1006 		ret = vmw_event_fence_action_queue(file_priv, fence,
1007 						   &event->base,
1008 						   &event->event.tv_sec,
1009 						   &event->event.tv_usec,
1010 						   interruptible);
1011 	else
1012 		ret = vmw_event_fence_action_queue(file_priv, fence,
1013 						   &event->base,
1014 						   NULL,
1015 						   NULL,
1016 						   interruptible);
1017 	if (ret != 0)
1018 		goto out_no_queue;
1019 
1020 	return 0;
1021 
1022 out_no_queue:
1023 	drm_event_cancel_free(dev, &event->base);
1024 out_no_space:
1025 	return ret;
1026 }
1027 
1028 int vmw_fence_event_ioctl(struct drm_device *dev, void *data,
1029 			  struct drm_file *file_priv)
1030 {
1031 	struct vmw_private *dev_priv = vmw_priv(dev);
1032 	struct drm_vmw_fence_event_arg *arg =
1033 		(struct drm_vmw_fence_event_arg *) data;
1034 	struct vmw_fence_obj *fence = NULL;
1035 	struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
1036 	struct ttm_object_file *tfile = vmw_fp->tfile;
1037 	struct drm_vmw_fence_rep __user *user_fence_rep =
1038 		(struct drm_vmw_fence_rep __user *)(unsigned long)
1039 		arg->fence_rep;
1040 	uint32_t handle;
1041 	int ret;
1042 
1043 	/*
1044 	 * Look up an existing fence object,
1045 	 * and if user-space wants a new reference,
1046 	 * add one.
1047 	 */
1048 	if (arg->handle) {
1049 		struct ttm_base_object *base =
1050 			vmw_fence_obj_lookup(tfile, arg->handle);
1051 
1052 		if (IS_ERR(base))
1053 			return PTR_ERR(base);
1054 
1055 		fence = &(container_of(base, struct vmw_user_fence,
1056 				       base)->fence);
1057 		(void) vmw_fence_obj_reference(fence);
1058 
1059 		if (user_fence_rep != NULL) {
1060 			ret = ttm_ref_object_add(vmw_fp->tfile, base,
1061 						 NULL, false);
1062 			if (unlikely(ret != 0)) {
1063 				DRM_ERROR("Failed to reference a fence "
1064 					  "object.\n");
1065 				goto out_no_ref_obj;
1066 			}
1067 			handle = base->handle;
1068 		}
1069 		ttm_base_object_unref(&base);
1070 	}
1071 
1072 	/*
1073 	 * Create a new fence object.
1074 	 */
1075 	if (!fence) {
1076 		ret = vmw_execbuf_fence_commands(file_priv, dev_priv,
1077 						 &fence,
1078 						 (user_fence_rep) ?
1079 						 &handle : NULL);
1080 		if (unlikely(ret != 0)) {
1081 			DRM_ERROR("Fence event failed to create fence.\n");
1082 			return ret;
1083 		}
1084 	}
1085 
1086 	BUG_ON(fence == NULL);
1087 
1088 	ret = vmw_event_fence_action_create(file_priv, fence,
1089 					    arg->flags,
1090 					    arg->user_data,
1091 					    true);
1092 	if (unlikely(ret != 0)) {
1093 		if (ret != -ERESTARTSYS)
1094 			DRM_ERROR("Failed to attach event to fence.\n");
1095 		goto out_no_create;
1096 	}
1097 
1098 	vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence,
1099 				    handle, -1);
1100 	vmw_fence_obj_unreference(&fence);
1101 	return 0;
1102 out_no_create:
1103 	if (user_fence_rep != NULL)
1104 		ttm_ref_object_base_unref(tfile, handle);
1105 out_no_ref_obj:
1106 	vmw_fence_obj_unreference(&fence);
1107 	return ret;
1108 }
1109