1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3  *
4  * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 
28 #include <drm/ttm/ttm_placement.h>
29 
30 #include "vmwgfx_binding.h"
31 #include "vmwgfx_bo.h"
32 #include "vmwgfx_drv.h"
33 #include "vmwgfx_resource_priv.h"
34 
35 #define VMW_RES_EVICT_ERR_COUNT 10
36 
37 /**
38  * vmw_resource_mob_attach - Mark a resource as attached to its backing mob
39  * @res: The resource
40  */
vmw_resource_mob_attach(struct vmw_resource * res)41 void vmw_resource_mob_attach(struct vmw_resource *res)
42 {
43 	struct vmw_bo *gbo = res->guest_memory_bo;
44 	struct rb_node **new = &gbo->res_tree.rb_node, *parent = NULL;
45 
46 	dma_resv_assert_held(gbo->tbo.base.resv);
47 	res->used_prio = (res->res_dirty) ? res->func->dirty_prio :
48 		res->func->prio;
49 
50 	while (*new) {
51 		struct vmw_resource *this =
52 			container_of(*new, struct vmw_resource, mob_node);
53 
54 		parent = *new;
55 		new = (res->guest_memory_offset < this->guest_memory_offset) ?
56 			&((*new)->rb_left) : &((*new)->rb_right);
57 	}
58 
59 	rb_link_node(&res->mob_node, parent, new);
60 	rb_insert_color(&res->mob_node, &gbo->res_tree);
61 
62 	vmw_bo_prio_add(gbo, res->used_prio);
63 }
64 
65 /**
66  * vmw_resource_mob_detach - Mark a resource as detached from its backing mob
67  * @res: The resource
68  */
vmw_resource_mob_detach(struct vmw_resource * res)69 void vmw_resource_mob_detach(struct vmw_resource *res)
70 {
71 	struct vmw_bo *gbo = res->guest_memory_bo;
72 
73 	dma_resv_assert_held(gbo->tbo.base.resv);
74 	if (vmw_resource_mob_attached(res)) {
75 		rb_erase(&res->mob_node, &gbo->res_tree);
76 		RB_CLEAR_NODE(&res->mob_node);
77 		vmw_bo_prio_del(gbo, res->used_prio);
78 	}
79 }
80 
vmw_resource_reference(struct vmw_resource * res)81 struct vmw_resource *vmw_resource_reference(struct vmw_resource *res)
82 {
83 	kref_get(&res->kref);
84 	return res;
85 }
86 
87 struct vmw_resource *
vmw_resource_reference_unless_doomed(struct vmw_resource * res)88 vmw_resource_reference_unless_doomed(struct vmw_resource *res)
89 {
90 	return kref_get_unless_zero(&res->kref) ? res : NULL;
91 }
92 
93 /**
94  * vmw_resource_release_id - release a resource id to the id manager.
95  *
96  * @res: Pointer to the resource.
97  *
98  * Release the resource id to the resource id manager and set it to -1
99  */
vmw_resource_release_id(struct vmw_resource * res)100 void vmw_resource_release_id(struct vmw_resource *res)
101 {
102 	struct vmw_private *dev_priv = res->dev_priv;
103 	struct idr *idr = &dev_priv->res_idr[res->func->res_type];
104 
105 	spin_lock(&dev_priv->resource_lock);
106 	if (res->id != -1)
107 		idr_remove(idr, res->id);
108 	res->id = -1;
109 	spin_unlock(&dev_priv->resource_lock);
110 }
111 
vmw_resource_release(struct kref * kref)112 static void vmw_resource_release(struct kref *kref)
113 {
114 	struct vmw_resource *res =
115 	    container_of(kref, struct vmw_resource, kref);
116 	struct vmw_private *dev_priv = res->dev_priv;
117 	int id;
118 	int ret;
119 	struct idr *idr = &dev_priv->res_idr[res->func->res_type];
120 
121 	spin_lock(&dev_priv->resource_lock);
122 	list_del_init(&res->lru_head);
123 	spin_unlock(&dev_priv->resource_lock);
124 	if (res->guest_memory_bo) {
125 		struct ttm_buffer_object *bo = &res->guest_memory_bo->tbo;
126 
127 		ret = ttm_bo_reserve(bo, false, false, NULL);
128 		BUG_ON(ret);
129 		if (vmw_resource_mob_attached(res) &&
130 		    res->func->unbind != NULL) {
131 			struct ttm_validate_buffer val_buf;
132 
133 			val_buf.bo = bo;
134 			val_buf.num_shared = 0;
135 			res->func->unbind(res, false, &val_buf);
136 		}
137 		res->guest_memory_size = false;
138 		vmw_resource_mob_detach(res);
139 		if (res->dirty)
140 			res->func->dirty_free(res);
141 		if (res->coherent)
142 			vmw_bo_dirty_release(res->guest_memory_bo);
143 		ttm_bo_unreserve(bo);
144 		vmw_user_bo_unref(&res->guest_memory_bo);
145 	}
146 
147 	if (likely(res->hw_destroy != NULL)) {
148 		mutex_lock(&dev_priv->binding_mutex);
149 		vmw_binding_res_list_kill(&res->binding_head);
150 		mutex_unlock(&dev_priv->binding_mutex);
151 		res->hw_destroy(res);
152 	}
153 
154 	id = res->id;
155 	if (res->res_free != NULL)
156 		res->res_free(res);
157 	else
158 		kfree(res);
159 
160 	spin_lock(&dev_priv->resource_lock);
161 	if (id != -1)
162 		idr_remove(idr, id);
163 	spin_unlock(&dev_priv->resource_lock);
164 }
165 
vmw_resource_unreference(struct vmw_resource ** p_res)166 void vmw_resource_unreference(struct vmw_resource **p_res)
167 {
168 	struct vmw_resource *res = *p_res;
169 
170 	*p_res = NULL;
171 	kref_put(&res->kref, vmw_resource_release);
172 }
173 
174 
175 /**
176  * vmw_resource_alloc_id - release a resource id to the id manager.
177  *
178  * @res: Pointer to the resource.
179  *
180  * Allocate the lowest free resource from the resource manager, and set
181  * @res->id to that id. Returns 0 on success and -ENOMEM on failure.
182  */
vmw_resource_alloc_id(struct vmw_resource * res)183 int vmw_resource_alloc_id(struct vmw_resource *res)
184 {
185 	struct vmw_private *dev_priv = res->dev_priv;
186 	int ret;
187 	struct idr *idr = &dev_priv->res_idr[res->func->res_type];
188 
189 	BUG_ON(res->id != -1);
190 
191 	idr_preload(GFP_KERNEL);
192 	spin_lock(&dev_priv->resource_lock);
193 
194 	ret = idr_alloc(idr, res, 1, 0, GFP_NOWAIT);
195 	if (ret >= 0)
196 		res->id = ret;
197 
198 	spin_unlock(&dev_priv->resource_lock);
199 	idr_preload_end();
200 	return ret < 0 ? ret : 0;
201 }
202 
203 /**
204  * vmw_resource_init - initialize a struct vmw_resource
205  *
206  * @dev_priv:       Pointer to a device private struct.
207  * @res:            The struct vmw_resource to initialize.
208  * @delay_id:       Boolean whether to defer device id allocation until
209  *                  the first validation.
210  * @res_free:       Resource destructor.
211  * @func:           Resource function table.
212  */
vmw_resource_init(struct vmw_private * dev_priv,struct vmw_resource * res,bool delay_id,void (* res_free)(struct vmw_resource * res),const struct vmw_res_func * func)213 int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res,
214 		      bool delay_id,
215 		      void (*res_free) (struct vmw_resource *res),
216 		      const struct vmw_res_func *func)
217 {
218 	kref_init(&res->kref);
219 	res->hw_destroy = NULL;
220 	res->res_free = res_free;
221 	res->dev_priv = dev_priv;
222 	res->func = func;
223 	RB_CLEAR_NODE(&res->mob_node);
224 	INIT_LIST_HEAD(&res->lru_head);
225 	INIT_LIST_HEAD(&res->binding_head);
226 	res->id = -1;
227 	res->guest_memory_bo = NULL;
228 	res->guest_memory_offset = 0;
229 	res->guest_memory_dirty = false;
230 	res->res_dirty = false;
231 	res->coherent = false;
232 	res->used_prio = 3;
233 	res->dirty = NULL;
234 	if (delay_id)
235 		return 0;
236 	else
237 		return vmw_resource_alloc_id(res);
238 }
239 
240 
241 /**
242  * vmw_user_resource_lookup_handle - lookup a struct resource from a
243  * TTM user-space handle and perform basic type checks
244  *
245  * @dev_priv:     Pointer to a device private struct
246  * @tfile:        Pointer to a struct ttm_object_file identifying the caller
247  * @handle:       The TTM user-space handle
248  * @converter:    Pointer to an object describing the resource type
249  * @p_res:        On successful return the location pointed to will contain
250  *                a pointer to a refcounted struct vmw_resource.
251  *
252  * If the handle can't be found or is associated with an incorrect resource
253  * type, -EINVAL will be returned.
254  */
vmw_user_resource_lookup_handle(struct vmw_private * dev_priv,struct ttm_object_file * tfile,uint32_t handle,const struct vmw_user_resource_conv * converter,struct vmw_resource ** p_res)255 int vmw_user_resource_lookup_handle(struct vmw_private *dev_priv,
256 				    struct ttm_object_file *tfile,
257 				    uint32_t handle,
258 				    const struct vmw_user_resource_conv
259 				    *converter,
260 				    struct vmw_resource **p_res)
261 {
262 	struct ttm_base_object *base;
263 	struct vmw_resource *res;
264 	int ret = -EINVAL;
265 
266 	base = ttm_base_object_lookup(tfile, handle);
267 	if (unlikely(!base))
268 		return -EINVAL;
269 
270 	if (unlikely(ttm_base_object_type(base) != converter->object_type))
271 		goto out_bad_resource;
272 
273 	res = converter->base_obj_to_res(base);
274 	kref_get(&res->kref);
275 
276 	*p_res = res;
277 	ret = 0;
278 
279 out_bad_resource:
280 	ttm_base_object_unref(&base);
281 
282 	return ret;
283 }
284 
285 /*
286  * Helper function that looks either a surface or bo.
287  *
288  * The pointer this pointed at by out_surf and out_buf needs to be null.
289  */
vmw_user_lookup_handle(struct vmw_private * dev_priv,struct drm_file * filp,uint32_t handle,struct vmw_surface ** out_surf,struct vmw_bo ** out_buf)290 int vmw_user_lookup_handle(struct vmw_private *dev_priv,
291 			   struct drm_file *filp,
292 			   uint32_t handle,
293 			   struct vmw_surface **out_surf,
294 			   struct vmw_bo **out_buf)
295 {
296 	struct ttm_object_file *tfile = vmw_fpriv(filp)->tfile;
297 	struct vmw_resource *res;
298 	int ret;
299 
300 	BUG_ON(*out_surf || *out_buf);
301 
302 	ret = vmw_user_resource_lookup_handle(dev_priv, tfile, handle,
303 					      user_surface_converter,
304 					      &res);
305 	if (!ret) {
306 		*out_surf = vmw_res_to_srf(res);
307 		return 0;
308 	}
309 
310 	*out_surf = NULL;
311 	ret = vmw_user_bo_lookup(filp, handle, out_buf);
312 	return ret;
313 }
314 
315 /**
316  * vmw_resource_buf_alloc - Allocate a guest memory buffer for a resource.
317  *
318  * @res:            The resource for which to allocate a gbo buffer.
319  * @interruptible:  Whether any sleeps during allocation should be
320  *                  performed while interruptible.
321  */
vmw_resource_buf_alloc(struct vmw_resource * res,bool interruptible)322 static int vmw_resource_buf_alloc(struct vmw_resource *res,
323 				  bool interruptible)
324 {
325 	unsigned long size = PFN_ALIGN(res->guest_memory_size);
326 	struct vmw_bo *gbo;
327 	struct vmw_bo_params bo_params = {
328 		.domain = res->func->domain,
329 		.busy_domain = res->func->busy_domain,
330 		.bo_type = ttm_bo_type_device,
331 		.size = res->guest_memory_size,
332 		.pin = false
333 	};
334 	int ret;
335 
336 	if (likely(res->guest_memory_bo)) {
337 		BUG_ON(res->guest_memory_bo->tbo.base.size < size);
338 		return 0;
339 	}
340 
341 	ret = vmw_gem_object_create(res->dev_priv, &bo_params, &gbo);
342 	if (unlikely(ret != 0))
343 		goto out_no_bo;
344 
345 	res->guest_memory_bo = gbo;
346 
347 out_no_bo:
348 	return ret;
349 }
350 
351 /**
352  * vmw_resource_do_validate - Make a resource up-to-date and visible
353  *                            to the device.
354  *
355  * @res:            The resource to make visible to the device.
356  * @val_buf:        Information about a buffer possibly
357  *                  containing backup data if a bind operation is needed.
358  * @dirtying:       Transfer dirty regions.
359  *
360  * On hardware resource shortage, this function returns -EBUSY and
361  * should be retried once resources have been freed up.
362  */
vmw_resource_do_validate(struct vmw_resource * res,struct ttm_validate_buffer * val_buf,bool dirtying)363 static int vmw_resource_do_validate(struct vmw_resource *res,
364 				    struct ttm_validate_buffer *val_buf,
365 				    bool dirtying)
366 {
367 	int ret = 0;
368 	const struct vmw_res_func *func = res->func;
369 
370 	if (unlikely(res->id == -1)) {
371 		ret = func->create(res);
372 		if (unlikely(ret != 0))
373 			return ret;
374 	}
375 
376 	if (func->bind &&
377 	    ((func->needs_guest_memory && !vmw_resource_mob_attached(res) &&
378 	      val_buf->bo) ||
379 	     (!func->needs_guest_memory && val_buf->bo))) {
380 		ret = func->bind(res, val_buf);
381 		if (unlikely(ret != 0))
382 			goto out_bind_failed;
383 		if (func->needs_guest_memory)
384 			vmw_resource_mob_attach(res);
385 	}
386 
387 	/*
388 	 * Handle the case where the backup mob is marked coherent but
389 	 * the resource isn't.
390 	 */
391 	if (func->dirty_alloc && vmw_resource_mob_attached(res) &&
392 	    !res->coherent) {
393 		if (res->guest_memory_bo->dirty && !res->dirty) {
394 			ret = func->dirty_alloc(res);
395 			if (ret)
396 				return ret;
397 		} else if (!res->guest_memory_bo->dirty && res->dirty) {
398 			func->dirty_free(res);
399 		}
400 	}
401 
402 	/*
403 	 * Transfer the dirty regions to the resource and update
404 	 * the resource.
405 	 */
406 	if (res->dirty) {
407 		if (dirtying && !res->res_dirty) {
408 			pgoff_t start = res->guest_memory_offset >> PAGE_SHIFT;
409 			pgoff_t end = __KERNEL_DIV_ROUND_UP
410 				(res->guest_memory_offset + res->guest_memory_size,
411 				 PAGE_SIZE);
412 
413 			vmw_bo_dirty_unmap(res->guest_memory_bo, start, end);
414 		}
415 
416 		vmw_bo_dirty_transfer_to_res(res);
417 		return func->dirty_sync(res);
418 	}
419 
420 	return 0;
421 
422 out_bind_failed:
423 	func->destroy(res);
424 
425 	return ret;
426 }
427 
428 /**
429  * vmw_resource_unreserve - Unreserve a resource previously reserved for
430  * command submission.
431  *
432  * @res:               Pointer to the struct vmw_resource to unreserve.
433  * @dirty_set:         Change dirty status of the resource.
434  * @dirty:             When changing dirty status indicates the new status.
435  * @switch_guest_memory: Guest memory buffer has been switched.
436  * @new_guest_memory_bo: Pointer to new guest memory buffer if command submission
437  *                     switched. May be NULL.
438  * @new_guest_memory_offset: New gbo offset if @switch_guest_memory is true.
439  *
440  * Currently unreserving a resource means putting it back on the device's
441  * resource lru list, so that it can be evicted if necessary.
442  */
vmw_resource_unreserve(struct vmw_resource * res,bool dirty_set,bool dirty,bool switch_guest_memory,struct vmw_bo * new_guest_memory_bo,unsigned long new_guest_memory_offset)443 void vmw_resource_unreserve(struct vmw_resource *res,
444 			    bool dirty_set,
445 			    bool dirty,
446 			    bool switch_guest_memory,
447 			    struct vmw_bo *new_guest_memory_bo,
448 			    unsigned long new_guest_memory_offset)
449 {
450 	struct vmw_private *dev_priv = res->dev_priv;
451 
452 	if (!list_empty(&res->lru_head))
453 		return;
454 
455 	if (switch_guest_memory && new_guest_memory_bo != res->guest_memory_bo) {
456 		if (res->guest_memory_bo) {
457 			vmw_resource_mob_detach(res);
458 			if (res->coherent)
459 				vmw_bo_dirty_release(res->guest_memory_bo);
460 			vmw_user_bo_unref(&res->guest_memory_bo);
461 		}
462 
463 		if (new_guest_memory_bo) {
464 			res->guest_memory_bo = vmw_user_bo_ref(new_guest_memory_bo);
465 
466 			/*
467 			 * The validation code should already have added a
468 			 * dirty tracker here.
469 			 */
470 			WARN_ON(res->coherent && !new_guest_memory_bo->dirty);
471 
472 			vmw_resource_mob_attach(res);
473 		} else {
474 			res->guest_memory_bo = NULL;
475 		}
476 	} else if (switch_guest_memory && res->coherent) {
477 		vmw_bo_dirty_release(res->guest_memory_bo);
478 	}
479 
480 	if (switch_guest_memory)
481 		res->guest_memory_offset = new_guest_memory_offset;
482 
483 	if (dirty_set)
484 		res->res_dirty = dirty;
485 
486 	if (!res->func->may_evict || res->id == -1 || res->pin_count)
487 		return;
488 
489 	spin_lock(&dev_priv->resource_lock);
490 	list_add_tail(&res->lru_head,
491 		      &res->dev_priv->res_lru[res->func->res_type]);
492 	spin_unlock(&dev_priv->resource_lock);
493 }
494 
495 /**
496  * vmw_resource_check_buffer - Check whether a backup buffer is needed
497  *                             for a resource and in that case, allocate
498  *                             one, reserve and validate it.
499  *
500  * @ticket:         The ww acquire context to use, or NULL if trylocking.
501  * @res:            The resource for which to allocate a backup buffer.
502  * @interruptible:  Whether any sleeps during allocation should be
503  *                  performed while interruptible.
504  * @val_buf:        On successful return contains data about the
505  *                  reserved and validated backup buffer.
506  */
507 static int
vmw_resource_check_buffer(struct ww_acquire_ctx * ticket,struct vmw_resource * res,bool interruptible,struct ttm_validate_buffer * val_buf)508 vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,
509 			  struct vmw_resource *res,
510 			  bool interruptible,
511 			  struct ttm_validate_buffer *val_buf)
512 {
513 	struct ttm_operation_ctx ctx = { true, false };
514 	struct list_head val_list;
515 	bool guest_memory_dirty = false;
516 	int ret;
517 
518 	if (unlikely(!res->guest_memory_bo)) {
519 		ret = vmw_resource_buf_alloc(res, interruptible);
520 		if (unlikely(ret != 0))
521 			return ret;
522 	}
523 
524 	INIT_LIST_HEAD(&val_list);
525 	ttm_bo_get(&res->guest_memory_bo->tbo);
526 	val_buf->bo = &res->guest_memory_bo->tbo;
527 	val_buf->num_shared = 0;
528 	list_add_tail(&val_buf->head, &val_list);
529 	ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL);
530 	if (unlikely(ret != 0))
531 		goto out_no_reserve;
532 
533 	if (res->func->needs_guest_memory && !vmw_resource_mob_attached(res))
534 		return 0;
535 
536 	guest_memory_dirty = res->guest_memory_dirty;
537 	vmw_bo_placement_set(res->guest_memory_bo, res->func->domain,
538 			     res->func->busy_domain);
539 	ret = ttm_bo_validate(&res->guest_memory_bo->tbo,
540 			      &res->guest_memory_bo->placement,
541 			      &ctx);
542 
543 	if (unlikely(ret != 0))
544 		goto out_no_validate;
545 
546 	return 0;
547 
548 out_no_validate:
549 	ttm_eu_backoff_reservation(ticket, &val_list);
550 out_no_reserve:
551 	ttm_bo_put(val_buf->bo);
552 	val_buf->bo = NULL;
553 	if (guest_memory_dirty)
554 		vmw_user_bo_unref(&res->guest_memory_bo);
555 
556 	return ret;
557 }
558 
559 /*
560  * vmw_resource_reserve - Reserve a resource for command submission
561  *
562  * @res:            The resource to reserve.
563  *
564  * This function takes the resource off the LRU list and make sure
565  * a guest memory buffer is present for guest-backed resources.
566  * However, the buffer may not be bound to the resource at this
567  * point.
568  *
569  */
vmw_resource_reserve(struct vmw_resource * res,bool interruptible,bool no_guest_memory)570 int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
571 			 bool no_guest_memory)
572 {
573 	struct vmw_private *dev_priv = res->dev_priv;
574 	int ret;
575 
576 	spin_lock(&dev_priv->resource_lock);
577 	list_del_init(&res->lru_head);
578 	spin_unlock(&dev_priv->resource_lock);
579 
580 	if (res->func->needs_guest_memory && !res->guest_memory_bo &&
581 	    !no_guest_memory) {
582 		ret = vmw_resource_buf_alloc(res, interruptible);
583 		if (unlikely(ret != 0)) {
584 			DRM_ERROR("Failed to allocate a guest memory buffer "
585 				  "of size %lu. bytes\n",
586 				  (unsigned long) res->guest_memory_size);
587 			return ret;
588 		}
589 	}
590 
591 	return 0;
592 }
593 
594 /**
595  * vmw_resource_backoff_reservation - Unreserve and unreference a
596  *                                    guest memory buffer
597  *.
598  * @ticket:         The ww acquire ctx used for reservation.
599  * @val_buf:        Guest memory buffer information.
600  */
601 static void
vmw_resource_backoff_reservation(struct ww_acquire_ctx * ticket,struct ttm_validate_buffer * val_buf)602 vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
603 				 struct ttm_validate_buffer *val_buf)
604 {
605 	struct list_head val_list;
606 
607 	if (likely(val_buf->bo == NULL))
608 		return;
609 
610 	INIT_LIST_HEAD(&val_list);
611 	list_add_tail(&val_buf->head, &val_list);
612 	ttm_eu_backoff_reservation(ticket, &val_list);
613 	ttm_bo_put(val_buf->bo);
614 	val_buf->bo = NULL;
615 }
616 
617 /**
618  * vmw_resource_do_evict - Evict a resource, and transfer its data
619  *                         to a backup buffer.
620  *
621  * @ticket:         The ww acquire ticket to use, or NULL if trylocking.
622  * @res:            The resource to evict.
623  * @interruptible:  Whether to wait interruptible.
624  */
vmw_resource_do_evict(struct ww_acquire_ctx * ticket,struct vmw_resource * res,bool interruptible)625 static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,
626 				 struct vmw_resource *res, bool interruptible)
627 {
628 	struct ttm_validate_buffer val_buf;
629 	const struct vmw_res_func *func = res->func;
630 	int ret;
631 
632 	BUG_ON(!func->may_evict);
633 
634 	val_buf.bo = NULL;
635 	val_buf.num_shared = 0;
636 	ret = vmw_resource_check_buffer(ticket, res, interruptible, &val_buf);
637 	if (unlikely(ret != 0))
638 		return ret;
639 
640 	if (unlikely(func->unbind != NULL &&
641 		     (!func->needs_guest_memory || vmw_resource_mob_attached(res)))) {
642 		ret = func->unbind(res, res->res_dirty, &val_buf);
643 		if (unlikely(ret != 0))
644 			goto out_no_unbind;
645 		vmw_resource_mob_detach(res);
646 	}
647 	ret = func->destroy(res);
648 	res->guest_memory_dirty = true;
649 	res->res_dirty = false;
650 out_no_unbind:
651 	vmw_resource_backoff_reservation(ticket, &val_buf);
652 
653 	return ret;
654 }
655 
656 
657 /**
658  * vmw_resource_validate - Make a resource up-to-date and visible
659  *                         to the device.
660  * @res: The resource to make visible to the device.
661  * @intr: Perform waits interruptible if possible.
662  * @dirtying: Pending GPU operation will dirty the resource
663  *
664  * On successful return, any backup DMA buffer pointed to by @res->backup will
665  * be reserved and validated.
666  * On hardware resource shortage, this function will repeatedly evict
667  * resources of the same type until the validation succeeds.
668  *
669  * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code
670  * on failure.
671  */
vmw_resource_validate(struct vmw_resource * res,bool intr,bool dirtying)672 int vmw_resource_validate(struct vmw_resource *res, bool intr,
673 			  bool dirtying)
674 {
675 	int ret;
676 	struct vmw_resource *evict_res;
677 	struct vmw_private *dev_priv = res->dev_priv;
678 	struct list_head *lru_list = &dev_priv->res_lru[res->func->res_type];
679 	struct ttm_validate_buffer val_buf;
680 	unsigned err_count = 0;
681 
682 	if (!res->func->create)
683 		return 0;
684 
685 	val_buf.bo = NULL;
686 	val_buf.num_shared = 0;
687 	if (res->guest_memory_bo)
688 		val_buf.bo = &res->guest_memory_bo->tbo;
689 	do {
690 		ret = vmw_resource_do_validate(res, &val_buf, dirtying);
691 		if (likely(ret != -EBUSY))
692 			break;
693 
694 		spin_lock(&dev_priv->resource_lock);
695 		if (list_empty(lru_list) || !res->func->may_evict) {
696 			DRM_ERROR("Out of device device resources "
697 				  "for %s.\n", res->func->type_name);
698 			ret = -EBUSY;
699 			spin_unlock(&dev_priv->resource_lock);
700 			break;
701 		}
702 
703 		evict_res = vmw_resource_reference
704 			(list_first_entry(lru_list, struct vmw_resource,
705 					  lru_head));
706 		list_del_init(&evict_res->lru_head);
707 
708 		spin_unlock(&dev_priv->resource_lock);
709 
710 		/* Trylock backup buffers with a NULL ticket. */
711 		ret = vmw_resource_do_evict(NULL, evict_res, intr);
712 		if (unlikely(ret != 0)) {
713 			spin_lock(&dev_priv->resource_lock);
714 			list_add_tail(&evict_res->lru_head, lru_list);
715 			spin_unlock(&dev_priv->resource_lock);
716 			if (ret == -ERESTARTSYS ||
717 			    ++err_count > VMW_RES_EVICT_ERR_COUNT) {
718 				vmw_resource_unreference(&evict_res);
719 				goto out_no_validate;
720 			}
721 		}
722 
723 		vmw_resource_unreference(&evict_res);
724 	} while (1);
725 
726 	if (unlikely(ret != 0))
727 		goto out_no_validate;
728 	else if (!res->func->needs_guest_memory && res->guest_memory_bo) {
729 		WARN_ON_ONCE(vmw_resource_mob_attached(res));
730 		vmw_user_bo_unref(&res->guest_memory_bo);
731 	}
732 
733 	return 0;
734 
735 out_no_validate:
736 	return ret;
737 }
738 
739 
740 /**
741  * vmw_resource_unbind_list
742  *
743  * @vbo: Pointer to the current backing MOB.
744  *
745  * Evicts the Guest Backed hardware resource if the backup
746  * buffer is being moved out of MOB memory.
747  * Note that this function will not race with the resource
748  * validation code, since resource validation and eviction
749  * both require the backup buffer to be reserved.
750  */
vmw_resource_unbind_list(struct vmw_bo * vbo)751 void vmw_resource_unbind_list(struct vmw_bo *vbo)
752 {
753 	struct ttm_validate_buffer val_buf = {
754 		.bo = &vbo->tbo,
755 		.num_shared = 0
756 	};
757 
758 	dma_resv_assert_held(vbo->tbo.base.resv);
759 	while (!RB_EMPTY_ROOT(&vbo->res_tree)) {
760 		struct rb_node *node = vbo->res_tree.rb_node;
761 		struct vmw_resource *res =
762 			container_of(node, struct vmw_resource, mob_node);
763 
764 		if (!WARN_ON_ONCE(!res->func->unbind))
765 			(void) res->func->unbind(res, res->res_dirty, &val_buf);
766 
767 		res->guest_memory_size = true;
768 		res->res_dirty = false;
769 		vmw_resource_mob_detach(res);
770 	}
771 
772 	(void) ttm_bo_wait(&vbo->tbo, false, false);
773 }
774 
775 
776 /**
777  * vmw_query_readback_all - Read back cached query states
778  *
779  * @dx_query_mob: Buffer containing the DX query MOB
780  *
781  * Read back cached states from the device if they exist.  This function
782  * assumes binding_mutex is held.
783  */
vmw_query_readback_all(struct vmw_bo * dx_query_mob)784 int vmw_query_readback_all(struct vmw_bo *dx_query_mob)
785 {
786 	struct vmw_resource *dx_query_ctx;
787 	struct vmw_private *dev_priv;
788 	struct {
789 		SVGA3dCmdHeader header;
790 		SVGA3dCmdDXReadbackAllQuery body;
791 	} *cmd;
792 
793 
794 	/* No query bound, so do nothing */
795 	if (!dx_query_mob || !dx_query_mob->dx_query_ctx)
796 		return 0;
797 
798 	dx_query_ctx = dx_query_mob->dx_query_ctx;
799 	dev_priv     = dx_query_ctx->dev_priv;
800 
801 	cmd = VMW_CMD_CTX_RESERVE(dev_priv, sizeof(*cmd), dx_query_ctx->id);
802 	if (unlikely(cmd == NULL))
803 		return -ENOMEM;
804 
805 	cmd->header.id   = SVGA_3D_CMD_DX_READBACK_ALL_QUERY;
806 	cmd->header.size = sizeof(cmd->body);
807 	cmd->body.cid    = dx_query_ctx->id;
808 
809 	vmw_cmd_commit(dev_priv, sizeof(*cmd));
810 
811 	/* Triggers a rebind the next time affected context is bound */
812 	dx_query_mob->dx_query_ctx = NULL;
813 
814 	return 0;
815 }
816 
817 
818 
819 /**
820  * vmw_query_move_notify - Read back cached query states
821  *
822  * @bo: The TTM buffer object about to move.
823  * @old_mem: The memory region @bo is moving from.
824  * @new_mem: The memory region @bo is moving to.
825  *
826  * Called before the query MOB is swapped out to read back cached query
827  * states from the device.
828  */
vmw_query_move_notify(struct ttm_buffer_object * bo,struct ttm_resource * old_mem,struct ttm_resource * new_mem)829 void vmw_query_move_notify(struct ttm_buffer_object *bo,
830 			   struct ttm_resource *old_mem,
831 			   struct ttm_resource *new_mem)
832 {
833 	struct vmw_bo *dx_query_mob;
834 	struct ttm_device *bdev = bo->bdev;
835 	struct vmw_private *dev_priv = vmw_priv_from_ttm(bdev);
836 
837 	mutex_lock(&dev_priv->binding_mutex);
838 
839 	/* If BO is being moved from MOB to system memory */
840 	if (old_mem &&
841 	    new_mem->mem_type == TTM_PL_SYSTEM &&
842 	    old_mem->mem_type == VMW_PL_MOB) {
843 		struct vmw_fence_obj *fence;
844 
845 		dx_query_mob = to_vmw_bo(&bo->base);
846 		if (!dx_query_mob || !dx_query_mob->dx_query_ctx) {
847 			mutex_unlock(&dev_priv->binding_mutex);
848 			return;
849 		}
850 
851 		(void) vmw_query_readback_all(dx_query_mob);
852 		mutex_unlock(&dev_priv->binding_mutex);
853 
854 		/* Create a fence and attach the BO to it */
855 		(void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
856 		vmw_bo_fence_single(bo, fence);
857 
858 		if (fence != NULL)
859 			vmw_fence_obj_unreference(&fence);
860 
861 		(void) ttm_bo_wait(bo, false, false);
862 	} else
863 		mutex_unlock(&dev_priv->binding_mutex);
864 }
865 
866 /**
867  * vmw_resource_needs_backup - Return whether a resource needs a backup buffer.
868  *
869  * @res:            The resource being queried.
870  */
vmw_resource_needs_backup(const struct vmw_resource * res)871 bool vmw_resource_needs_backup(const struct vmw_resource *res)
872 {
873 	return res->func->needs_guest_memory;
874 }
875 
876 /**
877  * vmw_resource_evict_type - Evict all resources of a specific type
878  *
879  * @dev_priv:       Pointer to a device private struct
880  * @type:           The resource type to evict
881  *
882  * To avoid thrashing starvation or as part of the hibernation sequence,
883  * try to evict all evictable resources of a specific type.
884  */
vmw_resource_evict_type(struct vmw_private * dev_priv,enum vmw_res_type type)885 static void vmw_resource_evict_type(struct vmw_private *dev_priv,
886 				    enum vmw_res_type type)
887 {
888 	struct list_head *lru_list = &dev_priv->res_lru[type];
889 	struct vmw_resource *evict_res;
890 	unsigned err_count = 0;
891 	int ret;
892 	struct ww_acquire_ctx ticket;
893 
894 	do {
895 		spin_lock(&dev_priv->resource_lock);
896 
897 		if (list_empty(lru_list))
898 			goto out_unlock;
899 
900 		evict_res = vmw_resource_reference(
901 			list_first_entry(lru_list, struct vmw_resource,
902 					 lru_head));
903 		list_del_init(&evict_res->lru_head);
904 		spin_unlock(&dev_priv->resource_lock);
905 
906 		/* Wait lock backup buffers with a ticket. */
907 		ret = vmw_resource_do_evict(&ticket, evict_res, false);
908 		if (unlikely(ret != 0)) {
909 			spin_lock(&dev_priv->resource_lock);
910 			list_add_tail(&evict_res->lru_head, lru_list);
911 			spin_unlock(&dev_priv->resource_lock);
912 			if (++err_count > VMW_RES_EVICT_ERR_COUNT) {
913 				vmw_resource_unreference(&evict_res);
914 				return;
915 			}
916 		}
917 
918 		vmw_resource_unreference(&evict_res);
919 	} while (1);
920 
921 out_unlock:
922 	spin_unlock(&dev_priv->resource_lock);
923 }
924 
925 /**
926  * vmw_resource_evict_all - Evict all evictable resources
927  *
928  * @dev_priv:       Pointer to a device private struct
929  *
930  * To avoid thrashing starvation or as part of the hibernation sequence,
931  * evict all evictable resources. In particular this means that all
932  * guest-backed resources that are registered with the device are
933  * evicted and the OTable becomes clean.
934  */
vmw_resource_evict_all(struct vmw_private * dev_priv)935 void vmw_resource_evict_all(struct vmw_private *dev_priv)
936 {
937 	enum vmw_res_type type;
938 
939 	mutex_lock(&dev_priv->cmdbuf_mutex);
940 
941 	for (type = 0; type < vmw_res_max; ++type)
942 		vmw_resource_evict_type(dev_priv, type);
943 
944 	mutex_unlock(&dev_priv->cmdbuf_mutex);
945 }
946 
947 /*
948  * vmw_resource_pin - Add a pin reference on a resource
949  *
950  * @res: The resource to add a pin reference on
951  *
952  * This function adds a pin reference, and if needed validates the resource.
953  * Having a pin reference means that the resource can never be evicted, and
954  * its id will never change as long as there is a pin reference.
955  * This function returns 0 on success and a negative error code on failure.
956  */
vmw_resource_pin(struct vmw_resource * res,bool interruptible)957 int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
958 {
959 	struct ttm_operation_ctx ctx = { interruptible, false };
960 	struct vmw_private *dev_priv = res->dev_priv;
961 	int ret;
962 
963 	mutex_lock(&dev_priv->cmdbuf_mutex);
964 	ret = vmw_resource_reserve(res, interruptible, false);
965 	if (ret)
966 		goto out_no_reserve;
967 
968 	if (res->pin_count == 0) {
969 		struct vmw_bo *vbo = NULL;
970 
971 		if (res->guest_memory_bo) {
972 			vbo = res->guest_memory_bo;
973 
974 			ret = ttm_bo_reserve(&vbo->tbo, interruptible, false, NULL);
975 			if (ret)
976 				goto out_no_validate;
977 			if (!vbo->tbo.pin_count) {
978 				vmw_bo_placement_set(vbo,
979 						     res->func->domain,
980 						     res->func->busy_domain);
981 				ret = ttm_bo_validate
982 					(&vbo->tbo,
983 					 &vbo->placement,
984 					 &ctx);
985 				if (ret) {
986 					ttm_bo_unreserve(&vbo->tbo);
987 					goto out_no_validate;
988 				}
989 			}
990 
991 			/* Do we really need to pin the MOB as well? */
992 			vmw_bo_pin_reserved(vbo, true);
993 		}
994 		ret = vmw_resource_validate(res, interruptible, true);
995 		if (vbo)
996 			ttm_bo_unreserve(&vbo->tbo);
997 		if (ret)
998 			goto out_no_validate;
999 	}
1000 	res->pin_count++;
1001 
1002 out_no_validate:
1003 	vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
1004 out_no_reserve:
1005 	mutex_unlock(&dev_priv->cmdbuf_mutex);
1006 
1007 	return ret;
1008 }
1009 
1010 /**
1011  * vmw_resource_unpin - Remove a pin reference from a resource
1012  *
1013  * @res: The resource to remove a pin reference from
1014  *
1015  * Having a pin reference means that the resource can never be evicted, and
1016  * its id will never change as long as there is a pin reference.
1017  */
vmw_resource_unpin(struct vmw_resource * res)1018 void vmw_resource_unpin(struct vmw_resource *res)
1019 {
1020 	struct vmw_private *dev_priv = res->dev_priv;
1021 	int ret;
1022 
1023 	mutex_lock(&dev_priv->cmdbuf_mutex);
1024 
1025 	ret = vmw_resource_reserve(res, false, true);
1026 	WARN_ON(ret);
1027 
1028 	WARN_ON(res->pin_count == 0);
1029 	if (--res->pin_count == 0 && res->guest_memory_bo) {
1030 		struct vmw_bo *vbo = res->guest_memory_bo;
1031 
1032 		(void) ttm_bo_reserve(&vbo->tbo, false, false, NULL);
1033 		vmw_bo_pin_reserved(vbo, false);
1034 		ttm_bo_unreserve(&vbo->tbo);
1035 	}
1036 
1037 	vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
1038 
1039 	mutex_unlock(&dev_priv->cmdbuf_mutex);
1040 }
1041 
1042 /**
1043  * vmw_res_type - Return the resource type
1044  *
1045  * @res: Pointer to the resource
1046  */
vmw_res_type(const struct vmw_resource * res)1047 enum vmw_res_type vmw_res_type(const struct vmw_resource *res)
1048 {
1049 	return res->func->res_type;
1050 }
1051 
1052 /**
1053  * vmw_resource_dirty_update - Update a resource's dirty tracker with a
1054  * sequential range of touched backing store memory.
1055  * @res: The resource.
1056  * @start: The first page touched.
1057  * @end: The last page touched + 1.
1058  */
vmw_resource_dirty_update(struct vmw_resource * res,pgoff_t start,pgoff_t end)1059 void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
1060 			       pgoff_t end)
1061 {
1062 	if (res->dirty)
1063 		res->func->dirty_range_add(res, start << PAGE_SHIFT,
1064 					   end << PAGE_SHIFT);
1065 }
1066 
1067 /**
1068  * vmw_resources_clean - Clean resources intersecting a mob range
1069  * @vbo: The mob buffer object
1070  * @start: The mob page offset starting the range
1071  * @end: The mob page offset ending the range
1072  * @num_prefault: Returns how many pages including the first have been
1073  * cleaned and are ok to prefault
1074  */
vmw_resources_clean(struct vmw_bo * vbo,pgoff_t start,pgoff_t end,pgoff_t * num_prefault)1075 int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,
1076 			pgoff_t end, pgoff_t *num_prefault)
1077 {
1078 	struct rb_node *cur = vbo->res_tree.rb_node;
1079 	struct vmw_resource *found = NULL;
1080 	unsigned long res_start = start << PAGE_SHIFT;
1081 	unsigned long res_end = end << PAGE_SHIFT;
1082 	unsigned long last_cleaned = 0;
1083 
1084 	/*
1085 	 * Find the resource with lowest backup_offset that intersects the
1086 	 * range.
1087 	 */
1088 	while (cur) {
1089 		struct vmw_resource *cur_res =
1090 			container_of(cur, struct vmw_resource, mob_node);
1091 
1092 		if (cur_res->guest_memory_offset >= res_end) {
1093 			cur = cur->rb_left;
1094 		} else if (cur_res->guest_memory_offset + cur_res->guest_memory_size <=
1095 			   res_start) {
1096 			cur = cur->rb_right;
1097 		} else {
1098 			found = cur_res;
1099 			cur = cur->rb_left;
1100 			/* Continue to look for resources with lower offsets */
1101 		}
1102 	}
1103 
1104 	/*
1105 	 * In order of increasing guest_memory_offset, clean dirty resources
1106 	 * intersecting the range.
1107 	 */
1108 	while (found) {
1109 		if (found->res_dirty) {
1110 			int ret;
1111 
1112 			if (!found->func->clean)
1113 				return -EINVAL;
1114 
1115 			ret = found->func->clean(found);
1116 			if (ret)
1117 				return ret;
1118 
1119 			found->res_dirty = false;
1120 		}
1121 		last_cleaned = found->guest_memory_offset + found->guest_memory_size;
1122 		cur = rb_next(&found->mob_node);
1123 		if (!cur)
1124 			break;
1125 
1126 		found = container_of(cur, struct vmw_resource, mob_node);
1127 		if (found->guest_memory_offset >= res_end)
1128 			break;
1129 	}
1130 
1131 	/*
1132 	 * Set number of pages allowed prefaulting and fence the buffer object
1133 	 */
1134 	*num_prefault = 1;
1135 	if (last_cleaned > res_start) {
1136 		struct ttm_buffer_object *bo = &vbo->tbo;
1137 
1138 		*num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start,
1139 						      PAGE_SIZE);
1140 		vmw_bo_fence_single(bo, NULL);
1141 	}
1142 
1143 	return 0;
1144 }
1145